fraim 2.0.338 → 2.0.340

package/dist/src/local-mcp-server/learning-usage-store.js

@@ -1,17 +1,26 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CLOCK_ADVANCING_OUTCOMES = exports.FIRING_OUTCOMES = void 0;
+exports.appendEvents = exports.listEventLogs = exports.resolveMachineId = exports.resolveEventDir = exports.AMBIENT_JOB_LABELS = exports.CLOCK_ADVANCING_OUTCOMES = exports.FIRING_OUTCOMES = void 0;
 exports.resolveUsageStorePath = resolveUsageStorePath;
 exports.resolveUsageLimits = resolveUsageLimits;
+exports.defaultUsageLimits = defaultUsageLimits;
 exports.normalizeEntryTitle = normalizeEntryTitle;
 exports.titleUsageKey = titleUsageKey;
 exports.ruleUsageKey = ruleUsageKey;
+exports.resolveEntryIdentity = resolveEntryIdentity;
+exports.normalizeModelName = normalizeModelName;
+exports.readUsageSnapshot = readUsageSnapshot;
+exports.resolveUsageCheckpointPath = resolveUsageCheckpointPath;
+exports.writeUsageSnapshot = writeUsageSnapshot;
+exports.foldUsageEvents = foldUsageEvents;
 exports.readUsageStore = readUsageStore;
 exports.sanitizeNote = sanitizeNote;
+exports.integrityViolation = integrityViolation;
 exports.recordOffers = recordOffers;
 exports.recordFirings = recordFirings;
 exports.noteTitleChange = noteTitleChange;
 exports.pruneUsageStore = pruneUsageStore;
+exports.compactUsageEvents = compactUsageEvents;
 /**
  * Issue #1103 — the usage record for learnings and rules.
  *
@@ -38,15 +47,39 @@ exports.pruneUsageStore = pruneUsageStore;
  * (`src/ai-hub/server.ts`). That placement is what satisfies R18; there is no
  * second guardrail here.
  *
- * It is derived data (R17): deleting it loses no lesson, because every firing is
- * still written in the retrospective that attested it and every offer is
- * re-recorded the next time the entry is delivered.
+ * Sleep consolidates the append-only journal into learning-usage.json and its
+ * recoverable .checkpoint copy before clearing processed events. Those snapshots
+ * preserve lifetime aggregates and source bookmarks after journal cleanup.
+ * Raw retrospectives preserve lessons and attestations, not every runtime offer.
+ *
+ * Four smaller corrections ride with it, each removing a place where the record
+ * gave a confident answer it could not prove:
+ *
+ *   A1  `resolveEntryIdentity` is the one definition of what an entry is. Four
+ *       call sites each built `id ?? titleUsageKey(...)` independently, so one
+ *       lesson could be two records.
+ *   C2  Model names are lowercased at write. `GPT-5` and `gpt-5` were two buckets
+ *       splitting one denominator (measured 6857 against 2652).
+ *   D3  Job-labelled offers are reserved against eviction by session traffic.
+ *       2067 of 2354 retained offer records were `session` or `__freeform__`, so
+ *       "was this in context for that job" had almost no resolution left.
+ *   E   A scoped integrity invariant. The literal `fired <= offered` the issue
+ *       asks for is unsound, because `runBackfill` legitimately writes firings
+ *       for 299 retrospectives that predate offer recording. Scoped to attested
+ *       firings dated at or after the store's first recorded offer, it holds: the
+ *       CLI throws on a violation, and the injection path only flags, because a
+ *       usage failure must never change what the agent is given (R2).
  */
 const fs_1 = require("fs");
 const path_1 = require("path");
 const pack_home_1 = require("../cli/utils/pack-home");
 const artifact_retention_cleanup_1 = require("./artifact-retention-cleanup");
 const project_fraim_paths_1 = require("../core/utils/project-fraim-paths");
+const learning_usage_events_1 = require("./learning-usage-events");
+Object.defineProperty(exports, "appendEvents", { enumerable: true, get: function () { return learning_usage_events_1.appendEvents; } });
+Object.defineProperty(exports, "listEventLogs", { enumerable: true, get: function () { return learning_usage_events_1.listEventLogs; } });
+Object.defineProperty(exports, "resolveEventDir", { enumerable: true, get: function () { return learning_usage_events_1.resolveEventDir; } });
+Object.defineProperty(exports, "resolveMachineId", { enumerable: true, get: function () { return learning_usage_events_1.resolveMachineId; } });
 // ── Vocabulary ───────────────────────────────────────────────────────────────
 /**
  * The five outcomes (spec: Outcome vocabulary). `ignored` and `not-offered` are
@@ -69,10 +102,29 @@ const DEFAULT_FIRING_LOG_LIMIT = 20;
  * needs per-job offer detail rather than a counter.
  */
 const DEFAULT_OFFER_LOG_LIMIT = 50;
+/**
+ * Issue #1752 D3. Of the offer log's slots, this many are held for records that
+ * carry a real job label, so `fraim_connect`'s per-session traffic cannot evict
+ * the only records that can answer R7.
+ */
+const DEFAULT_JOB_OFFER_RESERVE = 25;
 /** Open Question 2: configurable and greater than a handful. */
 const DEFAULT_RETIREMENT_OFFER_THRESHOLD = 10;
+/**
+ * Job labels the runtime writes when no job is selected. An offer carrying one of
+ * these says the entry was in a session's context, not that it was in context for
+ * any particular job, so it is not what R7 resolves against.
+ */
+exports.AMBIENT_JOB_LABELS = ['session', '__freeform__', 'unknown', ''];
 /** R16: one line, bounded, no fences. Long enough for a sentence, short enough to be useless as a transcript. */
 const MAX_NOTE_LENGTH = 240;
+/**
+ * Republish the snapshot once the unfolded tail passes this. Small enough that a
+ * cold fold stays rare, large enough that `fraim_connect` is not rewriting a
+ * 1 MB file on every call. At the measured ~250 bytes per event this is roughly
+ * two thousand events, which S4b folds in about 10 ms.
+ */
+const SNAPSHOT_REFRESH_BYTES = 512 * 1024;
 // ── Paths and config ─────────────────────────────────────────────────────────
 /**
  * The store path, inside the manager layer so the existing git-backend guardrail
@@ -108,10 +160,25 @@ function resolveUsageLimits(workspaceRoot) {
     return {
         firingLogLimit: positiveInt(block.firingLogLimit, DEFAULT_FIRING_LOG_LIMIT),
         offerLogLimit: positiveInt(block.offerLogLimit, DEFAULT_OFFER_LOG_LIMIT),
+        jobOfferReserve: positiveInt(block.jobOfferReserve, DEFAULT_JOB_OFFER_RESERVE),
         retirementOfferThreshold: positiveInt(block.retirementOfferThreshold, DEFAULT_RETIREMENT_OFFER_THRESHOLD),
         retentionDays: typeof retentionDays === 'number' ? retentionDays : -1,
     };
 }
+/**
+ * Bounds for a read that has no workspace to resolve them from. Retention is -1
+ * on purpose: a fold must not silently drop log records against a guessed
+ * retention window. Pruning is `pruneUsageStore`'s job and is explicit.
+ */
+function defaultUsageLimits() {
+    return {
+        firingLogLimit: DEFAULT_FIRING_LOG_LIMIT,
+        offerLogLimit: DEFAULT_OFFER_LOG_LIMIT,
+        jobOfferReserve: DEFAULT_JOB_OFFER_RESERVE,
+        retirementOfferThreshold: DEFAULT_RETIREMENT_OFFER_THRESHOLD,
+        retentionDays: -1,
+    };
+}
 // ── Keys ─────────────────────────────────────────────────────────────────────
 /** Normalize a title for keying. Case and whitespace only: never a similarity measure (D4). */
 function normalizeEntryTitle(title) {
@@ -129,9 +196,39 @@ function titleUsageKey(fileType, title) {
 function ruleUsageKey(displayPath) {
     return `R:${displayPath.replace(/\\/g, '/')}`;
 }
+/**
+ * Issue #1752 A1: the single definition of what an entry is.
+ *
+ * `enumerateFileEntries`, `buildEntryIndex`, `lookupUsage` and the `classify` CLI
+ * each built `id ?? titleUsageKey(...)` for themselves, which is four definitions
+ * that can drift. Defect 4 is what drift looks like: one lesson stamped in one
+ * tier and unstamped in another became two records, one collecting every offer
+ * and the other every firing, and `standing` then recommended promoting and
+ * demoting the same lesson in a single response.
+ *
+ * The id always wins. A title key is a fallback for a lesson no tier has stamped,
+ * never a second identity for one that has been.
+ */
+function resolveEntryIdentity(input) {
+    const id = typeof input.id === 'string' ? input.id.trim() : '';
+    if (id)
+        return { key: id, source: 'id' };
+    return { key: titleUsageKey(input.fileType ?? 'unknown', input.title ?? ''), source: 'title' };
+}
+/**
+ * Issue #1752 C2: one model, one bucket.
+ *
+ * Measured in the live store: `GPT-5` 6857 against `gpt-5` 2652, and `gpt-6` 1764
+ * against `GPT-6` 140. Every per-model calculation, including retirement
+ * candidacy, was splitting its own denominator.
+ */
+function normalizeModelName(model) {
+    const normalized = String(model ?? '').trim().toLowerCase();
+    return normalized || null;
+}
 // ── Read and write ───────────────────────────────────────────────────────────
 function emptyStore() {
-    return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {} };
+    return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {}, foldedThroughBytes: {}, offersEraStart: null };
 }
 /**
  * Is this parsed value a record every consumer can read without crashing?
@@ -167,38 +264,196 @@ function isUsableRecord(value) {
         && Array.isArray(record.titles);
 }
 /**
- * A missing or corrupt store reads as empty. It is derived data, so losing it is
- * recoverable and must never break the loader that reads it.
+ * The earliest offer date a pre-#1752 snapshot can evidence.
+ *
+ * Found by validating against a copy of the real store: without this, a store
+ * written before the event log existed has no `offersEraStart`, the scoped
+ * invariant has no era to scope to, and it reports zero violations on a corpus
+ * that has 62 entries holding more attested firings than recorded offers. Silent
+ * and wrong, which is the failure class this whole issue is about.
+ *
+ * Derived from the bounded offer logs, so it is at or later than the true first
+ * offer. Later means fewer firings counted as in-era, which under-reports rather
+ * than inventing a violation, and that is the right direction for a check whose
+ * CLI form refuses a write.
  */
-function readUsageStore(storePath = resolveUsageStorePath()) {
+function deriveOffersEraStart(entries) {
+    let earliest = null;
+    for (const record of Object.values(entries)) {
+        for (const offer of record.offered.log) {
+            if (!offer?.date)
+                continue;
+            if (earliest === null || offer.date < earliest)
+                earliest = offer.date;
+        }
+    }
+    return earliest;
+}
+function readCheckpoints(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        return {};
+    const out = {};
+    for (const [machineId, bytes] of Object.entries(value)) {
+        if (typeof bytes === 'number' && Number.isFinite(bytes) && bytes >= 0)
+            out[machineId] = bytes;
+    }
+    return out;
+}
+/** Read the normal snapshot, recovering a missing or torn copy from its checkpoint. */
+function readUsageSnapshot(storePath = resolveUsageStorePath()) {
+    for (const candidate of [storePath, resolveUsageCheckpointPath(storePath)]) {
+        const snapshot = parseUsageSnapshot(candidate);
+        if (snapshot)
+            return snapshot;
+    }
+    if ((0, fs_1.existsSync)(resolveUsageCheckpointPath(storePath))) {
+        throw new Error('Usage snapshot and checkpoint backup are unreadable. Restore a matching backup; do not delete the usage directory.');
+    }
+    return emptyStore();
+}
+function resolveUsageCheckpointPath(storePath) {
+    return storePath + '.checkpoint';
+}
+function parseUsageSnapshot(storePath) {
     try {
         if (!(0, fs_1.existsSync)(storePath))
-            return emptyStore();
+            return null;
         const parsed = JSON.parse((0, fs_1.readFileSync)(storePath, 'utf8'));
         // `typeof [] === 'object'`, so an array has to be rejected explicitly: otherwise
         // Object.values yields whatever the array held and every consumer that reads
         // `entry.offered` throws on it.
         if (!parsed || typeof parsed !== 'object' || !parsed.entries
             || typeof parsed.entries !== 'object' || Array.isArray(parsed.entries)) {
-            return emptyStore();
+            return null;
         }
         // Drop any value that is not a usable record rather than handing a caller
         // something it will crash on. A partially corrupt store degrades to the records
-        // that are still readable, which is the right behaviour for derived data.
+        // that are still readable before consolidation. Consolidated snapshots must be complete.
         const entries = {};
         for (const [key, value] of Object.entries(parsed.entries)) {
             if (isUsableRecord(value))
                 entries[key] = value;
+            else if (parsed.consolidated)
+                return null;
         }
-        return { version: parsed.version ?? STORE_VERSION, updatedAt: parsed.updatedAt ?? emptyStore().updatedAt, entries };
+        return {
+            consolidated: parsed.consolidated === true,
+            version: parsed.version ?? STORE_VERSION,
+            updatedAt: parsed.updatedAt ?? emptyStore().updatedAt,
+            entries,
+            foldedThroughBytes: readCheckpoints(parsed.foldedThroughBytes),
+            offersEraStart: typeof parsed.offersEraStart === 'string'
+                ? parsed.offersEraStart
+                : deriveOffersEraStart(entries),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Refresh the normal snapshot in place (Windows readers can prevent rename).
+ * Sleep alone publishes the separate flushed checkpoint backup before clearing
+ * journals, so a torn ordinary refresh can recover without replaying old history.
+ */
+function writeUsageSnapshot(store, storePath = resolveUsageStorePath()) {
+    try {
+        (0, fs_1.mkdirSync)((0, path_1.dirname)(storePath), { recursive: true });
+        (0, fs_1.writeFileSync)(storePath, `${JSON.stringify(store, null, 2)}\n`, { encoding: 'utf8', flush: true });
+        return true;
     }
     catch {
-        return emptyStore();
+        return false;
+    }
+}
+/**
+ * Fold an event log into a base record set.
+ *
+ * `base` defaults to empty, which is a cold full fold. `readUsageStore` passes the
+ * snapshot and its checkpoints, which is the warm path S4b measured flat at
+ * 2.6 ms from 9.9 MB to 79 MB of log.
+ */
+function foldUsageEvents(options) {
+    const limits = options.limits ?? defaultUsageLimits();
+    const now = options.now ?? new Date();
+    const base = options.base ?? emptyStore();
+    const tail = (0, learning_usage_events_1.readEventsSince)(options.eventDir, options.checkpoints ?? {});
+    // A checkpoint past the end of its log means the log was compacted or replaced
+    // under this snapshot, so the snapshot may already count events the log no
+    // longer holds. Re-folding onto it would double count. Discard and fold cold.
+    if (tail.restarted && base.consolidated)
+        throw new Error('Usage journal no longer matches its consolidated snapshot. Restore matching checkpoint files before continuing.');
+    if (tail.restarted) {
+        const cold = (0, learning_usage_events_1.readEventsSince)(options.eventDir, {});
+        return applyTail(emptyStore(), cold, limits, now, true);
+    }
+    return applyTail(base, tail, limits, now, false);
+}
+function applyTail(base, tail, limits, now, restarted) {
+    const store = {
+        consolidated: base.consolidated,
+        version: base.version,
+        updatedAt: base.updatedAt,
+        entries: base.entries,
+        foldedThroughBytes: { ...(base.foldedThroughBytes ?? {}) },
+        offersEraStart: base.offersEraStart ?? null,
+    };
+    let tailBytes = 0;
+    const touched = new Set();
+    for (const read of tail.reads) {
+        const from = restarted ? 0 : (store.foldedThroughBytes[read.machineId] ?? 0);
+        tailBytes += Math.max(0, read.through - from);
+        for (const event of read.records) {
+            const key = applyEvent(store, event, limits, now);
+            if (key)
+                touched.add(key);
+        }
+        store.foldedThroughBytes[read.machineId] = read.through;
+    }
+    for (const key of touched) {
+        const record = store.entries[key];
+        if (record)
+            trimLogs(record, limits, now);
     }
+    // E: the invariant is evaluated on the read path, where it is always current
+    // and costs the write path nothing.
+    flagIntegrity(store);
+    return { store, skippedLines: tail.skippedLines, restarted, tailBytes, logs: tail.reads };
 }
-function writeUsageStore(store, storePath = resolveUsageStorePath()) {
-    (0, fs_1.mkdirSync)((0, path_1.dirname)(storePath), { recursive: true });
-    (0, fs_1.writeFileSync)(storePath, `${JSON.stringify(store, null, 2)}\n`, 'utf8');
+/**
+ * The checkpointed read every consumer uses. Snapshot plus the bytes appended
+ * past it, republished once the tail is worth re-checkpointing.
+ */
+function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
+    const snapshot = readUsageSnapshot(storePath);
+    const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
+    if ((0, learning_usage_events_1.listEventLogs)(eventDir).length === 0)
+        return snapshot;
+    const fold = (base) => foldUsageEvents({
+        eventDir,
+        limits: options.limits,
+        now: options.now,
+        base,
+        checkpoints: base.foldedThroughBytes ?? {},
+    });
+    let folded;
+    try {
+        folded = fold(snapshot);
+    }
+    catch (error) {
+        const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
+        if (!backup)
+            throw error;
+        folded = fold(backup);
+    }
+    const mode = options.refreshSnapshot ?? 'auto';
+    const shouldRefresh = mode === 'always'
+        || (mode === 'auto' && (folded.tailBytes >= SNAPSHOT_REFRESH_BYTES || !(0, fs_1.existsSync)(storePath) || folded.restarted));
+    if (shouldRefresh) {
+        folded.store.updatedAt = (options.now ?? new Date()).toISOString();
+        writeUsageSnapshot(folded.store, storePath);
+    }
+    return folded.store;
 }
 // ── Mutation helpers ─────────────────────────────────────────────────────────
 function emptyOutcomeCounts() {
@@ -277,67 +532,288 @@ function withinRetention(date, retentionDays, now) {
         return true;
     return (now.getTime() - parsed) / 86_400_000 <= retentionDays;
 }
+const byDate = (a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0);
+function isAmbientJob(job) {
+    return exports.AMBIENT_JOB_LABELS.includes(job ?? '');
+}
+/**
+ * Issue #1752 D3: bound the offer log without letting session traffic evict the
+ * only records that can answer R7.
+ *
+ * Measured cause: `fraim_connect` records one offer per entry per session under
+ * the label `session`, and a plain newest-50 rule then evicted every
+ * job-labelled record within about a day. 2067 of the 2354 retained offer records
+ * in the live store carried `session` or `__freeform__`, and only three real job
+ * names appeared at all, which is why `classify --job <real-job>` could almost
+ * never find a match.
+ *
+ * Job-labelled records get the first `jobOfferReserve` slots; the rest of the
+ * budget goes to the newest of everything else.
+ */
+function selectRetainedOffers(log, limits) {
+    if (log.length <= limits.offerLogLimit)
+        return log;
+    const jobLabelled = log.filter((r) => !isAmbientJob(r.job));
+    const ambient = log.filter((r) => isAmbientJob(r.job));
+    const reserve = Math.min(limits.jobOfferReserve, limits.offerLogLimit);
+    const guaranteed = jobLabelled.slice(-Math.min(jobLabelled.length, reserve));
+    const remaining = limits.offerLogLimit - guaranteed.length;
+    const pool = [...jobLabelled.slice(0, jobLabelled.length - guaranteed.length), ...ambient].sort(byDate);
+    const filler = remaining > 0 ? pool.slice(-remaining) : [];
+    return [...guaranteed, ...filler].sort(byDate).slice(-limits.offerLogLimit);
+}
 /**
  * Trim one entry's logs: drop records outside the retention window, then keep the
  * newest N. Aggregates are never touched, so R16a's second criterion holds.
  */
 function trimLogs(record, limits, now) {
-    const byDate = (a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0);
-    record.offered.log = record.offered.log
-        .filter((r) => withinRetention(r.date, limits.retentionDays, now))
-        .sort(byDate)
-        .slice(-limits.offerLogLimit);
+    record.offered.log = selectRetainedOffers(record.offered.log.filter((r) => withinRetention(r.date, limits.retentionDays, now)).sort(byDate), limits);
     record.fired.log = record.fired.log
         .filter((r) => withinRetention(r.date, limits.retentionDays, now))
         .sort(byDate)
         .slice(-limits.firingLogLimit);
 }
+function offerEvent(offer) {
+    return {
+        kind: 'offer',
+        key: offer.key,
+        family: offer.family,
+        title: offer.title,
+        fileType: offer.fileType,
+        level: offer.level,
+        aboveThreshold: Boolean(offer.aboveThreshold),
+        job: offer.job,
+        date: offer.date,
+        agent: offer.agent ?? null,
+        model: normalizeModelName(offer.model),
+    };
+}
+function firingEvent(firing) {
+    return {
+        kind: 'firing',
+        key: firing.key,
+        family: firing.family,
+        title: firing.title,
+        fileType: firing.fileType,
+        level: firing.level,
+        outcome: firing.outcome,
+        note: sanitizeNote(firing.note),
+        job: firing.job,
+        date: firing.date,
+        agent: firing.agent ?? null,
+        model: normalizeModelName(firing.model),
+        source: firing.source ?? 'attested',
+        sourceRef: firing.sourceRef,
+    };
+}
+/**
+ * Apply one logged event to the record set. Returns the key it touched, or null
+ * when the event is unusable or already counted.
+ *
+ * Firing idempotency lives here rather than on the write path, so it holds
+ * however many times an event was appended: `(key, sourceRef)` is the complete
+ * identity of an attestation, which is the same fact the migration's exact firing
+ * recovery rests on.
+ */
+function applyEvent(store, event, limits, now) {
+    const kind = event.kind;
+    const key = typeof event.key === 'string' ? event.key : '';
+    const date = typeof event.date === 'string' ? event.date : '';
+    if (!key || !isValidDate(date))
+        return null;
+    const family = (event.family === 'rule' ? 'rule' : 'learning');
+    const fileType = typeof event.fileType === 'string' ? event.fileType : undefined;
+    const level = typeof event.level === 'string' ? event.level : undefined;
+    const model = normalizeModelName(event.model);
+    const job = typeof event.job === 'string' ? event.job : 'unknown';
+    if (kind === 'offer')
+        return applyOfferEvent(store, event, { key, date, family, fileType, level, model, job });
+    if (kind === 'firing')
+        return applyFiringEvent(store, event, { key, date, family, fileType, level, model, job });
+    // An event kind this version does not know is skipped, so a log written by a
+    // newer version degrades to what this one understands rather than failing.
+    void limits;
+    void now;
+    return null;
+}
+function applyOfferEvent(store, event, common) {
+    const record = ensureRecord(store, common.key, common.family, common.fileType, common.level);
+    rememberTitle(record, typeof event.title === 'string' ? event.title : undefined);
+    record.offered.total += 1;
+    if (event.aboveThreshold)
+        record.offered.aboveThreshold += 1;
+    if (common.model)
+        record.offered.byModel[common.model] = (record.offered.byModel[common.model] ?? 0) + 1;
+    record.offered.lastOfferedAt = maxDate(record.offered.lastOfferedAt, common.date);
+    record.offered.log.push({
+        date: common.date,
+        job: common.job,
+        agent: event.agent ?? null,
+        model: common.model,
+        aboveThreshold: Boolean(event.aboveThreshold),
+    });
+    // The era only ever moves earlier, so the scoped invariant's boundary is
+    // monotonic and a late-syncing log cannot shrink the window it applies to.
+    store.offersEraStart = minDate(store.offersEraStart, common.date);
+    return common.key;
+}
+function applyFiringEvent(store, event, common) {
+    const outcome = event.outcome;
+    if (!exports.FIRING_OUTCOMES.includes(outcome))
+        return null;
+    const record = ensureRecord(store, common.key, common.family, common.fileType, common.level);
+    // Idempotency lives here rather than on the write path, so it holds however
+    // many times an event was appended. `(key, sourceRef)` is the complete identity
+    // of an attestation, which is the same fact the migration's exact firing
+    // recovery rests on.
+    const sourceRef = typeof event.sourceRef === 'string' ? event.sourceRef : '';
+    if (sourceRef && record.fired.seenSources.includes(sourceRef))
+        return null;
+    rememberTitle(record, typeof event.title === 'string' ? event.title : undefined);
+    record.fired.total += 1;
+    record.fired.byOutcome[outcome] = (record.fired.byOutcome[outcome] ?? 0) + 1;
+    record.fired.lastAttestedAt = maxDate(record.fired.lastAttestedAt, common.date);
+    if (exports.CLOCK_ADVANCING_OUTCOMES.includes(outcome)) {
+        record.fired.firstFiredAt = minDate(record.fired.firstFiredAt, common.date);
+        record.fired.lastFiredAt = maxDate(record.fired.lastFiredAt, common.date);
+        if (common.model)
+            record.fired.byModel[common.model] = (record.fired.byModel[common.model] ?? 0) + 1;
+    }
+    record.fired.log.push({
+        date: common.date,
+        outcome,
+        job: common.job,
+        agent: event.agent ?? null,
+        model: common.model,
+        note: sanitizeNote(String(event.note ?? '')),
+        source: event.source === 'backfill-legacy' ? 'backfill-legacy' : 'attested',
+    });
+    if (sourceRef)
+        record.fired.seenSources.push(sourceRef);
+    return common.key;
+}
+// ── The scoped integrity invariant (issue #1752 E) ──────────────────────────
+/**
+ * How many of this entry's attested firings fall inside the offers era.
+ *
+ * The invariant the issue asks for, `fired <= offered`, is unsound as literally
+ * stated: `runBackfill` writes firings attested by 299 retrospectives that
+ * predate offer recording entirely, and those have no offer by construction.
+ * Scoped to attested firings dated at or after the store's first recorded offer,
+ * it holds and means something.
+ *
+ * Two bases, because the check has two jobs with opposite failure budgets.
+ *
+ *   'log'        Counts only the firings the bounded log still holds. It cannot
+ *                over-count, so it can never refuse a correct write. This is what
+ *                the CLI throws on.
+ *   'aggregate'  Starts from `fired.total` and subtracts the firings the log
+ *                shows are exempt. It sees an entry whose detail has aged out,
+ *                which is the shape all 58 violating entries in the live store
+ *                have, and it can over-count when the log no longer shows the
+ *                exemptions. That is acceptable for a report and not for a
+ *                refusal, so this is what the read path flags with and `verify`
+ *                lists.
+ */
+function attestedFiringsInOffersEra(record, offersEraStart, basis) {
+    if (!offersEraStart)
+        return 0;
+    const inLog = record.fired.log.filter((r) => r.source === 'attested' && r.date >= offersEraStart).length;
+    if (basis === 'log')
+        return inLog;
+    const exemptInLog = record.fired.log.filter((r) => r.source === 'backfill-legacy' || r.date < offersEraStart).length;
+    return Math.max(inLog, record.fired.total - exemptInLog);
+}
+function integrityViolation(record, offersEraStart, basis = 'aggregate') {
+    // An entry with no offer record at all is not an accounting overflow; it is the
+    // question defect 2 is about, and it has several innocent causes: the offers
+    // were recorded under the entry's other key before #1752 A2 merged them, the
+    // entry is a rule or a demoted lesson the offer path does not enumerate, or the
+    // offers are sitting in another machine's event log that has not synced yet.
+    // Throwing on any of those would block a correct attestation on a sync
+    // artefact. The overflow this invariant exists to catch is the one the issue
+    // measured: entries showing `offered: 1, fired: 10`, which all have an offer.
+    if (record.offered.total === 0)
+        return null;
+    const inEra = attestedFiringsInOffersEra(record, offersEraStart, basis);
+    if (inEra <= record.offered.total)
+        return null;
+    return {
+        flag: 'fired-exceeds-offered',
+        count: inEra - record.offered.total,
+        detail: `${inEra} attested firings dated on or after ${offersEraStart} against ${record.offered.total} recorded offers. An entry cannot fire without being offered, so one of the two counters is wrong for this entry.`,
+    };
+}
+function flagIntegrity(store) {
+    for (const record of Object.values(store.entries)) {
+        const violation = integrityViolation(record, store.offersEraStart);
+        if (violation)
+            record.integrity = violation;
+        else if (record.integrity)
+            delete record.integrity;
+    }
+}
+/**
+ * Materialize the snapshot when there is none.
+ *
+ * Before the first consolidation, readers publish snapshots, but a workspace that
+ * has only ever been written to would have no snapshot at all, so every reader
+ * would pay a cold fold and anything inspecting the store as a file would find
+ * nothing there. Bounded: it only runs when the file is absent, and at that point
+ * the fold is over a log that has just been created. A failure is swallowed, like
+ * every other snapshot write.
+ */
+function ensureSnapshotExists(storePath, eventDir, limits, now) {
+    try {
+        if ((0, fs_1.existsSync)(storePath))
+            return;
+        readUsageStore(storePath, { limits, now, eventDir, refreshSnapshot: 'always' });
+    }
+    catch {
+        // Initial cache publication can retry; consolidated history has a separate backup.
+    }
+}
 /**
  * R1 and R2: write one offer record per listed entry, without any agent
  * involvement and without depending on the agent reading the file.
+ *
+ * Issue #1752 B: this appends and does not read. The read-modify-write it
+ * replaces lost 1589 of 1600 writes under eight concurrent processes on a plain
+ * local disk (spike S1b), which is the dominant loss path behind both the
+ * impossible counters and the four conflict copies. Nothing here can throw on an
+ * integrity violation either: R2 requires that a usage failure never changes what
+ * the agent is given, so the violation is flagged by the read path instead.
  */
 function recordOffers(offers, options) {
     const storePath = options.storePath ?? resolveUsageStorePath();
-    const now = options.now ?? new Date();
-    const store = readUsageStore(storePath);
-    let recorded = 0;
-    for (const offer of offers) {
-        if (!offer.key || !isValidDate(offer.date))
-            continue;
-        const record = ensureRecord(store, offer.key, offer.family, offer.fileType, offer.level);
-        rememberTitle(record, offer.title);
-        record.offered.total += 1;
-        if (offer.aboveThreshold)
-            record.offered.aboveThreshold += 1;
-        if (offer.model)
-            record.offered.byModel[offer.model] = (record.offered.byModel[offer.model] ?? 0) + 1;
-        record.offered.lastOfferedAt = maxDate(record.offered.lastOfferedAt, offer.date);
-        record.offered.log.push({
-            date: offer.date,
-            job: offer.job,
-            agent: offer.agent ?? null,
-            model: offer.model ?? null,
-            aboveThreshold: Boolean(offer.aboveThreshold),
-        });
-        trimLogs(record, options.limits, now);
-        recorded += 1;
-    }
-    if (recorded > 0) {
-        store.updatedAt = now.toISOString();
-        writeUsageStore(store, storePath);
-    }
-    return { recorded, entries: Object.keys(store.entries).length };
+    const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
+    const usable = offers.filter((offer) => offer.key && isValidDate(offer.date));
+    if (usable.length === 0)
+        return { recorded: 0, entries: 0 };
+    (0, learning_usage_events_1.appendEvents)(eventDir, usable.map(offerEvent), options.machineId);
+    ensureSnapshotExists(storePath, eventDir, options.limits, options.now ?? new Date());
+    return { recorded: usable.length, entries: new Set(usable.map((o) => o.key)).size };
 }
 /**
  * R4 and R6: write the attested firings. Idempotent per (entry, attestation
  * source) so a re-run, or a backfill that overlaps a live attestation, cannot
  * inflate a count.
+ *
+ * Unlike the offer path this reads first, because it has two questions only the
+ * current record can answer: whether this attestation has already contributed,
+ * and whether the scoped invariant is about to be violated. It still writes by
+ * appending, and the fold applies the same `(key, sourceRef)` idempotency, so a
+ * racing duplicate append is folded once rather than counted twice.
+ *
+ * The invariant throws here. This is a CLI path with an operator watching, and a
+ * violation means a real accounting bug rather than a cosmetic one.
  */
 function recordFirings(firings, options) {
     const storePath = options.storePath ?? resolveUsageStorePath();
+    const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
     const now = options.now ?? new Date();
-    const store = readUsageStore(storePath);
-    let recorded = 0;
+    const store = readUsageStore(storePath, { limits: options.limits, now, eventDir });
+    const events = [];
     let alreadyRecorded = 0;
     for (const firing of firings) {
         if (!exports.FIRING_OUTCOMES.includes(firing.outcome)) {
@@ -345,40 +821,51 @@ function recordFirings(firings, options) {
         }
         if (!firing.key || !isValidDate(firing.date))
             continue;
-        const record = ensureRecord(store, firing.key, firing.family, firing.fileType, firing.level);
-        if (firing.sourceRef && record.fired.seenSources.includes(firing.sourceRef)) {
+        const existing = store.entries[firing.key];
+        if (firing.sourceRef && existing?.fired.seenSources.includes(firing.sourceRef)) {
             alreadyRecorded += 1;
             continue;
         }
-        rememberTitle(record, firing.title);
-        record.fired.total += 1;
-        record.fired.byOutcome[firing.outcome] += 1;
-        record.fired.lastAttestedAt = maxDate(record.fired.lastAttestedAt, firing.date);
-        if (exports.CLOCK_ADVANCING_OUTCOMES.includes(firing.outcome)) {
-            record.fired.firstFiredAt = minDate(record.fired.firstFiredAt, firing.date);
-            record.fired.lastFiredAt = maxDate(record.fired.lastFiredAt, firing.date);
-            if (firing.model)
-                record.fired.byModel[firing.model] = (record.fired.byModel[firing.model] ?? 0) + 1;
+        events.push(firingEvent(firing));
+    }
+    if (events.length > 0) {
+        if ((options.integrityMode ?? 'throw') === 'throw') {
+            assertScopedIntegrity(store, events, options.limits, now);
         }
-        record.fired.log.push({
-            date: firing.date,
-            outcome: firing.outcome,
-            job: firing.job,
-            agent: firing.agent ?? null,
-            model: firing.model ?? null,
-            note: sanitizeNote(firing.note),
-            source: firing.source ?? 'attested',
-        });
-        if (firing.sourceRef)
-            record.fired.seenSources.push(firing.sourceRef);
-        trimLogs(record, options.limits, now);
-        recorded += 1;
+        (0, learning_usage_events_1.appendEvents)(eventDir, events, options.machineId);
+        // Unlike the offer path, this republishes the snapshot. `record-firings` runs
+        // from the CLI once per retrospective, so the fold is cheap, and it makes the
+        // store file current the moment an operator's write returns rather than at
+        // whatever point a later reader happens to refresh it. The offer path stays
+        // append-only, because that is the one that runs on every `fraim_connect`.
+        readUsageStore(storePath, { limits: options.limits, now, eventDir, refreshSnapshot: 'always' });
     }
-    if (recorded > 0 || alreadyRecorded > 0) {
-        store.updatedAt = now.toISOString();
-        writeUsageStore(store, storePath);
+    return { recorded: events.length, alreadyRecorded };
+}
+/**
+ * E, CLI form: refuse to write a batch that would put an entry into the
+ * impossible state the issue reports on 58 of its 223 entries.
+ */
+function assertScopedIntegrity(store, events, limits, now) {
+    const projected = {
+        version: store.version,
+        updatedAt: store.updatedAt,
+        entries: JSON.parse(JSON.stringify(store.entries)),
+        foldedThroughBytes: store.foldedThroughBytes,
+        offersEraStart: store.offersEraStart,
+    };
+    for (const event of events)
+        applyEvent(projected, event, limits, now);
+    for (const event of events) {
+        const record = projected.entries[event.key];
+        if (!record)
+            continue;
+        // 'log' basis: a refusal must never rest on a number that can over-count.
+        const violation = integrityViolation(record, projected.offersEraStart, 'log');
+        if (violation) {
+            throw new Error(`Refusing to record firings for ${event.key}: ${violation.detail} Run \`learning-usage verify\` for the full integrity picture before forcing this through.`);
+        }
     }
-    return { recorded, alreadyRecorded };
 }
 /**
  * R5: record that an entry's title changed, so a firing written before the edit
@@ -387,19 +874,28 @@ function recordFirings(firings, options) {
  */
 function noteTitleChange(key, newTitle, options) {
     const storePath = options.storePath ?? resolveUsageStorePath();
-    const store = readUsageStore(storePath);
+    const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
+    const store = readUsageStore(storePath, { limits: options.limits, now: options.now, eventDir });
     const record = store.entries[key];
     if (!record)
         return;
     rememberTitle(record, newTitle);
     store.updatedAt = (options.now ?? new Date()).toISOString();
-    writeUsageStore(store, storePath);
+    writeUsageSnapshot(store, storePath);
 }
-/** Apply retention and log bounds across the whole store. Reports what it dropped. */
+/**
+ * Apply retention and log bounds across the whole store, and compact the event
+ * log behind it.
+ *
+ * Compaction is a requirement rather than an optimization: S4 measured a cold
+ * full fold at 84 ms on today's volume but 794 ms at ten times it, and the
+ * checkpoint only makes a cold fold rare, not bounded.
+ */
 function pruneUsageStore(limits, options = {}) {
     const storePath = options.storePath ?? resolveUsageStorePath();
+    const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
     const now = options.now ?? new Date();
-    const store = readUsageStore(storePath);
+    const store = readUsageStore(storePath, { limits, now, eventDir });
     let prunedOfferRecords = 0;
     let prunedFiringRecords = 0;
     for (const record of Object.values(store.entries)) {
@@ -409,9 +905,51 @@ function pruneUsageStore(limits, options = {}) {
         prunedOfferRecords += offersBefore - record.offered.log.length;
         prunedFiringRecords += firingsBefore - record.fired.log.length;
     }
-    if (prunedOfferRecords > 0 || prunedFiringRecords > 0) {
-        store.updatedAt = now.toISOString();
-        writeUsageStore(store, storePath);
+    const compaction = compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store });
+    return { prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length, compaction };
+}
+/** Save totals and source bookmarks in both copies, then clear processed journals.
+ * Requires an interval without concurrent writers or sync activity.
+ */
+function compactUsageEvents(options) {
+    const now = options.now ?? new Date();
+    const store = options.store
+        ?? readUsageStore(options.snapshotPath, { limits: options.limits, now, eventDir: options.eventDir });
+    for (const log of (0, learning_usage_events_1.listEventLogs)(options.eventDir)) {
+        if ((store.foldedThroughBytes?.[log.machineId] ?? 0) < log.baseOffset) {
+            throw new Error('Cannot consolidate from a snapshot older than the journal checkpoint.');
+        }
+    }
+    for (const entry of Object.values(store.entries))
+        trimLogs(entry, options.limits, now);
+    store.consolidated = true;
+    store.updatedAt = now.toISOString();
+    // Publish the recoverable checkpoint first, then the normal snapshot. No
+    // journal is cleared unless both writes succeed. The backup is never written
+    // by ordinary readers, so a torn cache refresh cannot destroy both copies.
+    const checkpoint = resolveUsageCheckpointPath(options.snapshotPath);
+    const temp = checkpoint + '.tmp';
+    try {
+        (0, fs_1.mkdirSync)((0, path_1.dirname)(checkpoint), { recursive: true });
+        (0, fs_1.writeFileSync)(temp, JSON.stringify(store, null, 2) + '\n', { encoding: 'utf8', flush: true });
+        (0, fs_1.renameSync)(temp, checkpoint);
+    }
+    catch {
+        try {
+            (0, fs_1.rmSync)(temp, { force: true });
+        }
+        catch { /* Best effort. */ }
+        throw new Error('Cannot consolidate usage events: checkpoint backup could not be saved.');
+    }
+    if (!writeUsageSnapshot(store, options.snapshotPath)) {
+        throw new Error('Cannot consolidate usage events: aggregate snapshot could not be saved.');
     }
-    return { prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length };
+    return (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
+        if (event.kind !== 'offer' && event.kind !== 'firing')
+            return true;
+        const date = typeof event.date === 'string' ? event.date : '';
+        if (typeof event.key !== 'string' || !event.key || !isValidDate(date))
+            return true;
+        return event.kind === 'firing' && !exports.FIRING_OUTCOMES.includes(event.outcome);
+    }, store.foldedThroughBytes ?? {});
 }