fraim 2.0.340 → 2.0.342

package/dist/src/local-mcp-server/learning-usage-store.js

@@ -230,6 +230,13 @@ function normalizeModelName(model) {
 function emptyStore() {
     return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {}, foldedThroughBytes: {}, offersEraStart: null };
 }
+function withEvidence(store, evidence) {
+    Object.defineProperty(store, 'evidence', { value: evidence, enumerable: false, configurable: true });
+    return store;
+}
+function unavailableStore(observation) {
+    return withEvidence(emptyStore(), { status: 'unavailable', excludedKeys: [], observations: [observation] });
+}
 /**
  * Is this parsed value a record every consumer can read without crashing?
  *
@@ -254,14 +261,21 @@ function isUsableRecord(value) {
     const fired = record.fired;
     if (!offered || typeof offered !== 'object' || !fired || typeof fired !== 'object')
         return false;
-    return Array.isArray(offered.log)
-        && typeof offered.total === 'number'
-        && typeof offered.aboveThreshold === 'number'
-        && !!offered.byModel && typeof offered.byModel === 'object'
-        && Array.isArray(fired.log)
-        && typeof fired.total === 'number'
-        && !!fired.byOutcome && typeof fired.byOutcome === 'object'
-        && Array.isArray(record.titles);
+    const count = (v) => Number.isSafeInteger(v) && v >= 0;
+    const counts = (v) => !!v && typeof v === 'object' && !Array.isArray(v)
+        && Object.values(v).every(count);
+    const nullableString = (v) => v === null || typeof v === 'string';
+    return typeof record.key === 'string' && (record.family === 'learning' || record.family === 'rule')
+        && Array.isArray(offered.log) && offered.log.every(r => r && isValidDate(r.date)
+        && typeof r.job === 'string' && nullableString(r.model))
+        && count(offered.total) && count(offered.aboveThreshold)
+        && counts(offered.byModel)
+        && Array.isArray(fired.log) && fired.log.every(r => r && isValidDate(r.date)
+        && typeof r.job === 'string' && nullableString(r.model) && exports.FIRING_OUTCOMES.includes(r.outcome))
+        && count(fired.total) && counts(fired.byOutcome) && counts(fired.byModel)
+        && exports.FIRING_OUTCOMES.every(outcome => count(fired.byOutcome[outcome]))
+        && Array.isArray(fired.seenSources) && fired.seenSources.every(s => typeof s === 'string')
+        && Array.isArray(record.titles) && record.titles.every(t => typeof t === 'string');
 }
 /**
  * The earliest offer date a pre-#1752 snapshot can evidence.
@@ -301,15 +315,23 @@ function readCheckpoints(value) {
 }
 /** Read the normal snapshot, recovering a missing or torn copy from its checkpoint. */
 function readUsageSnapshot(storePath = resolveUsageStorePath()) {
-    for (const candidate of [storePath, resolveUsageCheckpointPath(storePath)]) {
-        const snapshot = parseUsageSnapshot(candidate);
-        if (snapshot)
-            return snapshot;
-    }
+    const primary = parseUsageSnapshot(storePath);
+    if (primary && primary.evidence?.status !== 'unavailable')
+        return primary;
+    // An unknown schema is not evidence that an older checkpoint can replace it.
+    if (primary?.evidence?.observations.includes('unsupported-schema'))
+        return primary;
+    const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
+    if (backup && !backup.evidence)
+        return withEvidence(backup, {
+            status: 'reliable', excludedKeys: [],
+            observations: [(0, fs_1.existsSync)(storePath) ? 'primary-unreadable' : 'primary-missing'],
+            recovery: 'checkpoint', persisted: false,
+        });
     if ((0, fs_1.existsSync)(resolveUsageCheckpointPath(storePath))) {
         throw new Error('Usage snapshot and checkpoint backup are unreadable. Restore a matching backup; do not delete the usage directory.');
     }
-    return emptyStore();
+    return primary ?? emptyStore();
 }
 function resolveUsageCheckpointPath(storePath) {
     return storePath + '.checkpoint';
@@ -324,19 +346,29 @@ function parseUsageSnapshot(storePath) {
         // `entry.offered` throws on it.
         if (!parsed || typeof parsed !== 'object' || !parsed.entries
             || typeof parsed.entries !== 'object' || Array.isArray(parsed.entries)) {
-            return null;
+            return unavailableStore('malformed-snapshot');
+        }
+        if (parsed.version !== undefined && parsed.version !== STORE_VERSION)
+            return unavailableStore('unsupported-schema');
+        if (parsed.foldedThroughBytes !== undefined && (!parsed.foldedThroughBytes
+            || typeof parsed.foldedThroughBytes !== 'object' || Array.isArray(parsed.foldedThroughBytes)
+            || Object.values(parsed.foldedThroughBytes).some(v => !Number.isSafeInteger(v) || v < 0))) {
+            return unavailableStore('malformed-checkpoints');
         }
         // Drop any value that is not a usable record rather than handing a caller
         // something it will crash on. A partially corrupt store degrades to the records
         // that are still readable before consolidation. Consolidated snapshots must be complete.
         const entries = {};
+        const excludedKeys = [];
         for (const [key, value] of Object.entries(parsed.entries)) {
-            if (isUsableRecord(value))
+            if (isUsableRecord(value) && value.key === key)
                 entries[key] = value;
             else if (parsed.consolidated)
-                return null;
+                return unavailableStore('malformed-consolidated-snapshot');
+            else
+                excludedKeys.push(key);
         }
-        return {
+        const store = {
             consolidated: parsed.consolidated === true,
             version: parsed.version ?? STORE_VERSION,
             updatedAt: parsed.updatedAt ?? emptyStore().updatedAt,
@@ -346,9 +378,12 @@ function parseUsageSnapshot(storePath) {
                 ? parsed.offersEraStart
                 : deriveOffersEraStart(entries),
         };
+        return excludedKeys.length ? withEvidence(store, {
+            status: 'partial', excludedKeys, observations: ['malformed-records'],
+        }) : store;
     }
     catch {
-        return null;
+        return unavailableStore('unreadable-snapshot');
     }
 }
 /**
@@ -357,9 +392,12 @@ function parseUsageSnapshot(storePath) {
  * journals, so a torn ordinary refresh can recover without replaying old history.
  */
 function writeUsageSnapshot(store, storePath = resolveUsageStorePath()) {
+    if (store.evidence && store.evidence.status !== 'reliable')
+        return false;
     try {
         (0, fs_1.mkdirSync)((0, path_1.dirname)(storePath), { recursive: true });
-        (0, fs_1.writeFileSync)(storePath, `${JSON.stringify(store, null, 2)}\n`, { encoding: 'utf8', flush: true });
+        const { evidence: _evidence, ...snapshot } = store;
+        (0, fs_1.writeFileSync)(storePath, `${JSON.stringify(snapshot, null, 2)}\n`, { encoding: 'utf8', flush: true });
         return true;
     }
     catch {
@@ -385,7 +423,10 @@ function foldUsageEvents(options) {
         throw new Error('Usage journal no longer matches its consolidated snapshot. Restore matching checkpoint files before continuing.');
     if (tail.restarted) {
         const cold = (0, learning_usage_events_1.readEventsSince)(options.eventDir, {});
-        return applyTail(emptyStore(), cold, limits, now, true);
+        return applyTail(withEvidence(emptyStore(), {
+            status: 'unavailable', excludedKeys: base.evidence?.excludedKeys ?? [],
+            observations: [...(base.evidence?.observations ?? []), 'journal-restarted'],
+        }), cold, limits, now, true);
     }
     return applyTail(base, tail, limits, now, false);
 }
@@ -398,12 +439,19 @@ function applyTail(base, tail, limits, now, restarted) {
         foldedThroughBytes: { ...(base.foldedThroughBytes ?? {}) },
         offersEraStart: base.offersEraStart ?? null,
     };
+    if (base.evidence)
+        withEvidence(store, base.evidence);
     let tailBytes = 0;
+    let unsupportedEvents = false;
     const touched = new Set();
     for (const read of tail.reads) {
         const from = restarted ? 0 : (store.foldedThroughBytes[read.machineId] ?? 0);
         tailBytes += Math.max(0, read.through - from);
         for (const event of read.records) {
+            if (!isSupportedUsageEvent(event)) {
+                unsupportedEvents = true;
+                continue;
+            }
             const key = applyEvent(store, event, limits, now);
             if (key)
                 touched.add(key);
@@ -418,17 +466,63 @@ function applyTail(base, tail, limits, now, restarted) {
     // E: the invariant is evaluated on the read path, where it is always current
     // and costs the write path nothing.
     flagIntegrity(store);
+    const incompleteJournal = tail.skippedLines > 0
+        || tail.reads.some(read => read.through < read.baseOffset + read.size - read.headerBytes);
+    if (incompleteJournal || unsupportedEvents) {
+        withEvidence(store, {
+            status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [],
+            observations: [...(store.evidence?.observations ?? []),
+                ...(incompleteJournal ? ['incomplete-journal'] : []),
+                ...(unsupportedEvents ? ['unsupported-journal-events'] : [])],
+        });
+    }
     return { store, skippedLines: tail.skippedLines, restarted, tailBytes, logs: tail.reads };
 }
 /**
  * The checkpointed read every consumer uses. Snapshot plus the bytes appended
  * past it, republished once the tail is worth re-checkpointing.
  */
+/** Metadata detects observed writes/sync during a read; it is not a writer lock. */
+function usageReadState(storePath, eventDir) {
+    const dir = (0, path_1.dirname)(storePath);
+    const copies = (0, fs_1.existsSync)(dir) ? (0, fs_1.readdirSync)(dir)
+        .filter(name => /^learning-usage(-.+)?\.json$/.test(name) && name !== STORE_FILE_NAME).sort() : [];
+    const paths = [storePath, resolveUsageCheckpointPath(storePath),
+        ...(0, learning_usage_events_1.listEventLogs)(eventDir).map(log => log.path), ...copies.map(name => (0, path_1.join)(dir, name))];
+    const fingerprint = JSON.stringify(paths.map(file => {
+        try {
+            const stat = (0, fs_1.statSync)(file);
+            return [file, stat.size, stat.mtimeMs, stat.ctimeMs, stat.ino];
+        }
+        catch {
+            return [file, 'missing'];
+        }
+    }));
+    return { fingerprint, copies };
+}
 function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
-    const snapshot = readUsageSnapshot(storePath);
     const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
-    if ((0, learning_usage_events_1.listEventLogs)(eventDir).length === 0)
-        return snapshot;
+    const before = usageReadState(storePath, eventDir);
+    const snapshot = readUsageSnapshot(storePath);
+    const logs = (0, learning_usage_events_1.listEventLogs)(eventDir);
+    const finish = (store) => {
+        const after = usageReadState(storePath, eventDir);
+        const observations = [...(store.evidence?.observations ?? [])];
+        if (before.fingerprint !== after.fingerprint)
+            observations.push('inputs-changed-during-read');
+        if (after.copies.length)
+            observations.push('unmerged-store-copies');
+        if (store.consolidated && Object.keys(store.foldedThroughBytes ?? {}).some(machineId => !logs.some(log => log.machineId === machineId)))
+            observations.push('consolidated-journal-missing');
+        if (observations.length > (store.evidence?.observations.length ?? 0))
+            withEvidence(store, {
+                status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [], observations,
+            });
+        flagIntegrity(store);
+        return store;
+    };
+    if (logs.length === 0 || snapshot.evidence?.observations.includes('unsupported-schema'))
+        return finish(snapshot);
     const fold = (base) => foldUsageEvents({
         eventDir,
         limits: options.limits,
@@ -442,14 +536,20 @@ function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
     }
     catch (error) {
         const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
-        if (!backup)
+        if (!backup || backup.evidence)
             throw error;
-        folded = fold(backup);
+        folded = fold(withEvidence(backup, {
+            status: 'reliable', excludedKeys: [], observations: ['primary-journal-mismatch'],
+            recovery: 'checkpoint', persisted: false,
+        }));
     }
+    finish(folded.store);
     const mode = options.refreshSnapshot ?? 'auto';
     const shouldRefresh = mode === 'always'
         || (mode === 'auto' && (folded.tailBytes >= SNAPSHOT_REFRESH_BYTES || !(0, fs_1.existsSync)(storePath) || folded.restarted));
-    if (shouldRefresh) {
+    // Recovery is usable in memory. Persisting a primary replacement requires the
+    // explicit maintenance interval; an ordinary reader never establishes it.
+    if (shouldRefresh && !folded.store.evidence) {
         folded.store.updatedAt = (options.now ?? new Date()).toISOString();
         writeUsageSnapshot(folded.store, storePath);
     }
@@ -636,6 +736,20 @@ function applyEvent(store, event, limits, now) {
     void now;
     return null;
 }
+function isSupportedUsageEvent(event) {
+    if (typeof event.key !== 'string' || !event.key || typeof event.date !== 'string' || !isValidDate(event.date))
+        return false;
+    if (event.family !== 'learning' && event.family !== 'rule')
+        return false;
+    if (typeof event.job !== 'string' || (event.model !== null && typeof event.model !== 'string'))
+        return false;
+    if (event.kind === 'offer')
+        return typeof event.aboveThreshold === 'boolean';
+    if (event.kind === 'firing')
+        return exports.FIRING_OUTCOMES.includes(event.outcome)
+            && (event.source === undefined || event.source === 'attested' || event.source === 'backfill-legacy');
+    return false;
+}
 function applyOfferEvent(store, event, common) {
     const record = ensureRecord(store, common.key, common.family, common.fileType, common.level);
     rememberTitle(record, typeof event.title === 'string' ? event.title : undefined);
@@ -883,19 +997,17 @@ function noteTitleChange(key, newTitle, options) {
     store.updatedAt = (options.now ?? new Date()).toISOString();
     writeUsageSnapshot(store, storePath);
 }
-/**
- * Apply retention and log bounds across the whole store, and compact the event
- * log behind it.
- *
- * Compaction is a requirement rather than an optimization: S4 measured a cold
- * full fold at 84 ms on today's volume but 794 ms at ten times it, and the
- * checkpoint only makes a cold fold rare, not bounded.
- */
 function pruneUsageStore(limits, options = {}) {
     const storePath = options.storePath ?? resolveUsageStorePath();
     const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
     const now = options.now ?? new Date();
     const store = readUsageStore(storePath, { limits, now, eventDir });
+    if (store.evidence && store.evidence.status !== 'reliable')
+        return {
+            maintenanceStatus: 'deferred', evidence: store.evidence,
+            prunedOfferRecords: 0, prunedFiringRecords: 0, entries: Object.keys(store.entries).length,
+            compaction: compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store }),
+        };
     let prunedOfferRecords = 0;
     let prunedFiringRecords = 0;
     for (const record of Object.values(store.entries)) {
@@ -906,7 +1018,10 @@ function pruneUsageStore(limits, options = {}) {
         prunedFiringRecords += firingsBefore - record.fired.log.length;
     }
     const compaction = compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store });
-    return { prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length, compaction };
+    return {
+        maintenanceStatus: compaction.maintenanceStatus, evidence: compaction.evidence,
+        prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length, compaction,
+    };
 }
 /** Save totals and source bookmarks in both copies, then clear processed journals.
  * Requires an interval without concurrent writers or sync activity.
@@ -915,6 +1030,14 @@ function compactUsageEvents(options) {
     const now = options.now ?? new Date();
     const store = options.store
         ?? readUsageStore(options.snapshotPath, { limits: options.limits, now, eventDir: options.eventDir });
+    // Do not turn a partial read into a durable replacement or advance bookmarks
+    // over evidence this reader could not understand. Independent review can run.
+    if (store.evidence && store.evidence.status !== 'reliable')
+        return {
+            maintenanceStatus: 'deferred', evidence: store.evidence,
+            scanned: 0, kept: 0, dropped: 0, skippedLines: 0, logs: [],
+            checkpoints: { ...(store.foldedThroughBytes ?? {}) },
+        };
     for (const log of (0, learning_usage_events_1.listEventLogs)(options.eventDir)) {
         if ((store.foldedThroughBytes?.[log.machineId] ?? 0) < log.baseOffset) {
             throw new Error('Cannot consolidate from a snapshot older than the journal checkpoint.');
@@ -931,7 +1054,8 @@ function compactUsageEvents(options) {
     const temp = checkpoint + '.tmp';
     try {
         (0, fs_1.mkdirSync)((0, path_1.dirname)(checkpoint), { recursive: true });
-        (0, fs_1.writeFileSync)(temp, JSON.stringify(store, null, 2) + '\n', { encoding: 'utf8', flush: true });
+        const { evidence: _evidence, ...snapshot } = store;
+        (0, fs_1.writeFileSync)(temp, JSON.stringify(snapshot, null, 2) + '\n', { encoding: 'utf8', flush: true });
         (0, fs_1.renameSync)(temp, checkpoint);
     }
     catch {
@@ -944,7 +1068,7 @@ function compactUsageEvents(options) {
     if (!writeUsageSnapshot(store, options.snapshotPath)) {
         throw new Error('Cannot consolidate usage events: aggregate snapshot could not be saved.');
     }
-    return (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
+    const compaction = (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
         if (event.kind !== 'offer' && event.kind !== 'firing')
             return true;
         const date = typeof event.date === 'string' ? event.date : '';
@@ -952,4 +1076,5 @@ function compactUsageEvents(options) {
             return true;
         return event.kind === 'firing' && !exports.FIRING_OUTCOMES.includes(event.outcome);
     }, store.foldedThroughBytes ?? {});
+    return { ...compaction, maintenanceStatus: 'completed', evidence: store.evidence };
 }

package/dist/src/local-mcp-server/learning-usage-verify.js

@@ -32,6 +32,7 @@ const path_1 = require("path");
 const learning_usage_store_1 = require("./learning-usage-store");
 const learning_usage_events_1 = require("./learning-usage-events");
 const learning_usage_projection_1 = require("./learning-usage-projection");
+const learning_usage_eligibility_1 = require("./learning-usage-eligibility");
 const CANONICAL_FILE_NAME = 'learning-usage.json';
 /** `learning-usage.json` plus anything a sync client forked off it. */
 const STORE_COPY_PATTERN = /^learning-usage(-.+)?\.json$/;
@@ -164,7 +165,7 @@ function scanRecordsForIntegrity(records, offersEraStart) {
         // unrecorded. An entry whose outcomes are all `ignored` has an empty bucket
         // set correctly, and that is information rather than a gap.
         const clockAdvancing = record.fired.byOutcome.prevented + record.fired.byOutcome.applied;
-        if (clockAdvancing > 0 && Object.keys(record.fired.byModel).length === 0) {
+        if ((0, learning_usage_eligibility_1.modelAttributionIncomplete)(record)) {
             unknownModelAttribution.push({ key: record.key, title, fired: clockAdvancing });
         }
     }
@@ -183,6 +184,7 @@ function buildVerifyReport(options) {
     const checkpoints = snapshot.foldedThroughBytes ?? {};
     const totalLogBytes = logs.reduce((sum, log) => sum + log.size, 0);
     return {
+        evidence: store.evidence,
         snapshotPath: options.snapshotPath,
         eventDir,
         currentModel: (0, learning_usage_projection_1.resolveCurrentModel)(options.currentModel),