fraim 2.0.339 → 2.0.341

package/dist/src/local-mcp-server/learning-usage-command.js

@@ -21,15 +21,10 @@ const learning_usage_attestation_1 = require("./learning-usage-attestation");
 const learning_usage_verify_1 = require("./learning-usage-verify");
 const pack_home_1 = require("../cli/utils/pack-home");
 /**
- * The repair for a usage record that has gone wrong.
- *
- * The record is derived (#1103 R17), so the fix is to rebuild it rather than to
- * reconcile it in place. Measured on a 733-file retrospective corpus: a rebuild
- * lands with zero split identities and one model bucket, because the identity
- * resolution and the model normalization fix the causes and a rebuilt store
- * carries no legacy shape.
+ * Preserve checkpointed history and distinguish backup recovery from attestation backfill.
+ * Backfill supplements older firings; it cannot reconstruct missing offers.
  */
-const REBUILD_NOTE = 'Delete the `usage/` directory and run `learning-usage backfill` to rebuild the record from the retrospectives. The record is derived: every firing is also written in the retrospective that attested it, so a rebuild restores one identity per lesson and one bucket per model. It costs the offer history, which rebuilds itself from live traffic.';
+const REBUILD_NOTE = "Keep `learning-usage.json`, its `.checkpoint` backup, `events/` and any conflict copies. After consolidation, the snapshot and checkpoint backup preserve lifetime counters and already-counted source markers; the remaining journal cannot rebuild that history. A missing or torn snapshot is read from its checkpoint backup automatically. Restore matching checkpoint and journal files from backup if both copies are damaged. Inspect split identities and conflict copies before repairing them; do not delete the snapshot to clear a warning. Separately, `learning-usage backfill` can recover older firings from retrospectives; raw learnings and retrospectives cannot restore runtime offers.";
 function resolveWorkspaceRoot(options) {
     return node_path_1.default.resolve(options.workspaceRoot || options.root || process.cwd());
 }
@@ -282,7 +277,8 @@ const reportCommand = addModelOption(addCommonOptions(new commander_1.Command('r
     });
     emit(report, options.json, () => {
         const lines = [
-            `${report.totals.entriesOffered} entries were offered. ${report.totals.entriesWithFirings} fired. ${report.totals.entriesNeverFired} have never fired.`,
+            report.totals.entriesOffered === null ? 'Corpus totals are unavailable; listed entries have independent usable evidence.'
+                : `${report.totals.entriesOffered} entries were offered. ${report.totals.entriesWithFirings} fired. ${report.totals.entriesNeverFired} have never fired.`,
         ];
         if (report.workingHardest.length) {
             lines.push('', 'Working hardest');
@@ -300,7 +296,7 @@ const reportCommand = addModelOption(addCommonOptions(new commander_1.Command('r
             report.ignoredWhileInContext.forEach((e, i) => lines.push(`${i + 1}. ${e.title}. Attested ignored ${e.ignored} time${e.ignored === 1 ? '' : 's'} (fired ${e.fired}). In context and the work went against it, so the entry needs work rather than another recurrence.`));
         }
         if (report.integrityFlagged > 0) {
-            lines.push('', `${report.integrityFlagged} entr${report.integrityFlagged === 1 ? 'y has' : 'ies have'} an integrity flag: more attested firings than recorded offers. Run \`learning-usage verify\` before acting on anything below.`);
+            lines.push('', `${report.integrityFlagged} entries were excluded from usage conclusions. Run \`learning-usage verify\` for diagnostic details.`);
         }
         lines.push('', 'Nothing is retired from this view. Run sleep-on-learnings to decide.');
         return lines.join('\n');
@@ -455,7 +451,8 @@ const pruneCommand = addCommonOptions(new commander_1.Command('prune'))
     const workspaceRoot = resolveWorkspaceRoot(options);
     const limits = (0, learning_usage_store_1.resolveUsageLimits)(workspaceRoot);
     const result = (0, learning_usage_store_1.pruneUsageStore)(limits);
-    emit({ ...result, storePath: (0, learning_usage_store_1.resolveUsageStorePath)(), limits }, options.json, () => `Pruned ${result.prunedOfferRecords} offer records and ${result.prunedFiringRecords} firing records across ${result.entries} entries. Aggregates unchanged.`);
+    emit({ ...result, storePath: (0, learning_usage_store_1.resolveUsageStorePath)(), limits }, options.json, () => result.maintenanceStatus === 'deferred' ? `Maintenance deferred; history unchanged. ${result.evidence?.observations.join(', ') ?? ''}`
+        : `Pruned ${result.prunedOfferRecords} offer records and ${result.prunedFiringRecords} firing records across ${result.entries} entries. Aggregates unchanged.`);
 });
 function mb(bytes) {
     return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
@@ -477,9 +474,11 @@ const verifyCommand = addModelOption(addCommonOptions(new commander_1.Command('v
             '',
         ];
         const finding = (count, singular, plural, remedy) => count === 0 ? `  none: ${singular}` : `  ${count} ${count === 1 ? singular : plural}. ${remedy}`;
+        if (report.evidence)
+            lines.push(`Evidence: ${report.evidence.status}. ${report.evidence.observations.join(', ')}.`, ...(report.evidence.recovery ? [`Recovered from ${report.evidence.recovery}; primary replacement was not persisted by this read.`] : []), '');
         lines.push('Identity', finding(report.twinPairs.length, 'lesson tracked under two keys', 'lessons tracked under two keys', 'Its offers and its firings are on different records, so every count about it is partial.'), report.unmatchedTitleKeys.length
             ? `  ${report.unmatchedTitleKeys.length} title-keyed record${report.unmatchedTitleKeys.length === 1 ? ' has' : 's have'} no id twin. Expected for entries no tier has stamped; run \`stamp-ids\` to reduce it.`
-            : '  every title-keyed record has been matched or has no twin', '', 'Counters', finding(report.invariantViolations.length, 'entry holds more attested firings than recorded offers', 'entries hold more attested firings than recorded offers', 'An entry cannot fire without being offered, so one of the two counters is wrong. History stranded in a store copy nothing reads is the usual cause.'), finding(report.caseVariantModelBuckets.length, 'model bucket differs only by case', 'model buckets differ only by case', 'One model spelled two ways splits its own denominator, so every per-model conclusion about it is computed on a fraction of its traffic.'), finding(report.unknownModelAttribution.length, 'entry fired with no model recorded', 'entries fired with no model recorded', 'Per-model retirement cannot be answered for these, and they are held back rather than counted as zero. These were attested before record-firings carried --agent and --model, so the model was never recorded against them and a rebuild cannot invent it; attestations recorded from now on carry both.'), '', 'Storage');
+            : '  every title-keyed record has been matched or has no twin', '', 'Counters', finding(report.invariantViolations.length, 'entry holds more attested firings than recorded offers', 'entries hold more attested firings than recorded offers', 'The counts are inconsistent. This observation does not establish their cause.'), finding(report.caseVariantModelBuckets.length, 'model bucket differs only by case', 'model buckets differ only by case', 'One model spelled two ways splits its own denominator, so every per-model conclusion about it is computed on a fraction of its traffic.'), finding(report.unknownModelAttribution.length, 'entry has incomplete model attribution', 'entries have incomplete model attribution', 'Affected model conclusions are withheld. The cause is not established; unknown historical attribution cannot be inferred from the current model.'), '', 'Storage');
         if (report.storeCopies.length === 0) {
             lines.push('  one store file, no unmerged copies');
         }
@@ -488,14 +487,14 @@ const verifyCommand = addModelOption(addCommonOptions(new commander_1.Command('v
             for (const copy of report.storeCopies) {
                 lines.push(`    ${copy.path}: ${copy.readable ? `${copy.entries} entries, ${copy.offers} offers, ${copy.firings} firings` : `unreadable (${copy.error ?? 'unknown error'})`}`);
             }
-            lines.push('    Nothing reads these. Their history is recoverable only through a rebuild.');
+            lines.push('    Nothing reads these. Preserve them and the checkpoint backup for inspection; do not delete snapshots to clear this warning.');
         }
         lines.push(`  event logs folded: ${report.eventLogs.length === 0 ? 'none' : report.eventLogs.map((l) => `${l.machineId} (${l.bytes} bytes)`).join(', ')}`);
         for (const [machineId, lag] of Object.entries(report.checkpointLag)) {
             lines.push(`  checkpoint lag for ${machineId}: ${lag} bytes${lag > 0 ? ' behind the log head' : ''}`);
         }
         lines.push(report.compactionAdvised
-            ? `  log size ${mb(report.totalLogBytes)} is past the ${mb(report.compactionAdvisoryBytes)} point where a cold fold stops fitting the context-injection budget. Run \`learning-usage prune\` to compact it to the configured retention window.`
+            ? `  log size ${mb(report.totalLogBytes)} is past the ${mb(report.compactionAdvisoryBytes)} point where a cold fold stops fitting the context-injection budget. Run \`learning-usage prune\` to consolidate processed events into the snapshot and checkpoint backup, then clear them independently of retention.`
             : `  log size ${mb(report.totalLogBytes)} of ${mb(report.compactionAdvisoryBytes)} before compaction is worth running.`);
         if (report.rebuildAdvised)
             lines.push('', REBUILD_NOTE);

package/dist/src/local-mcp-server/learning-usage-eligibility.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.modelAttributionIncomplete = modelAttributionIncomplete;
+exports.countEvidenceProblem = countEvidenceProblem;
+exports.usageEvidenceExclusions = usageEvidenceExclusions;
+const learning_usage_store_1 = require("./learning-usage-store");
+const validCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
+/** Missing attribution is unknown, including a known bucket beside unattributed history. */
+function modelAttributionIncomplete(record) {
+    const buckets = [record.offered.byModel, record.fired.byModel];
+    if (buckets.some(map => !map || Object.entries(map).some(([name, n]) => !validCount(n) || !(0, learning_usage_store_1.normalizeModelName)(name) || (0, learning_usage_store_1.normalizeModelName)(name) !== name)))
+        return true;
+    return Object.values(record.offered.byModel).reduce((a, b) => a + b, 0) !== record.offered.total
+        || Object.values(record.fired.byModel).reduce((a, b) => a + b, 0)
+            !== record.fired.byOutcome.applied + record.fired.byOutcome.prevented;
+}
+/** Validate counts independently of a cached integrity flag. Never alter the record. */
+function countEvidenceProblem(record, offersEraStart) {
+    const counts = [record.offered.total, record.offered.aboveThreshold, record.fired.total,
+        ...Object.values(record.offered.byModel), ...Object.values(record.fired.byModel),
+        ...Object.values(record.fired.byOutcome)];
+    if (counts.some(n => !validCount(n)) || learning_usage_store_1.FIRING_OUTCOMES.some(outcome => !validCount(record.fired.byOutcome[outcome]))
+        || record.offered.aboveThreshold > record.offered.total
+        || Object.values(record.fired.byOutcome).reduce((a, b) => a + b, 0) !== record.fired.total) {
+        return 'Usage counters are inconsistent.';
+    }
+    const violation = record.integrity ?? (0, learning_usage_store_1.integrityViolation)(record, offersEraStart);
+    if (!offersEraStart && record.offered.total > 0 && record.fired.total > record.offered.total) {
+        return 'Excess uses lack provenance establishing the offer-recording era.';
+    }
+    return violation ? violation.detail : null;
+}
+/** Read-time exclusions follow the actual record, never ambient UI or model state. */
+function usageEvidenceExclusions(store) {
+    const excluded = new Map();
+    for (const key of store.evidence?.excludedKeys ?? [])
+        excluded.set(key, 'Unreadable usage record.');
+    const byTitle = new Map();
+    for (const record of Object.values(store.entries)) {
+        const problem = countEvidenceProblem(record, store.offersEraStart);
+        if (store.evidence?.status === 'unavailable')
+            excluded.set(record.key, 'Usage history is unavailable.');
+        else if (problem)
+            excluded.set(record.key, problem);
+        if (!record.fileType)
+            continue;
+        for (const title of record.titles) {
+            const titleKey = (0, learning_usage_store_1.titleUsageKey)(record.fileType, title);
+            byTitle.set(titleKey, [...new Set([...(byTitle.get(titleKey) ?? []), record.key])]);
+        }
+    }
+    for (const keys of byTitle.values()) {
+        if (keys.length > 1 && keys.some(key => key.startsWith('T:'))) {
+            for (const key of keys)
+                excluded.set(key, 'Ambiguous duplicate usage identity.');
+        }
+    }
+    return excluded;
+}

package/dist/src/local-mcp-server/learning-usage-events.js

@@ -7,43 +7,17 @@ exports.listEventLogs = listEventLogs;
 exports.readEventsSince = readEventsSince;
 exports.compactEventLogs = compactEventLogs;
 /**
- * Issue #1752 — the durable half of the usage record.
+ * Per-machine usage journals: job runs append complete JSON lines; sleep folds
+ * them into a recoverable snapshot and removes processed records.
  *
- * The store used to be one mutable JSON document rewritten by every writer with a
- * plain `writeFileSync`. Measured (spike S1b): eight concurrent processes doing
- * that on a *plain local disk*, no sync folder involved, lost 1589 of 1600
- * writes. The four OneDrive conflict copies in issue #1752 are a symptom of the
- * same defect, not its cause.
+ * A cleared journal contains only {_usageJournalThroughBytes:N}. This framing
+ * marker preserves its logical position across atomic replacement, so a crash
+ * between checkpoint publication and journal cleanup cannot reset offsets or
+ * count an event twice. It contains no activity details. New appends continue
+ * from N; an older/missing snapshot cannot silently cold-fold compacted history.
  *
- * So the durable artifact is an append-only log, one file per machine:
- *
- *   OFFER/FIRING event  →  appendFileSync of one whole JSON line
- *   usage/learning-usage.json  →  a derived snapshot with a byte checkpoint
- *
- * Three measurements decided the shape, and each closed off an alternative:
- *
- *   S2   16 processes x 1000 lines on the real OneDrive-synced path produced
- *        16000/16000 complete parseable lines, 0 interleaved, 0 conflict copies.
- *        That is what removes the need for a same-machine lock. The per-machine
- *        filename is what removes the cross-machine question, because two
- *        machines then never write one file.
- *   S3   `renameSync` over a target any process holds open fails EPERM 100 times
- *        in 100 on Windows, on a synced path *and* a local one, because Node's
- *        `openSync(path,'r')` does not request FILE_SHARE_DELETE. The usage store
- *        is read by concurrent MCP processes by design, so rename-publish is not
- *        available here. Recorded so it is not re-proposed. See also #820.
- *   S4b  A full fold is linear: 84 ms at today's volume, 794 ms at ten times it.
- *        A snapshot plus a tail fold is 2.6 ms at both 9.9 MB and 79 MB. Flat.
- *
- * This module owns the log files and nothing else: framing, appending, reading a
- * tail from a byte offset, and compaction. It knows a line is a JSON object and
- * knows nothing about what a usage record looks like. That is deliberate — the
- * record shape lives in `learning-usage-store.ts`, which imports this, so there
- * is no cycle between them.
- *
- * The same primitive is already used by `src/ai-hub/raw-event-log-store.ts`
- * (append one JSON line, track byte offsets); this is the learning-usage
- * equivalent rather than a second general mechanism.
+ * Maintenance requires no concurrent writers or sync activity. Ordinary writers
+ * append one payload; different machines use different filenames.
  */
 const fs_1 = require("fs");
 const os_1 = require("os");
@@ -51,6 +25,7 @@ const path_1 = require("path");
 /** Subdirectory of the usage folder holding the per-machine logs. */
 const EVENT_DIR_NAME = 'events';
 const LOG_SUFFIX = '.jsonl';
+const CHECKPOINT_MARKER = '_usageJournalThroughBytes';
 /** The log directory that belongs to a snapshot: `<usage>/events/`. */
 function resolveEventDir(snapshotPath) {
     return (0, path_1.join)((0, path_1.dirname)(snapshotPath), EVENT_DIR_NAME);
@@ -136,10 +111,25 @@ function listEventLogs(eventDir) {
             continue;
         const path = (0, path_1.join)(eventDir, name);
         let size = 0;
+        let baseOffset = 0;
+        let headerBytes = 0;
         try {
             const fd = (0, fs_1.openSync)(path, 'r');
             try {
                 size = (0, fs_1.fstatSync)(fd).size;
+                const prefix = Buffer.alloc(Math.min(size, 128));
+                (0, fs_1.readSync)(fd, prefix, 0, prefix.length, 0);
+                const newline = prefix.indexOf(0x0a);
+                if (newline >= 0) {
+                    try {
+                        const header = JSON.parse(prefix.subarray(0, newline).toString('utf8'));
+                        if (Object.keys(header).length === 1 && Number.isSafeInteger(header[CHECKPOINT_MARKER]) && header[CHECKPOINT_MARKER] >= 0) {
+                            baseOffset = header[CHECKPOINT_MARKER];
+                            headerBytes = newline + 1;
+                        }
+                    }
+                    catch { /* An ordinary first event, not a checkpoint marker. */ }
+                }
             }
             finally {
                 (0, fs_1.closeSync)(fd);
@@ -148,14 +138,20 @@ function listEventLogs(eventDir) {
         catch {
             continue;
         }
-        logs.push({ machineId: name.slice(0, -LOG_SUFFIX.length), path, size });
+        logs.push({ machineId: name.slice(0, -LOG_SUFFIX.length), path, size, baseOffset, headerBytes });
     }
     return logs.sort((a, b) => a.machineId.localeCompare(b.machineId));
 }
 function readTail(log, from) {
-    const restarted = from > log.size;
+    const end = log.baseOffset + log.size - log.headerBytes;
+    if (from < log.baseOffset)
+        throw new Error('Usage checkpoint is missing or older than the consolidated journal. Restore learning-usage.json or its .checkpoint backup; raw retrospectives cannot restore runtime offers.');
+    const restarted = from > end;
+    if (restarted && log.baseOffset > 0)
+        throw new Error('Usage journal is behind its checkpoint; restore matching files before continuing.');
     const start = restarted ? 0 : Math.max(0, from);
-    const length = log.size - start;
+    const physicalStart = start - log.baseOffset + log.headerBytes;
+    const length = log.size - physicalStart;
     if (length <= 0) {
         return { ...log, records: [], skippedLines: 0, through: start, restarted };
     }
@@ -164,7 +160,7 @@ function readTail(log, from) {
         const fd = (0, fs_1.openSync)(log.path, 'r');
         try {
             const buffer = Buffer.alloc(length);
-            const read = (0, fs_1.readSync)(fd, buffer, 0, length, start);
+            const read = (0, fs_1.readSync)(fd, buffer, 0, length, physicalStart);
             text = buffer.subarray(0, read).toString('utf8');
         }
         finally {
@@ -202,7 +198,7 @@ function readTail(log, from) {
             skippedLines += 1;
         }
     }
-    return { ...log, records, skippedLines, through: log.size - partialBytes, restarted };
+    return { ...log, records, skippedLines, through: end - partialBytes, restarted };
 }
 /**
  * Read every log's tail past its checkpoint.
@@ -220,32 +216,42 @@ function readEventsSince(eventDir, checkpoints) {
     };
 }
 /**
- * Rewrite every log keeping only the records `keep` accepts.
- *
- * Required rather than an optimization: S4 measured a cold full fold growing
- * linearly to 794 ms at ten times today's volume, and the checkpoint only makes a
- * cold fold rare, not bounded.
- *
- * Published by writing a temp file and renaming it. That is safe here and not on
- * the snapshot (S3), because nothing holds a compacted log open for reading in
- * the way concurrent MCP processes hold the snapshot: this runs from the CLI, and
- * a failed rename leaves the original log untouched.
+ * Clear fully checkpointed logs whose records are all understood by this reader.
+ * Preserve unknown, malformed, partial or newly appended logs byte-for-byte.
+ * Flush then atomically replace each log with its logical-position header; the
+ * checkpoint saved before this call remains correct on either side of a crash.
  */
-function compactEventLogs(eventDir, keep) {
-    const result = { scanned: 0, kept: 0, dropped: 0, skippedLines: 0, logs: [] };
+function compactEventLogs(eventDir, keep, checkpoints) {
+    const result = { scanned: 0, kept: 0, dropped: 0, skippedLines: 0, logs: [], checkpoints: { ...checkpoints } };
     for (const log of listEventLogs(eventDir)) {
-        const read = readTail(log, 0);
+        const read = readTail(log, log.baseOffset);
         result.scanned += read.records.length;
         result.skippedLines += read.skippedLines;
+        result.kept += read.records.length;
+        // The snapshot knows nothing about a later append or torn
+        // tail. Leave that log and its old checkpoint alone for the next fold.
+        if (read.through !== log.baseOffset + log.size - log.headerBytes
+            || read.through !== (checkpoints[log.machineId] ?? 0) || read.skippedLines > 0)
+            continue;
         const retained = read.records.filter(keep);
-        result.kept += retained.length;
-        result.dropped += read.records.length - retained.length;
-        if (retained.length === read.records.length && read.skippedLines === 0)
+        // Unknown records may belong to a newer reader. Leave their whole log
+        // intact so their original positions remain meaningful.
+        if (retained.length > 0 || read.records.length === 0)
             continue;
         const temp = `${log.path}.compacting`;
         try {
-            (0, fs_1.writeFileSync)(temp, retained.length ? `${retained.map((r) => JSON.stringify(r)).join('\n')}\n` : '', 'utf8');
+            const payload = `${JSON.stringify({ [CHECKPOINT_MARKER]: read.through })}\n`;
+            (0, fs_1.writeFileSync)(temp, payload, { encoding: 'utf8', flush: true });
+            if ((0, fs_1.statSync)(log.path).size !== log.size) {
+                (0, fs_1.rmSync)(temp, { force: true });
+                continue;
+            }
             (0, fs_1.renameSync)(temp, log.path);
+            const dropped = read.records.length - retained.length;
+            result.kept -= dropped;
+            result.dropped += dropped;
+            // The logical checkpoint remains unchanged across the atomic replacement.
+            result.checkpoints[log.machineId] = read.through;
         }
         catch {
             // The log is authoritative and the original is still intact, so a failed

package/dist/src/local-mcp-server/learning-usage-projection.js

@@ -15,6 +15,7 @@ exports.resolveCurrentModel = resolveCurrentModel;
  * Nothing here writes.
  */
 const learning_usage_store_1 = require("./learning-usage-store");
+const learning_usage_eligibility_1 = require("./learning-usage-eligibility");
 function toLookupValue(record) {
     return {
         key: record.key,
@@ -24,7 +25,7 @@ function toLookupValue(record) {
         firedTotalAttestations: record.fired.total,
         lastFired: record.fired.lastFiredAt,
         lastOffered: record.offered.lastOfferedAt,
-        firedUnder: Object.keys(record.fired.byModel),
+        firedUnder: (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record) ? undefined : Object.keys(record.fired.byModel),
         byOutcome: record.fired.byOutcome,
         log: record.fired.log,
     };
@@ -36,7 +37,10 @@ function toLookupValue(record) {
  */
 function buildUsageLookup(store) {
     const lookup = new Map();
+    const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
     for (const record of Object.values(store.entries)) {
+        if (excluded.has(record.key))
+            continue;
         const value = toLookupValue(record);
         lookup.set(record.key, value);
         if (record.fileType) {