fraim 2.0.337 → 2.0.339

package/dist/src/local-mcp-server/learning-usage-verify.js

@@ -0,0 +1,222 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.COMPACTION_ADVISORY_BYTES = void 0;
+exports.listUnmergedStoreCopies = listUnmergedStoreCopies;
+exports.findTwinPairs = findTwinPairs;
+exports.buildVerifyReport = buildVerifyReport;
+/**
+ * Issue #1752 E2 — the integrity picture for the usage record, in one output.
+ *
+ * Every defect in issue #1752 was found by a manager reading raw JSON during a
+ * `sleep-on-learnings` run, one at a time, over one session. This is the command
+ * that answers "can I trust what the record just told me" before acting on a
+ * recommendation, and it is the regression guard that makes every defect class
+ * visible in one place instead of one at a time.
+ *
+ * It reports and never blocks. Nothing here treats a finding as a gate: it states
+ * what it found, and the manager decides.
+ *
+ * ## Why there is no repair command here
+ *
+ * The usage record is derived data (#1103 R17): every firing is also written in
+ * the retrospective that attested it, and every offer is re-recorded the next
+ * time the entry is delivered. So the repair for a store that has gone wrong is
+ * to delete it and rebuild, not to reconcile it in place.
+ *
+ * That is not a preference, it is what the numbers say. Measured on a 733-file
+ * retrospective corpus with four unmerged store copies beside the canonical one:
+ *
+ *   delete `usage/` then `learning-usage backfill`  ->  149 entries, 2874 firings,
+ *                                                       0 split identities,
+ *                                                       1 clean model bucket
+ *
+ * A rebuild lands with zero split identities and one model bucket for free,
+ * because the identity resolution and the model normalization fix the causes and
+ * a rebuilt store carries no legacy shape. The only thing an in-place
+ * reconciliation recovers beyond that is old firings whose retrospectives
+ * retention has already deleted, and a decay clock reads the most recent firing
+ * rather than the count. What a rebuild does cost is the offer history, which
+ * rebuilds itself from live traffic and degrades toward `undetermined` and
+ * hold-back in the meantime, which is this module's posture anyway.
+ *
+ * So `verify` names the rebuild when it finds a stale shape, and the destructive
+ * step stays the manager's.
+ */
+const fs_1 = require("fs");
+const path_1 = require("path");
+const learning_usage_store_1 = require("./learning-usage-store");
+const learning_usage_events_1 = require("./learning-usage-events");
+const learning_usage_projection_1 = require("./learning-usage-projection");
+const CANONICAL_FILE_NAME = 'learning-usage.json';
+/** `learning-usage.json` plus anything a sync client forked off it. */
+const STORE_COPY_PATTERN = /^learning-usage(-.+)?\.json$/;
+/**
+ * The size past which a cold fold stops fitting the 150 ms context-injection
+ * budget. Spike S4 measured 84 ms at 9.86 MB and 794 ms at 79 MB, so the budget
+ * runs out around here. Compaction is operator-triggered (`learning-usage
+ * prune`), so this is the number that tells the operator when to run it.
+ */
+exports.COMPACTION_ADVISORY_BYTES = 16 * 1024 * 1024;
+/**
+ * Deliberately looser than the store's own `isUsableRecord`.
+ *
+ * That one guards what a consumer will dereference, so it drops anything it
+ * cannot hand out safely. This one decides what `verify` will count, and a record
+ * it rejects is a finding that never gets reported, so it checks only the two
+ * counter objects and the title list.
+ */
+function usableRecord(value) {
+    if (!value || typeof value !== 'object')
+        return false;
+    const record = value;
+    return !!record.offered && typeof record.offered === 'object'
+        && !!record.fired && typeof record.fired === 'object'
+        && Array.isArray(record.titles);
+}
+function parseCopy(path) {
+    const summary = { path, readable: false, entries: 0, offers: 0, firings: 0 };
+    try {
+        const parsed = JSON.parse((0, fs_1.readFileSync)(path, 'utf8'));
+        if (!parsed || typeof parsed !== 'object' || !parsed.entries || typeof parsed.entries !== 'object' || Array.isArray(parsed.entries)) {
+            summary.error = 'No readable entries object.';
+            return summary;
+        }
+        const entries = [];
+        for (const value of Object.values(parsed.entries)) {
+            if (usableRecord(value))
+                entries.push(value);
+        }
+        summary.readable = true;
+        summary.entries = entries.length;
+        summary.offers = entries.reduce((sum, r) => sum + (r.offered.total ?? 0), 0);
+        summary.firings = entries.reduce((sum, r) => sum + (r.fired.seenSources?.length ?? 0), 0);
+    }
+    catch (error) {
+        summary.error = error instanceof Error ? error.message : String(error);
+    }
+    return summary;
+}
+/** The store copies beside the canonical snapshot: history nothing reads. */
+function listUnmergedStoreCopies(snapshotPath) {
+    const dir = (0, path_1.dirname)(snapshotPath);
+    if (!(0, fs_1.existsSync)(dir))
+        return [];
+    let names;
+    try {
+        names = (0, fs_1.readdirSync)(dir);
+    }
+    catch {
+        return [];
+    }
+    return names
+        .filter((name) => STORE_COPY_PATTERN.test(name) && name !== CANONICAL_FILE_NAME)
+        .sort()
+        .map((name) => parseCopy((0, path_1.join)(dir, name)));
+}
+/**
+ * Find lessons tracked under both a title-derived key and an id key.
+ *
+ * Matching is the title under the same family and nothing else. #1103 R5 forbids
+ * resolving a reference to the closest title, and reporting a wrong pair here
+ * would tell the manager two unrelated lessons are one.
+ */
+function findTwinPairs(entries) {
+    const idRecordsByTitleKey = new Map();
+    for (const record of Object.values(entries)) {
+        if (record.key.startsWith('T:') || !record.fileType)
+            continue;
+        for (const title of record.titles) {
+            idRecordsByTitleKey.set((0, learning_usage_store_1.titleUsageKey)(record.fileType, title), record.key);
+        }
+    }
+    const pairs = [];
+    const unmatched = [];
+    for (const record of Object.values(entries)) {
+        if (!record.key.startsWith('T:'))
+            continue;
+        const idKey = idRecordsByTitleKey.get(record.key);
+        if (idKey)
+            pairs.push({ titleKey: record.key, idKey, title: record.titles[record.titles.length - 1] ?? record.key });
+        else
+            unmatched.push(record.key);
+    }
+    return { pairs, unmatched };
+}
+/**
+ * The three per-record defect classes, collected in one pass.
+ *
+ * Split out of `buildVerifyReport` so that function reads as what it is: an
+ * assembly of independently-computed parts.
+ */
+function scanRecordsForIntegrity(records, offersEraStart) {
+    const invariantViolations = [];
+    const caseVariantModelBuckets = [];
+    const unknownModelAttribution = [];
+    for (const record of records) {
+        const title = record.titles[record.titles.length - 1] ?? record.key;
+        const violation = record.integrity ?? (0, learning_usage_store_1.integrityViolation)(record, offersEraStart);
+        if (violation) {
+            invariantViolations.push({
+                key: record.key, title,
+                offered: record.offered.total,
+                firedInEra: record.offered.total + violation.count,
+                detail: violation.detail,
+            });
+        }
+        for (const buckets of [record.offered.byModel, record.fired.byModel]) {
+            const byNormalized = new Map();
+            for (const raw of Object.keys(buckets)) {
+                const normalized = (0, learning_usage_store_1.normalizeModelName)(raw) ?? raw;
+                byNormalized.set(normalized, [...(byNormalized.get(normalized) ?? []), raw]);
+            }
+            for (const [normalized, raws] of byNormalized) {
+                if (raws.length > 1 || raws[0] !== normalized) {
+                    caseVariantModelBuckets.push({ key: record.key, title, buckets: raws, normalized });
+                }
+            }
+        }
+        // Only a clock-advancing firing that produced no bucket means the model went
+        // unrecorded. An entry whose outcomes are all `ignored` has an empty bucket
+        // set correctly, and that is information rather than a gap.
+        const clockAdvancing = record.fired.byOutcome.prevented + record.fired.byOutcome.applied;
+        if (clockAdvancing > 0 && Object.keys(record.fired.byModel).length === 0) {
+            unknownModelAttribution.push({ key: record.key, title, fired: clockAdvancing });
+        }
+    }
+    return { invariantViolations, caseVariantModelBuckets, unknownModelAttribution };
+}
+function buildVerifyReport(options) {
+    const now = options.now ?? new Date();
+    const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(options.snapshotPath);
+    const store = (0, learning_usage_store_1.readUsageStore)(options.snapshotPath, { limits: options.limits, now, eventDir, refreshSnapshot: 'never' });
+    const snapshot = (0, learning_usage_store_1.readUsageSnapshot)(options.snapshotPath);
+    const logs = (0, learning_usage_events_1.listEventLogs)(eventDir);
+    const records = Object.values(store.entries);
+    const { invariantViolations, caseVariantModelBuckets, unknownModelAttribution } = scanRecordsForIntegrity(records, store.offersEraStart);
+    const twins = findTwinPairs(store.entries);
+    const storeCopies = listUnmergedStoreCopies(options.snapshotPath);
+    const checkpoints = snapshot.foldedThroughBytes ?? {};
+    const totalLogBytes = logs.reduce((sum, log) => sum + log.size, 0);
+    return {
+        snapshotPath: options.snapshotPath,
+        eventDir,
+        currentModel: (0, learning_usage_projection_1.resolveCurrentModel)(options.currentModel),
+        entries: records.length,
+        twinPairs: twins.pairs,
+        unmatchedTitleKeys: twins.unmatched,
+        invariantViolations,
+        caseVariantModelBuckets,
+        unknownModelAttribution,
+        storeCopies,
+        eventLogs: logs.map((log) => ({ machineId: log.machineId, path: log.path, bytes: log.size })),
+        // The one number that makes a stale read visible: a snapshot that stops
+        // advancing while the log grows is a failing snapshot write, which S3 showed
+        // will happen intermittently on a synced folder and is otherwise silent.
+        checkpointLag: Object.fromEntries(logs.map((log) => [log.machineId, Math.max(0, log.size - (checkpoints[log.machineId] ?? 0))])),
+        skippedLines: 0,
+        totalLogBytes,
+        compactionAdvisoryBytes: exports.COMPACTION_ADVISORY_BYTES,
+        compactionAdvised: totalLogBytes >= exports.COMPACTION_ADVISORY_BYTES,
+        rebuildAdvised: storeCopies.length > 0 || twins.pairs.length > 0,
+    };
+}

package/dist/src/local-mcp-server/stdio-server.js

@@ -1612,17 +1612,8 @@ class FraimLocalMCPServer {
         // 4. After get_fraim_job succeeds, inject learning context with job-focus frame.
         if (toolName === 'get_fraim_job' && !finalizedResponse.error) {
             const text = finalizedResponse.result?.content?.[0]?.text;
-            const userEmail = this.ensureEngine().getUserEmail();
-            if (typeof text === 'string' && userEmail) {
-                const workspaceRoot = this.findProjectRoot() || process.cwd();
-                const jobDomain = await this.resolveJobDomainForJob(args.job, requestSessionId);
-                const learningSection = (0, learning_context_builder_js_1.buildLearningContextSection)(workspaceRoot, userEmail, true, jobDomain);
-                const teamContextSection = (0, learning_context_builder_js_1.buildTeamContextSection)(workspaceRoot, true);
-                if (learningSection || teamContextSection) {
-                    finalizedResponse.result.content[0].text = text + `\n\n---` + learningSection + teamContextSection;
-                    this.log(`[req:${requestId}] Injected job-focus learning/team context for ${userEmail} (domain: ${jobDomain ?? 'global'})`);
-                }
-                this.recordUsageOffers(workspaceRoot, userEmail, true, jobDomain, String(args.job ?? 'unknown'), requestId);
+            if (typeof text === 'string') {
+                finalizedResponse.result.content[0].text = await this.appendJobContext(text, args.job, requestSessionId, requestId);
             }
             // Issue #1345: forward executionMode from get_fraim_job response to Hub so
             // the chip reads from run data (no local file read, no separate endpoint needed).
@@ -1640,6 +1631,22 @@ class FraimLocalMCPServer {
         }
         return this.processResponseWithHydration(finalizedResponse, requestSessionId);
     }
+    /** Keep context delivery and offer attribution together for local and remote jobs. */
+    async appendJobContext(text, jobName, requestSessionId, requestId) {
+        const userEmail = this.ensureEngine().getUserEmail();
+        if (!userEmail)
+            return text;
+        const workspaceRoot = this.findProjectRoot() || process.cwd();
+        const jobDomain = await this.resolveJobDomainForJob(jobName, requestSessionId);
+        const learningSection = (0, learning_context_builder_js_1.buildLearningContextSection)(workspaceRoot, userEmail, true, jobDomain);
+        const teamContextSection = (0, learning_context_builder_js_1.buildTeamContextSection)(workspaceRoot, true);
+        if (learningSection || teamContextSection) {
+            text += `\n\n---` + learningSection + teamContextSection;
+            this.log(`[req:${requestId}] Injected job-focus learning/team context for ${userEmail} (domain: ${jobDomain ?? 'global'})`);
+        }
+        this.recordUsageOffers(workspaceRoot, userEmail, true, jobDomain, String(jobName ?? 'unknown'), requestId);
+        return text;
+    }
     /**
      * Issue #1103 R1, R2 and R3: record what this context injection delivered.
      *
@@ -1672,12 +1679,17 @@ class FraimLocalMCPServer {
                 job,
                 date,
                 agent: collector?.getAgentName() ?? null,
-                model: collector?.getAgentModel() ?? null,
+                // Issue #1752 C2: one model, one bucket. `GPT-5` and `gpt-5` were two,
+                // splitting the denominator of every per-model conclusion about it.
+                model: (0, learning_usage_store_js_1.normalizeModelName)(collector?.getAgentModel()),
             })), { limits });
-            this.log(`[req:${requestId}] Recorded ${result.recorded} usage offers for job ${job}`);
+            this.log(`[req:${requestId}] Recorded ${result.recorded} usage offers for job ${job} (write mode: append)`);
         }
         catch (error) {
-            this.log(`[req:${requestId}] Usage offer recording skipped: ${error instanceof Error ? error.message : String(error)}`);
+            // Issue #1752: name the log and the write mode. During the migration window
+            // "which path did this write take" is the first question on any report of
+            // missing data, and the answer was not in the logs.
+            this.log(`[req:${requestId}] Usage offer recording skipped (append, log ${(0, learning_usage_store_js_1.resolveEventDir)((0, learning_usage_store_js_1.resolveUsageStorePath)())}): ${error instanceof Error ? error.message : String(error)}`);
         }
     }
     /**
@@ -2627,18 +2639,7 @@ class FraimLocalMCPServer {
                                 responseText = `${mentor.getCompactPhaseAuthority()}\n\n${responseText}`;
                                 responseText += `\n\n---\n\n**Job ID:** \`${jobId}\`\n\n**This job has phases.** Use \`seekMentoring\` with the jobId above to get phase-specific instructions.`;
                             }
-                            // Inject local learning context for job requests (RFC 177).
-                            const userEmail = this.ensureEngine().getUserEmail();
-                            if (userEmail) {
-                                const workspaceRoot = this.findProjectRoot() || process.cwd();
-                                const jobDomain = await this.resolveJobDomainForJob(name, requestSessionId);
-                                const learningSection = (0, learning_context_builder_js_1.buildLearningContextSection)(workspaceRoot, userEmail, true, jobDomain);
-                                const teamContextSection = (0, learning_context_builder_js_1.buildTeamContextSection)(workspaceRoot, true);
-                                if (learningSection || teamContextSection) {
-                                    responseText += `\n\n---` + learningSection + teamContextSection;
-                                    this.log(`✅ Injected job-focus learning/team context for ${userEmail} (local override path, domain: ${jobDomain ?? 'global'})`);
-                                }
-                            }
+                            responseText = await this.appendJobContext(responseText, name, requestSessionId, requestId);
                             return await this.finalizeLocalToolTextResponse(request, requestSessionId, requestId, responseText);
                         }
                     }

package/package.json

@@ -1,12 +1,14 @@
 {
   "name": "fraim",
-  "version": "2.0.337",
+  "version": "2.0.339",
   "description": "FRAIM core CLI and MCP package.",
   "main": "index.js",
   "bin": {
     "fraim": "bin/fraim.js"
   },
   "files": [
+    "dist/registry/scripts/learning/learning-domains.json",
+    "dist/registry/scripts/learning/entry-format.js",
     "dist/src/local-mcp-server/",
     "dist/src/cli/",
     "!dist/src/cli/fraim-hub.js",