fraim 2.0.346 → 2.0.348
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/registry/scripts/learning/completion-state.js +80 -4
- package/dist/registry/scripts/learning/corpus-audit.js +7 -7
- package/dist/registry/scripts/learning/entry-authoring.js +181 -0
- package/dist/registry/scripts/learning/entry-format.js +10 -1
- package/dist/registry/scripts/learning/outcome-assessment.js +162 -0
- package/dist/registry/scripts/learning/outcome-types.js +92 -0
- package/dist/registry/scripts/learning/owner-identity.js +11 -0
- package/dist/registry/scripts/learning/proposal-queue.js +81 -0
- package/dist/registry/scripts/learning/run-application.js +1003 -0
- package/dist/registry/scripts/learning/run-publication.js +364 -0
- package/dist/registry/scripts/learning/run-record.js +895 -0
- package/dist/registry/scripts/learning-usage/learning-source-record.js +131 -0
- package/dist/src/cli/commands/learning.js +64 -0
- package/dist/src/cli/doctor/checks/scripts-checks.js +53 -1
- package/dist/src/cli/fraim.js +2 -0
- package/dist/src/cli/learning/inventory.js +285 -0
- package/dist/src/cli/learning/queue-operations.js +199 -0
- package/dist/src/cli/learning/runtime.js +82 -0
- package/dist/src/cli/learning/usage.js +277 -0
- package/dist/src/core/fraim-config-schema.generated.js +0 -4
- package/dist/src/local-mcp-server/learning-context-builder.js +91 -377
- package/dist/src/local-mcp-server/learning-entry-format.js +25 -0
- package/dist/src/local-mcp-server/learning-usage-analysis.js +12 -410
- package/dist/src/local-mcp-server/learning-usage-eligibility.js +8 -14
- package/dist/src/local-mcp-server/learning-usage-projection.js +5 -22
- package/dist/src/local-mcp-server/learning-usage-store.js +93 -393
- package/dist/src/local-mcp-server/learning-usage-verify.js +9 -40
- package/dist/src/local-mcp-server/stdio-server.js +0 -48
- package/package.json +9 -1
- package/dist/src/local-mcp-server/learning-firing-parser.js +0 -247
- package/dist/src/local-mcp-server/learning-usage-attestation.js +0 -238
- package/dist/src/local-mcp-server/learning-usage-command.js +0 -518
|
@@ -15,106 +15,40 @@ exports.writeUsageSnapshot = writeUsageSnapshot;
|
|
|
15
15
|
exports.foldUsageEvents = foldUsageEvents;
|
|
16
16
|
exports.readUsageStore = readUsageStore;
|
|
17
17
|
exports.sanitizeNote = sanitizeNote;
|
|
18
|
-
exports.
|
|
19
|
-
exports.
|
|
20
|
-
exports.recordFirings = recordFirings;
|
|
21
|
-
exports.noteTitleChange = noteTitleChange;
|
|
18
|
+
exports.readAssessedOutcomes = readAssessedOutcomes;
|
|
19
|
+
exports.recordAssessedOutcome = recordAssessedOutcome;
|
|
22
20
|
exports.pruneUsageStore = pruneUsageStore;
|
|
23
21
|
exports.compactUsageEvents = compactUsageEvents;
|
|
24
|
-
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* Two sides of one boundary, because only one of them is observable:
|
|
28
|
-
*
|
|
29
|
-
* OFFERED — written by the runtime with no agent involvement. The proxy composes
|
|
30
|
-
* the injected context block, so it knows which files it listed and,
|
|
31
|
-
* for the score-gated families, which entries in them were above
|
|
32
|
-
* threshold. That is what an offer means here.
|
|
33
|
-
* FIRED — attested by the agent at retrospective time. FRAIM cannot observe a
|
|
34
|
-
* file read, and a read is not a use, so influence has to be claimed
|
|
35
|
-
* rather than inferred.
|
|
36
|
-
*
|
|
37
|
-
* The record is metadata only (R16): entry identity, outcome, date, job, agent,
|
|
38
|
-
* model. Never prompts, diffs, or work content. Notes are reduced to one bounded
|
|
39
|
-
* line so a multi-line diff cannot be smuggled in through the attestation.
|
|
40
|
-
*
|
|
41
|
-
* Aggregate counters are kept in full; the detailed logs are bounded (R16a), on
|
|
42
|
-
* Mem0's precedent of retaining the last 20 accesses rather than an unbounded
|
|
43
|
-
* behavioural history.
|
|
44
|
-
*
|
|
45
|
-
* The store lives inside the manager layer's content root, which is the layer the
|
|
46
|
-
* existing GDPR guardrail blocks from a git backend without an explicit override
|
|
47
|
-
* (`src/ai-hub/server.ts`). That placement is what satisfies R18; there is no
|
|
48
|
-
* second guardrail here.
|
|
49
|
-
*
|
|
50
|
-
* Sleep consolidates the append-only journal into learning-usage.json and its
|
|
51
|
-
* recoverable .checkpoint copy before clearing processed events. Those snapshots
|
|
52
|
-
* preserve lifetime aggregates and source bookmarks after journal cleanup.
|
|
53
|
-
* Raw retrospectives preserve lessons and attestations, not every runtime offer.
|
|
54
|
-
*
|
|
55
|
-
* Four smaller corrections ride with it, each removing a place where the record
|
|
56
|
-
* gave a confident answer it could not prove:
|
|
57
|
-
*
|
|
58
|
-
* A1 `resolveEntryIdentity` is the one definition of what an entry is. Four
|
|
59
|
-
* call sites each built `id ?? titleUsageKey(...)` independently, so one
|
|
60
|
-
* lesson could be two records.
|
|
61
|
-
* C2 Model names are lowercased at write. `GPT-5` and `gpt-5` were two buckets
|
|
62
|
-
* splitting one denominator (measured 6857 against 2652).
|
|
63
|
-
* D3 Job-labelled offers are reserved against eviction by session traffic.
|
|
64
|
-
* 2067 of 2354 retained offer records were `session` or `__freeform__`, so
|
|
65
|
-
* "was this in context for that job" had almost no resolution left.
|
|
66
|
-
* E A scoped integrity invariant. The literal `fired <= offered` the issue
|
|
67
|
-
* asks for is unsound, because `runBackfill` legitimately writes firings
|
|
68
|
-
* for 299 retrospectives that predate offer recording. Scoped to attested
|
|
69
|
-
* firings dated at or after the store's first recorded offer, it holds: the
|
|
70
|
-
* CLI throws on a violation, and the injection path only flags, because a
|
|
71
|
-
* usage failure must never change what the agent is given (R2).
|
|
22
|
+
/** Assessed incidents and recoverable historical outcomes share the durable journal.
|
|
23
|
+
* Legacy offer/firing records decode only to preserve historical counts and recovery;
|
|
24
|
+
* their partial validity does not govern independent validated assessments.
|
|
72
25
|
*/
|
|
73
26
|
const fs_1 = require("fs");
|
|
74
27
|
const path_1 = require("path");
|
|
75
28
|
const pack_home_1 = require("../cli/utils/pack-home");
|
|
76
29
|
const artifact_retention_cleanup_1 = require("./artifact-retention-cleanup");
|
|
77
30
|
const project_fraim_paths_1 = require("../core/utils/project-fraim-paths");
|
|
31
|
+
const outcome_types_1 = require("../../registry/scripts/learning/outcome-types");
|
|
32
|
+
const proposal_queue_1 = require("../../registry/scripts/learning/proposal-queue");
|
|
78
33
|
const learning_usage_events_1 = require("./learning-usage-events");
|
|
79
34
|
Object.defineProperty(exports, "appendEvents", { enumerable: true, get: function () { return learning_usage_events_1.appendEvents; } });
|
|
80
35
|
Object.defineProperty(exports, "listEventLogs", { enumerable: true, get: function () { return learning_usage_events_1.listEventLogs; } });
|
|
81
36
|
Object.defineProperty(exports, "resolveEventDir", { enumerable: true, get: function () { return learning_usage_events_1.resolveEventDir; } });
|
|
82
37
|
Object.defineProperty(exports, "resolveMachineId", { enumerable: true, get: function () { return learning_usage_events_1.resolveMachineId; } });
|
|
83
38
|
// ── Vocabulary ───────────────────────────────────────────────────────────────
|
|
84
|
-
/**
|
|
85
|
-
* The five outcomes (spec: Outcome vocabulary). `ignored` and `not-offered` are
|
|
86
|
-
* the two that change what the manager should do: the first says fix the entry,
|
|
87
|
-
* the second says fix the delivery.
|
|
88
|
-
*/
|
|
39
|
+
/** Outcome vocabulary retained in historical records. */
|
|
89
40
|
exports.FIRING_OUTCOMES = ['prevented', 'applied', 'not-applicable', 'ignored', 'not-offered'];
|
|
90
|
-
/**
|
|
91
|
-
* R8: only these advance the decay clock. An entry that was offered, read, and
|
|
92
|
-
* correctly not relevant has not proved anything about its usefulness.
|
|
93
|
-
*/
|
|
41
|
+
/** Positive recorded outcomes; the historical name does not imply a guidance clock. */
|
|
94
42
|
exports.CLOCK_ADVANCING_OUTCOMES = ['prevented', 'applied'];
|
|
95
43
|
const STORE_VERSION = 1;
|
|
96
44
|
const STORE_FILE_NAME = 'learning-usage.json';
|
|
97
45
|
/** Default bounds. The firing bound follows Mem0's published 20-access retention. */
|
|
98
46
|
const DEFAULT_FIRING_LOG_LIMIT = 20;
|
|
99
|
-
/**
|
|
100
|
-
* The offer log is larger than the firing log because R7 has to answer "was this
|
|
101
|
-
* entry in context for that job" across a plausible recurrence window, which
|
|
102
|
-
* needs per-job offer detail rather than a counter.
|
|
103
|
-
*/
|
|
47
|
+
/** Bound retained historical offer detail without changing lifetime totals. */
|
|
104
48
|
const DEFAULT_OFFER_LOG_LIMIT = 50;
|
|
105
|
-
/**
|
|
106
|
-
* Issue #1752 D3. Of the offer log's slots, this many are held for records that
|
|
107
|
-
* carry a real job label, so `fraim_connect`'s per-session traffic cannot evict
|
|
108
|
-
* the only records that can answer R7.
|
|
109
|
-
*/
|
|
49
|
+
/** Preserve job-specific historical observations when trimming detail. */
|
|
110
50
|
const DEFAULT_JOB_OFFER_RESERVE = 25;
|
|
111
|
-
/**
|
|
112
|
-
const DEFAULT_RETIREMENT_OFFER_THRESHOLD = 10;
|
|
113
|
-
/**
|
|
114
|
-
* Job labels the runtime writes when no job is selected. An offer carrying one of
|
|
115
|
-
* these says the entry was in a session's context, not that it was in context for
|
|
116
|
-
* any particular job, so it is not what R7 resolves against.
|
|
117
|
-
*/
|
|
51
|
+
/** Historical labels without a specific job. */
|
|
118
52
|
exports.AMBIENT_JOB_LABELS = ['session', '__freeform__', 'unknown', ''];
|
|
119
53
|
/** R16: one line, bounded, no fences. Long enough for a sentence, short enough to be useless as a transcript. */
|
|
120
54
|
const MAX_NOTE_LENGTH = 240;
|
|
@@ -161,7 +95,6 @@ function resolveUsageLimits(workspaceRoot) {
|
|
|
161
95
|
firingLogLimit: positiveInt(block.firingLogLimit, DEFAULT_FIRING_LOG_LIMIT),
|
|
162
96
|
offerLogLimit: positiveInt(block.offerLogLimit, DEFAULT_OFFER_LOG_LIMIT),
|
|
163
97
|
jobOfferReserve: positiveInt(block.jobOfferReserve, DEFAULT_JOB_OFFER_RESERVE),
|
|
164
|
-
retirementOfferThreshold: positiveInt(block.retirementOfferThreshold, DEFAULT_RETIREMENT_OFFER_THRESHOLD),
|
|
165
98
|
retentionDays: typeof retentionDays === 'number' ? retentionDays : -1,
|
|
166
99
|
};
|
|
167
100
|
}
|
|
@@ -175,7 +108,6 @@ function defaultUsageLimits() {
|
|
|
175
108
|
firingLogLimit: DEFAULT_FIRING_LOG_LIMIT,
|
|
176
109
|
offerLogLimit: DEFAULT_OFFER_LOG_LIMIT,
|
|
177
110
|
jobOfferReserve: DEFAULT_JOB_OFFER_RESERVE,
|
|
178
|
-
retirementOfferThreshold: DEFAULT_RETIREMENT_OFFER_THRESHOLD,
|
|
179
111
|
retentionDays: -1,
|
|
180
112
|
};
|
|
181
113
|
}
|
|
@@ -196,39 +128,21 @@ function titleUsageKey(fileType, title) {
|
|
|
196
128
|
function ruleUsageKey(displayPath) {
|
|
197
129
|
return `R:${displayPath.replace(/\\/g, '/')}`;
|
|
198
130
|
}
|
|
199
|
-
/**
|
|
200
|
-
* Issue #1752 A1: the single definition of what an entry is.
|
|
201
|
-
*
|
|
202
|
-
* `enumerateFileEntries`, `buildEntryIndex`, `lookupUsage` and the `classify` CLI
|
|
203
|
-
* each built `id ?? titleUsageKey(...)` for themselves, which is four definitions
|
|
204
|
-
* that can drift. Defect 4 is what drift looks like: one lesson stamped in one
|
|
205
|
-
* tier and unstamped in another became two records, one collecting every offer
|
|
206
|
-
* and the other every firing, and `standing` then recommended promoting and
|
|
207
|
-
* demoting the same lesson in a single response.
|
|
208
|
-
*
|
|
209
|
-
* The id always wins. A title key is a fallback for a lesson no tier has stamped,
|
|
210
|
-
* never a second identity for one that has been.
|
|
211
|
-
*/
|
|
131
|
+
/** A stamped ID wins over a historical title identity. */
|
|
212
132
|
function resolveEntryIdentity(input) {
|
|
213
133
|
const id = typeof input.id === 'string' ? input.id.trim() : '';
|
|
214
134
|
if (id)
|
|
215
135
|
return { key: id, source: 'id' };
|
|
216
136
|
return { key: titleUsageKey(input.fileType ?? 'unknown', input.title ?? ''), source: 'title' };
|
|
217
137
|
}
|
|
218
|
-
/**
|
|
219
|
-
* Issue #1752 C2: one model, one bucket.
|
|
220
|
-
*
|
|
221
|
-
* Measured in the live store: `GPT-5` 6857 against `gpt-5` 2652, and `gpt-6` 1764
|
|
222
|
-
* against `GPT-6` 140. Every per-model calculation, including retirement
|
|
223
|
-
* candidacy, was splitting its own denominator.
|
|
224
|
-
*/
|
|
138
|
+
/** Historical model bucket names compare without case or surrounding whitespace. */
|
|
225
139
|
function normalizeModelName(model) {
|
|
226
140
|
const normalized = String(model ?? '').trim().toLowerCase();
|
|
227
141
|
return normalized || null;
|
|
228
142
|
}
|
|
229
143
|
// ── Read and write ───────────────────────────────────────────────────────────
|
|
230
144
|
function emptyStore() {
|
|
231
|
-
return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {}, foldedThroughBytes: {}
|
|
145
|
+
return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {}, foldedThroughBytes: {} };
|
|
232
146
|
}
|
|
233
147
|
function withEvidence(store, evidence) {
|
|
234
148
|
Object.defineProperty(store, 'evidence', { value: evidence, enumerable: false, configurable: true });
|
|
@@ -237,22 +151,7 @@ function withEvidence(store, evidence) {
|
|
|
237
151
|
function unavailableStore(observation) {
|
|
238
152
|
return withEvidence(emptyStore(), { status: 'unavailable', excludedKeys: [], observations: [observation] });
|
|
239
153
|
}
|
|
240
|
-
/**
|
|
241
|
-
* Is this parsed value a record every consumer can read without crashing?
|
|
242
|
-
*
|
|
243
|
-
* The shallow `record.offered && record.fired` this replaced was not enough. A
|
|
244
|
-
* record whose sub-objects exist but are empty passed it and then threw in
|
|
245
|
-
* `buildUsageReport` (`record.offered.log is not iterable`),
|
|
246
|
-
* `findRetirementCandidates` and `deriveStanding`, so usage reports and
|
|
247
|
-
* candidate analysis failed outright instead of degrading. That state is
|
|
248
|
-
* reachable: the store is a JSON file inside the manager home, which is commonly
|
|
249
|
-
* a synced folder, so an interrupted write or a partial sync produces valid JSON
|
|
250
|
-
* with a hollow record.
|
|
251
|
-
*
|
|
252
|
-
* Checks only what a consumer actually dereferences, so a record carrying an
|
|
253
|
-
* unknown extra field is still accepted and a future field addition does not
|
|
254
|
-
* silently start discarding history.
|
|
255
|
-
*/
|
|
154
|
+
/** Validate required historical fields while ignoring optional metadata. */
|
|
256
155
|
function isUsableRecord(value) {
|
|
257
156
|
if (!value || typeof value !== 'object')
|
|
258
157
|
return false;
|
|
@@ -277,32 +176,6 @@ function isUsableRecord(value) {
|
|
|
277
176
|
&& Array.isArray(fired.seenSources) && fired.seenSources.every(s => typeof s === 'string')
|
|
278
177
|
&& Array.isArray(record.titles) && record.titles.every(t => typeof t === 'string');
|
|
279
178
|
}
|
|
280
|
-
/**
|
|
281
|
-
* The earliest offer date a pre-#1752 snapshot can evidence.
|
|
282
|
-
*
|
|
283
|
-
* Found by validating against a copy of the real store: without this, a store
|
|
284
|
-
* written before the event log existed has no `offersEraStart`, the scoped
|
|
285
|
-
* invariant has no era to scope to, and it reports zero violations on a corpus
|
|
286
|
-
* that has 62 entries holding more attested firings than recorded offers. Silent
|
|
287
|
-
* and wrong, which is the failure class this whole issue is about.
|
|
288
|
-
*
|
|
289
|
-
* Derived from the bounded offer logs, so it is at or later than the true first
|
|
290
|
-
* offer. Later means fewer firings counted as in-era, which under-reports rather
|
|
291
|
-
* than inventing a violation, and that is the right direction for a check whose
|
|
292
|
-
* CLI form refuses a write.
|
|
293
|
-
*/
|
|
294
|
-
function deriveOffersEraStart(entries) {
|
|
295
|
-
let earliest = null;
|
|
296
|
-
for (const record of Object.values(entries)) {
|
|
297
|
-
for (const offer of record.offered.log) {
|
|
298
|
-
if (!offer?.date)
|
|
299
|
-
continue;
|
|
300
|
-
if (earliest === null || offer.date < earliest)
|
|
301
|
-
earliest = offer.date;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
return earliest;
|
|
305
|
-
}
|
|
306
179
|
function readCheckpoints(value) {
|
|
307
180
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
308
181
|
return {};
|
|
@@ -374,10 +247,17 @@ function parseUsageSnapshot(storePath) {
|
|
|
374
247
|
updatedAt: parsed.updatedAt ?? emptyStore().updatedAt,
|
|
375
248
|
entries,
|
|
376
249
|
foldedThroughBytes: readCheckpoints(parsed.foldedThroughBytes),
|
|
377
|
-
|
|
378
|
-
? parsed.offersEraStart
|
|
379
|
-
: deriveOffersEraStart(entries),
|
|
250
|
+
...(parsed.outcomeAssessments !== undefined ? { outcomeAssessments: parsed.outcomeAssessments } : {}),
|
|
380
251
|
};
|
|
252
|
+
if (store.outcomeAssessments !== undefined) {
|
|
253
|
+
if (!store.outcomeAssessments || typeof store.outcomeAssessments !== 'object' || Array.isArray(store.outcomeAssessments))
|
|
254
|
+
return unavailableStore('malformed-assessments');
|
|
255
|
+
for (const [digest, assessment] of Object.entries(store.outcomeAssessments)) {
|
|
256
|
+
(0, outcome_types_1.validateOutcomeAssessmentRecord)(assessment);
|
|
257
|
+
if (digest !== assessment.digest)
|
|
258
|
+
return unavailableStore('malformed-assessments');
|
|
259
|
+
}
|
|
260
|
+
}
|
|
381
261
|
return excludedKeys.length ? withEvidence(store, {
|
|
382
262
|
status: 'partial', excludedKeys, observations: ['malformed-records'],
|
|
383
263
|
}) : store;
|
|
@@ -437,7 +317,7 @@ function applyTail(base, tail, limits, now, restarted) {
|
|
|
437
317
|
updatedAt: base.updatedAt,
|
|
438
318
|
entries: base.entries,
|
|
439
319
|
foldedThroughBytes: { ...(base.foldedThroughBytes ?? {}) },
|
|
440
|
-
|
|
320
|
+
...(base.outcomeAssessments ? { outcomeAssessments: { ...base.outcomeAssessments } } : {}),
|
|
441
321
|
};
|
|
442
322
|
if (base.evidence)
|
|
443
323
|
withEvidence(store, base.evidence);
|
|
@@ -463,9 +343,6 @@ function applyTail(base, tail, limits, now, restarted) {
|
|
|
463
343
|
if (record)
|
|
464
344
|
trimLogs(record, limits, now);
|
|
465
345
|
}
|
|
466
|
-
// E: the invariant is evaluated on the read path, where it is always current
|
|
467
|
-
// and costs the write path nothing.
|
|
468
|
-
flagIntegrity(store);
|
|
469
346
|
const incompleteJournal = tail.skippedLines > 0
|
|
470
347
|
|| tail.reads.some(read => read.through < read.baseOffset + read.size - read.headerBytes);
|
|
471
348
|
if (incompleteJournal || unsupportedEvents) {
|
|
@@ -518,7 +395,6 @@ function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
|
|
|
518
395
|
withEvidence(store, {
|
|
519
396
|
status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [], observations,
|
|
520
397
|
});
|
|
521
|
-
flagIntegrity(store);
|
|
522
398
|
return store;
|
|
523
399
|
};
|
|
524
400
|
if (logs.length === 0 || snapshot.evidence?.observations.includes('unsupported-schema'))
|
|
@@ -550,7 +426,6 @@ function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
|
|
|
550
426
|
// Recovery is usable in memory. Persisting a primary replacement requires the
|
|
551
427
|
// explicit maintenance interval; an ordinary reader never establishes it.
|
|
552
428
|
if (shouldRefresh && !folded.store.evidence) {
|
|
553
|
-
folded.store.updatedAt = (options.now ?? new Date()).toISOString();
|
|
554
429
|
writeUsageSnapshot(folded.store, storePath);
|
|
555
430
|
}
|
|
556
431
|
return folded.store;
|
|
@@ -636,20 +511,7 @@ const byDate = (a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0);
|
|
|
636
511
|
function isAmbientJob(job) {
|
|
637
512
|
return exports.AMBIENT_JOB_LABELS.includes(job ?? '');
|
|
638
513
|
}
|
|
639
|
-
/**
|
|
640
|
-
* Issue #1752 D3: bound the offer log without letting session traffic evict the
|
|
641
|
-
* only records that can answer R7.
|
|
642
|
-
*
|
|
643
|
-
* Measured cause: `fraim_connect` records one offer per entry per session under
|
|
644
|
-
* the label `session`, and a plain newest-50 rule then evicted every
|
|
645
|
-
* job-labelled record within about a day. 2067 of the 2354 retained offer records
|
|
646
|
-
* in the live store carried `session` or `__freeform__`, and only three real job
|
|
647
|
-
* names appeared at all, which is why `classify --job <real-job>` could almost
|
|
648
|
-
* never find a match.
|
|
649
|
-
*
|
|
650
|
-
* Job-labelled records get the first `jobOfferReserve` slots; the rest of the
|
|
651
|
-
* budget goes to the newest of everything else.
|
|
652
|
-
*/
|
|
514
|
+
/** Preserve job-specific historical detail within the configured log bound. */
|
|
653
515
|
function selectRetainedOffers(log, limits) {
|
|
654
516
|
if (log.length <= limits.offerLogLimit)
|
|
655
517
|
return log;
|
|
@@ -673,50 +535,21 @@ function trimLogs(record, limits, now) {
|
|
|
673
535
|
.sort(byDate)
|
|
674
536
|
.slice(-limits.firingLogLimit);
|
|
675
537
|
}
|
|
676
|
-
function offerEvent(offer) {
|
|
677
|
-
return {
|
|
678
|
-
kind: 'offer',
|
|
679
|
-
key: offer.key,
|
|
680
|
-
family: offer.family,
|
|
681
|
-
title: offer.title,
|
|
682
|
-
fileType: offer.fileType,
|
|
683
|
-
level: offer.level,
|
|
684
|
-
aboveThreshold: Boolean(offer.aboveThreshold),
|
|
685
|
-
job: offer.job,
|
|
686
|
-
date: offer.date,
|
|
687
|
-
agent: offer.agent ?? null,
|
|
688
|
-
model: normalizeModelName(offer.model),
|
|
689
|
-
};
|
|
690
|
-
}
|
|
691
|
-
function firingEvent(firing) {
|
|
692
|
-
return {
|
|
693
|
-
kind: 'firing',
|
|
694
|
-
key: firing.key,
|
|
695
|
-
family: firing.family,
|
|
696
|
-
title: firing.title,
|
|
697
|
-
fileType: firing.fileType,
|
|
698
|
-
level: firing.level,
|
|
699
|
-
outcome: firing.outcome,
|
|
700
|
-
note: sanitizeNote(firing.note),
|
|
701
|
-
job: firing.job,
|
|
702
|
-
date: firing.date,
|
|
703
|
-
agent: firing.agent ?? null,
|
|
704
|
-
model: normalizeModelName(firing.model),
|
|
705
|
-
source: firing.source ?? 'attested',
|
|
706
|
-
sourceRef: firing.sourceRef,
|
|
707
|
-
};
|
|
708
|
-
}
|
|
709
538
|
/**
|
|
710
539
|
* Apply one logged event to the record set. Returns the key it touched, or null
|
|
711
540
|
* when the event is unusable or already counted.
|
|
712
541
|
*
|
|
713
|
-
*
|
|
714
|
-
*
|
|
715
|
-
* identity of an attestation, which is the same fact the migration's exact firing
|
|
716
|
-
* recovery rests on.
|
|
542
|
+
* Historical duplicate suppression remains in the reader: retained `(key, sourceRef)`
|
|
543
|
+
* identities prevent old journal replays from increasing preserved totals.
|
|
717
544
|
*/
|
|
718
545
|
function applyEvent(store, event, limits, now) {
|
|
719
546
|
const kind = event.kind;
|
|
547
|
+
if (kind === 'assessment') {
|
|
548
|
+
const assessment = event.assessment;
|
|
549
|
+
(0, outcome_types_1.validateOutcomeAssessmentRecord)(assessment);
|
|
550
|
+
(store.outcomeAssessments ??= {})[assessment.digest] = assessment;
|
|
551
|
+
return null;
|
|
552
|
+
}
|
|
720
553
|
const key = typeof event.key === 'string' ? event.key : '';
|
|
721
554
|
const date = typeof event.date === 'string' ? event.date : '';
|
|
722
555
|
if (!key || !isValidDate(date))
|
|
@@ -737,6 +570,15 @@ function applyEvent(store, event, limits, now) {
|
|
|
737
570
|
return null;
|
|
738
571
|
}
|
|
739
572
|
function isSupportedUsageEvent(event) {
|
|
573
|
+
if (event.kind === 'assessment') {
|
|
574
|
+
try {
|
|
575
|
+
(0, outcome_types_1.validateOutcomeAssessmentRecord)(event.assessment);
|
|
576
|
+
return true;
|
|
577
|
+
}
|
|
578
|
+
catch {
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
740
582
|
if (typeof event.key !== 'string' || !event.key || typeof event.date !== 'string' || !isValidDate(event.date))
|
|
741
583
|
return false;
|
|
742
584
|
if (event.family !== 'learning' && event.family !== 'rule')
|
|
@@ -768,7 +610,6 @@ function applyOfferEvent(store, event, common) {
|
|
|
768
610
|
});
|
|
769
611
|
// The era only ever moves earlier, so the scoped invariant's boundary is
|
|
770
612
|
// monotonic and a late-syncing log cannot shrink the window it applies to.
|
|
771
|
-
store.offersEraStart = minDate(store.offersEraStart, common.date);
|
|
772
613
|
return common.key;
|
|
773
614
|
}
|
|
774
615
|
function applyFiringEvent(store, event, common) {
|
|
@@ -776,14 +617,13 @@ function applyFiringEvent(store, event, common) {
|
|
|
776
617
|
if (!exports.FIRING_OUTCOMES.includes(outcome))
|
|
777
618
|
return null;
|
|
778
619
|
const record = ensureRecord(store, common.key, common.family, common.fileType, common.level);
|
|
779
|
-
//
|
|
780
|
-
//
|
|
781
|
-
// of an attestation, which is the same fact the migration's exact firing
|
|
782
|
-
// recovery rests on.
|
|
620
|
+
// Retained source identities survive detail pruning and suppress duplicate
|
|
621
|
+
// historical journal rows during recovery.
|
|
783
622
|
const sourceRef = typeof event.sourceRef === 'string' ? event.sourceRef : '';
|
|
784
623
|
if (sourceRef && record.fired.seenSources.includes(sourceRef))
|
|
785
624
|
return null;
|
|
786
625
|
rememberTitle(record, typeof event.title === 'string' ? event.title : undefined);
|
|
626
|
+
store.updatedAt = maxDate(store.updatedAt, common.date);
|
|
787
627
|
record.fired.total += 1;
|
|
788
628
|
record.fired.byOutcome[outcome] = (record.fired.byOutcome[outcome] ?? 0) + 1;
|
|
789
629
|
record.fired.lastAttestedAt = maxDate(record.fired.lastAttestedAt, common.date);
|
|
@@ -806,196 +646,55 @@ function applyFiringEvent(store, event, common) {
|
|
|
806
646
|
record.fired.seenSources.push(sourceRef);
|
|
807
647
|
return common.key;
|
|
808
648
|
}
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
* 'log' Counts only the firings the bounded log still holds. It cannot
|
|
822
|
-
* over-count, so it can never refuse a correct write. This is what
|
|
823
|
-
* the CLI throws on.
|
|
824
|
-
* 'aggregate' Starts from `fired.total` and subtracts the firings the log
|
|
825
|
-
* shows are exempt. It sees an entry whose detail has aged out,
|
|
826
|
-
* which is the shape all 58 violating entries in the live store
|
|
827
|
-
* have, and it can over-count when the log no longer shows the
|
|
828
|
-
* exemptions. That is acceptable for a report and not for a
|
|
829
|
-
* refusal, so this is what the read path flags with and `verify`
|
|
830
|
-
* lists.
|
|
831
|
-
*/
|
|
832
|
-
function attestedFiringsInOffersEra(record, offersEraStart, basis) {
|
|
833
|
-
if (!offersEraStart)
|
|
834
|
-
return 0;
|
|
835
|
-
const inLog = record.fired.log.filter((r) => r.source === 'attested' && r.date >= offersEraStart).length;
|
|
836
|
-
if (basis === 'log')
|
|
837
|
-
return inLog;
|
|
838
|
-
const exemptInLog = record.fired.log.filter((r) => r.source === 'backfill-legacy' || r.date < offersEraStart).length;
|
|
839
|
-
return Math.max(inLog, record.fired.total - exemptInLog);
|
|
840
|
-
}
|
|
841
|
-
function integrityViolation(record, offersEraStart, basis = 'aggregate') {
|
|
842
|
-
// An entry with no offer record at all is not an accounting overflow; it is the
|
|
843
|
-
// question defect 2 is about, and it has several innocent causes: the offers
|
|
844
|
-
// were recorded under the entry's other key before #1752 A2 merged them, the
|
|
845
|
-
// entry is a rule or a demoted lesson the offer path does not enumerate, or the
|
|
846
|
-
// offers are sitting in another machine's event log that has not synced yet.
|
|
847
|
-
// Throwing on any of those would block a correct attestation on a sync
|
|
848
|
-
// artefact. The overflow this invariant exists to catch is the one the issue
|
|
849
|
-
// measured: entries showing `offered: 1, fired: 10`, which all have an offer.
|
|
850
|
-
if (record.offered.total === 0)
|
|
851
|
-
return null;
|
|
852
|
-
const inEra = attestedFiringsInOffersEra(record, offersEraStart, basis);
|
|
853
|
-
if (inEra <= record.offered.total)
|
|
854
|
-
return null;
|
|
855
|
-
return {
|
|
856
|
-
flag: 'fired-exceeds-offered',
|
|
857
|
-
count: inEra - record.offered.total,
|
|
858
|
-
detail: `${inEra} attested firings dated on or after ${offersEraStart} against ${record.offered.total} recorded offers. An entry cannot fire without being offered, so one of the two counters is wrong for this entry.`,
|
|
859
|
-
};
|
|
860
|
-
}
|
|
861
|
-
function flagIntegrity(store) {
|
|
862
|
-
for (const record of Object.values(store.entries)) {
|
|
863
|
-
const violation = integrityViolation(record, store.offersEraStart);
|
|
864
|
-
if (violation)
|
|
865
|
-
record.integrity = violation;
|
|
866
|
-
else if (record.integrity)
|
|
867
|
-
delete record.integrity;
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
/**
|
|
871
|
-
* Materialize the snapshot when there is none.
|
|
872
|
-
*
|
|
873
|
-
* Before the first consolidation, readers publish snapshots, but a workspace that
|
|
874
|
-
* has only ever been written to would have no snapshot at all, so every reader
|
|
875
|
-
* would pay a cold fold and anything inspecting the store as a file would find
|
|
876
|
-
* nothing there. Bounded: it only runs when the file is absent, and at that point
|
|
877
|
-
* the fold is over a log that has just been created. A failure is swallowed, like
|
|
878
|
-
* every other snapshot write.
|
|
879
|
-
*/
|
|
880
|
-
function ensureSnapshotExists(storePath, eventDir, limits, now) {
|
|
881
|
-
try {
|
|
882
|
-
if ((0, fs_1.existsSync)(storePath))
|
|
883
|
-
return;
|
|
884
|
-
readUsageStore(storePath, { limits, now, eventDir, refreshSnapshot: 'always' });
|
|
885
|
-
}
|
|
886
|
-
catch {
|
|
887
|
-
// Initial cache publication can retry; consolidated history has a separate backup.
|
|
888
|
-
}
|
|
889
|
-
}
|
|
890
|
-
/**
|
|
891
|
-
* R1 and R2: write one offer record per listed entry, without any agent
|
|
892
|
-
* involvement and without depending on the agent reading the file.
|
|
893
|
-
*
|
|
894
|
-
* Issue #1752 B: this appends and does not read. The read-modify-write it
|
|
895
|
-
* replaces lost 1589 of 1600 writes under eight concurrent processes on a plain
|
|
896
|
-
* local disk (spike S1b), which is the dominant loss path behind both the
|
|
897
|
-
* impossible counters and the four conflict copies. Nothing here can throw on an
|
|
898
|
-
* integrity violation either: R2 requires that a usage failure never changes what
|
|
899
|
-
* the agent is given, so the violation is flagged by the read path instead.
|
|
900
|
-
*/
|
|
901
|
-
function recordOffers(offers, options) {
|
|
902
|
-
const storePath = options.storePath ?? resolveUsageStorePath();
|
|
903
|
-
const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
|
|
904
|
-
const usable = offers.filter((offer) => offer.key && isValidDate(offer.date));
|
|
905
|
-
if (usable.length === 0)
|
|
906
|
-
return { recorded: 0, entries: 0 };
|
|
907
|
-
(0, learning_usage_events_1.appendEvents)(eventDir, usable.map(offerEvent), options.machineId);
|
|
908
|
-
ensureSnapshotExists(storePath, eventDir, options.limits, options.now ?? new Date());
|
|
909
|
-
return { recorded: usable.length, entries: new Set(usable.map((o) => o.key)).size };
|
|
910
|
-
}
|
|
911
|
-
/**
|
|
912
|
-
* R4 and R6: write the attested firings. Idempotent per (entry, attestation
|
|
913
|
-
* source) so a re-run, or a backfill that overlaps a live attestation, cannot
|
|
914
|
-
* inflate a count.
|
|
915
|
-
*
|
|
916
|
-
* Unlike the offer path this reads first, because it has two questions only the
|
|
917
|
-
* current record can answer: whether this attestation has already contributed,
|
|
918
|
-
* and whether the scoped invariant is about to be violated. It still writes by
|
|
919
|
-
* appending, and the fold applies the same `(key, sourceRef)` idempotency, so a
|
|
920
|
-
* racing duplicate append is folded once rather than counted twice.
|
|
921
|
-
*
|
|
922
|
-
* The invariant throws here. This is a CLI path with an operator watching, and a
|
|
923
|
-
* violation means a real accounting bug rather than a cosmetic one.
|
|
924
|
-
*/
|
|
925
|
-
function recordFirings(firings, options) {
|
|
926
|
-
const storePath = options.storePath ?? resolveUsageStorePath();
|
|
927
|
-
const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
|
|
928
|
-
const now = options.now ?? new Date();
|
|
929
|
-
const store = readUsageStore(storePath, { limits: options.limits, now, eventDir });
|
|
930
|
-
const events = [];
|
|
931
|
-
let alreadyRecorded = 0;
|
|
932
|
-
for (const firing of firings) {
|
|
933
|
-
if (!exports.FIRING_OUTCOMES.includes(firing.outcome)) {
|
|
934
|
-
throw new Error(`Unknown firing outcome "${firing.outcome}". Use one of: ${exports.FIRING_OUTCOMES.join(', ')}.`);
|
|
935
|
-
}
|
|
936
|
-
if (!firing.key || !isValidDate(firing.date))
|
|
649
|
+
/** Resolve correction chains independent of machine-log order. Conflicting siblings remain unresolved. */
|
|
650
|
+
function currentAssessedOutcomes(store, scope = {}) {
|
|
651
|
+
const legacyOnlyDamage = store.evidence?.status === 'partial'
|
|
652
|
+
&& store.evidence.observations.length === 1 && store.evidence.observations[0] === 'malformed-records';
|
|
653
|
+
// Invalid historical totals are excluded independently. They neither erase
|
|
654
|
+
// validated incident judgments nor authorize replacing the damaged snapshot.
|
|
655
|
+
if (store.evidence && store.evidence.status !== 'reliable' && !legacyOnlyDamage)
|
|
656
|
+
throw new Error('assessment-history-unavailable');
|
|
657
|
+
const groups = new Map();
|
|
658
|
+
for (const record of Object.values(store.outcomeAssessments ?? {})) {
|
|
659
|
+
(0, outcome_types_1.validateOutcomeAssessmentRecord)(record);
|
|
660
|
+
if (!(0, outcome_types_1.matchesOutcomeScope)(record, scope))
|
|
937
661
|
continue;
|
|
938
|
-
const
|
|
939
|
-
|
|
940
|
-
alreadyRecorded += 1;
|
|
941
|
-
continue;
|
|
942
|
-
}
|
|
943
|
-
events.push(firingEvent(firing));
|
|
662
|
+
const key = (0, outcome_types_1.assessmentKey)(record);
|
|
663
|
+
groups.set(key, [...(groups.get(key) ?? []), record]);
|
|
944
664
|
}
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
665
|
+
return [...groups.values()].map(records => {
|
|
666
|
+
const revisions = [...records].sort((a, b) => a.revision - b.revision);
|
|
667
|
+
let previous;
|
|
668
|
+
for (const record of revisions) {
|
|
669
|
+
if (record.revision !== (previous?.revision ?? 0) + 1 || record.expectedPriorDigest !== (previous?.digest ?? null)) {
|
|
670
|
+
throw new Error('assessment-history-conflict');
|
|
671
|
+
}
|
|
672
|
+
previous = record;
|
|
948
673
|
}
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
// from the CLI once per retrospective, so the fold is cheap, and it makes the
|
|
952
|
-
// store file current the moment an operator's write returns rather than at
|
|
953
|
-
// whatever point a later reader happens to refresh it. The offer path stays
|
|
954
|
-
// append-only, because that is the one that runs on every `fraim_connect`.
|
|
955
|
-
readUsageStore(storePath, { limits: options.limits, now, eventDir, refreshSnapshot: 'always' });
|
|
956
|
-
}
|
|
957
|
-
return { recorded: events.length, alreadyRecorded };
|
|
674
|
+
return previous;
|
|
675
|
+
});
|
|
958
676
|
}
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
* impossible state the issue reports on 58 of its 223 entries.
|
|
962
|
-
*/
|
|
963
|
-
function assertScopedIntegrity(store, events, limits, now) {
|
|
964
|
-
const projected = {
|
|
965
|
-
version: store.version,
|
|
966
|
-
updatedAt: store.updatedAt,
|
|
967
|
-
entries: JSON.parse(JSON.stringify(store.entries)),
|
|
968
|
-
foldedThroughBytes: store.foldedThroughBytes,
|
|
969
|
-
offersEraStart: store.offersEraStart,
|
|
970
|
-
};
|
|
971
|
-
for (const event of events)
|
|
972
|
-
applyEvent(projected, event, limits, now);
|
|
973
|
-
for (const event of events) {
|
|
974
|
-
const record = projected.entries[event.key];
|
|
975
|
-
if (!record)
|
|
976
|
-
continue;
|
|
977
|
-
// 'log' basis: a refusal must never rest on a number that can over-count.
|
|
978
|
-
const violation = integrityViolation(record, projected.offersEraStart, 'log');
|
|
979
|
-
if (violation) {
|
|
980
|
-
throw new Error(`Refusing to record firings for ${event.key}: ${violation.detail} Run \`learning-usage verify\` for the full integrity picture before forcing this through.`);
|
|
981
|
-
}
|
|
982
|
-
}
|
|
677
|
+
function readAssessedOutcomes(storePath = resolveUsageStorePath(), scope = {}) {
|
|
678
|
+
return currentAssessedOutcomes(readUsageStore(storePath, { refreshSnapshot: 'never' }), scope);
|
|
983
679
|
}
|
|
984
|
-
/**
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
* index able to follow a rename.
|
|
988
|
-
*/
|
|
989
|
-
function noteTitleChange(key, newTitle, options) {
|
|
680
|
+
/** Sleep commits one validated decision through the existing recoverable usage journal. */
|
|
681
|
+
function recordAssessedOutcome(record, options = {}) {
|
|
682
|
+
(0, outcome_types_1.validateOutcomeAssessmentRecord)(record);
|
|
990
683
|
const storePath = options.storePath ?? resolveUsageStorePath();
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
684
|
+
return (0, proposal_queue_1.withFileLock)(storePath, () => {
|
|
685
|
+
const store = readUsageStore(storePath, { refreshSnapshot: 'never' });
|
|
686
|
+
const scope = { key: (0, outcome_types_1.assessmentKey)(record) };
|
|
687
|
+
const current = currentAssessedOutcomes(store, scope)[0];
|
|
688
|
+
if (store.outcomeAssessments?.[record.digest])
|
|
689
|
+
return store.outcomeAssessments[record.digest];
|
|
690
|
+
if (record.expectedPriorDigest !== (current?.digest ?? null) || record.revision !== (current?.revision ?? 0) + 1)
|
|
691
|
+
throw new Error('assessment-stale-revision');
|
|
692
|
+
(0, learning_usage_events_1.appendEvents)((0, learning_usage_events_1.resolveEventDir)(storePath), [{ kind: 'assessment', assessment: record }]);
|
|
693
|
+
const persisted = readAssessedOutcomes(storePath, scope).find(item => item.digest === record.digest);
|
|
694
|
+
if (!persisted)
|
|
695
|
+
throw new Error('assessment-write-unverified');
|
|
696
|
+
return persisted;
|
|
697
|
+
});
|
|
999
698
|
}
|
|
1000
699
|
function pruneUsageStore(limits, options = {}) {
|
|
1001
700
|
const storePath = options.storePath ?? resolveUsageStorePath();
|
|
@@ -1046,7 +745,6 @@ function compactUsageEvents(options) {
|
|
|
1046
745
|
for (const entry of Object.values(store.entries))
|
|
1047
746
|
trimLogs(entry, options.limits, now);
|
|
1048
747
|
store.consolidated = true;
|
|
1049
|
-
store.updatedAt = now.toISOString();
|
|
1050
748
|
// Publish the recoverable checkpoint first, then the normal snapshot. No
|
|
1051
749
|
// journal is cleared unless both writes succeed. The backup is never written
|
|
1052
750
|
// by ordinary readers, so a torn cache refresh cannot destroy both copies.
|
|
@@ -1069,6 +767,8 @@ function compactUsageEvents(options) {
|
|
|
1069
767
|
throw new Error('Cannot consolidate usage events: aggregate snapshot could not be saved.');
|
|
1070
768
|
}
|
|
1071
769
|
const compaction = (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
|
|
770
|
+
if (event.kind === 'assessment')
|
|
771
|
+
return !isSupportedUsageEvent(event);
|
|
1072
772
|
if (event.kind !== 'offer' && event.kind !== 'firing')
|
|
1073
773
|
return true;
|
|
1074
774
|
const date = typeof event.date === 'string' ? event.date : '';
|