fraim 2.0.340 → 2.0.341
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/registry/scripts/learning/completion-state.js +10 -2
- package/dist/src/core/handoff-contracts.js +122 -21
- package/dist/src/core/proposal-set-contract.js +214 -0
- package/dist/src/core/run-recovery-state.js +9 -0
- package/dist/src/local-mcp-server/learning-context-builder.js +128 -1
- package/dist/src/local-mcp-server/learning-usage-analysis.js +35 -19
- package/dist/src/local-mcp-server/learning-usage-command.js +8 -4
- package/dist/src/local-mcp-server/learning-usage-eligibility.js +59 -0
- package/dist/src/local-mcp-server/learning-usage-projection.js +5 -1
- package/dist/src/local-mcp-server/learning-usage-store.js +163 -38
- package/dist/src/local-mcp-server/learning-usage-verify.js +3 -1
- package/dist/src/local-mcp-server/stdio-server.js +266 -112
- package/dist/src/local-mcp-server/usage-collector.js +5 -29
- package/dist/src/mcp/tool-schemas.js +2 -2
- package/dist/src/routes/analytics.js +10 -3
- package/dist/src/services/mcp-service.js +71 -11
- package/dist/src/services/usage-analytics-service.js +6 -3
- package/package.json +1 -1
|
@@ -27,6 +27,7 @@ const fs_1 = require("fs");
|
|
|
27
27
|
const path_1 = require("path");
|
|
28
28
|
const learning_usage_store_1 = require("./learning-usage-store");
|
|
29
29
|
const learning_usage_projection_1 = require("./learning-usage-projection");
|
|
30
|
+
const learning_usage_eligibility_1 = require("./learning-usage-eligibility");
|
|
30
31
|
function daysBetween(from, to) {
|
|
31
32
|
const a = new Date(from).getTime();
|
|
32
33
|
const b = new Date(to).getTime();
|
|
@@ -39,6 +40,7 @@ function firedCount(record) {
|
|
|
39
40
|
}
|
|
40
41
|
function classifyRecurrence(store, options) {
|
|
41
42
|
const record = store.entries[options.key];
|
|
43
|
+
const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
|
|
42
44
|
const base = {
|
|
43
45
|
key: options.key,
|
|
44
46
|
title: record?.titles[record.titles.length - 1] ?? null,
|
|
@@ -51,6 +53,14 @@ function classifyRecurrence(store, options) {
|
|
|
51
53
|
firedTotal: record ? firedCount(record) : 0,
|
|
52
54
|
},
|
|
53
55
|
};
|
|
56
|
+
if (record && excluded.has(record.key)) {
|
|
57
|
+
return {
|
|
58
|
+
...base, classification: 'undetermined',
|
|
59
|
+
diagnosis: 'The available usage evidence cannot establish this recurrence.',
|
|
60
|
+
recommendation: 'Review independent source evidence without attributing this recurrence to usage.',
|
|
61
|
+
recurrenceBump: false, targetOfFix: 'none',
|
|
62
|
+
};
|
|
63
|
+
}
|
|
54
64
|
if (!record) {
|
|
55
65
|
return {
|
|
56
66
|
...base,
|
|
@@ -94,7 +104,7 @@ function classifyRecurrence(store, options) {
|
|
|
94
104
|
// per entry per session and a plain newest-N bound evicted the job-labelled
|
|
95
105
|
// ones within about a day. Asking about a job the store has never retained an
|
|
96
106
|
// offer for is a gap in the record, not a delivery failure.
|
|
97
|
-
const jobCoverage = Object.values(store.entries).some((entry) => entry.offered.log.some((offer) => offer.job === options.job && daysBetween(offer.date, options.date) <= options.windowDays));
|
|
107
|
+
const jobCoverage = Object.values(store.entries).some((entry) => !excluded.has(entry.key) && entry.offered.log.some((offer) => offer.job === options.job && daysBetween(offer.date, options.date) <= options.windowDays));
|
|
98
108
|
if (matchingOffers.length > 0) {
|
|
99
109
|
return {
|
|
100
110
|
...base,
|
|
@@ -231,7 +241,7 @@ function firingsUnderModel(record, model) {
|
|
|
231
241
|
* unrecorded.
|
|
232
242
|
*/
|
|
233
243
|
function modelAttributionUnknown(record) {
|
|
234
|
-
return
|
|
244
|
+
return (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record);
|
|
235
245
|
}
|
|
236
246
|
function earlierModelFirings(record, currentModel) {
|
|
237
247
|
return Object.entries(record.fired.byModel)
|
|
@@ -261,24 +271,22 @@ function findRetirementCandidates(store, options) {
|
|
|
261
271
|
}
|
|
262
272
|
const candidates = [];
|
|
263
273
|
const notCandidates = [];
|
|
274
|
+
const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
|
|
264
275
|
for (const record of Object.values(store.entries)) {
|
|
265
276
|
if (record.family !== 'learning')
|
|
266
277
|
continue;
|
|
267
278
|
const title = latestTitle(record);
|
|
279
|
+
if (excluded.has(record.key) || modelAttributionUnknown(record)) {
|
|
280
|
+
notCandidates.push({ key: record.key, title,
|
|
281
|
+
reason: excluded.get(record.key) ?? 'Model attribution is incomplete; unknown use is not zero use.' });
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
268
284
|
const offered = offersUnderModel(record, currentModel);
|
|
269
285
|
const fired = firingsUnderModel(record, currentModel);
|
|
270
286
|
if (fired > 0) {
|
|
271
287
|
notCandidates.push({ key: record.key, title, reason: `It fired ${fired} time${fired === 1 ? '' : 's'} under the current model, so it is still earning its place.` });
|
|
272
288
|
continue;
|
|
273
289
|
}
|
|
274
|
-
if (modelAttributionUnknown(record)) {
|
|
275
|
-
notCandidates.push({
|
|
276
|
-
key: record.key,
|
|
277
|
-
title,
|
|
278
|
-
reason: `Per-model attribution is unknown for this entry: it fired ${firedCount(record)} time${firedCount(record) === 1 ? '' : 's'} and no firing carries a model, so how hard it is working under ${currentModel} cannot be measured. Held back rather than counted as zero.`,
|
|
279
|
-
});
|
|
280
|
-
continue;
|
|
281
|
-
}
|
|
282
290
|
if (offered < threshold) {
|
|
283
291
|
notCandidates.push({
|
|
284
292
|
key: record.key,
|
|
@@ -326,6 +334,10 @@ function deriveStanding(record, options) {
|
|
|
326
334
|
const offered = record.offered.aboveThreshold;
|
|
327
335
|
const title = latestTitle(record);
|
|
328
336
|
const threshold = options.limits.retirementOfferThreshold;
|
|
337
|
+
if ((0, learning_usage_eligibility_1.countEvidenceProblem)(record, options.offersEraStart)) {
|
|
338
|
+
return { key: record.key, title, standing: options.currentStanding?.(record.key) ?? 'standard',
|
|
339
|
+
reason: 'No usage-based change is supported.', offered: null, fired: null, recommendation: null };
|
|
340
|
+
}
|
|
329
341
|
let standing;
|
|
330
342
|
let reason;
|
|
331
343
|
if (fired > 0 && daysSince(record.fired.lastFiredAt, now) <= STANDING_RECENT_DAYS) {
|
|
@@ -356,10 +368,11 @@ function deriveStanding(record, options) {
|
|
|
356
368
|
*/
|
|
357
369
|
function assessStanding(store, options) {
|
|
358
370
|
const now = options.now ?? new Date();
|
|
371
|
+
const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
|
|
359
372
|
const assessments = Object.values(store.entries)
|
|
360
|
-
.filter((r) => r.family === 'learning')
|
|
361
|
-
.map((r) => deriveStanding(r, { ...options, now }))
|
|
362
|
-
.sort((a, b) => b.fired - a.fired || b.offered - a.offered);
|
|
373
|
+
.filter((r) => r.family === 'learning' && !excluded.has(r.key))
|
|
374
|
+
.map((r) => deriveStanding(r, { ...options, now, offersEraStart: store.offersEraStart }))
|
|
375
|
+
.sort((a, b) => (b.fired ?? 0) - (a.fired ?? 0) || (b.offered ?? 0) - (a.offered ?? 0));
|
|
363
376
|
return {
|
|
364
377
|
generatedAt: now.toISOString(),
|
|
365
378
|
assessments,
|
|
@@ -374,7 +387,7 @@ function toReportEntry(record) {
|
|
|
374
387
|
fired: firedCount(record),
|
|
375
388
|
ignored: record.fired.byOutcome.ignored,
|
|
376
389
|
lastFired: record.fired.lastFiredAt,
|
|
377
|
-
firedUnder: Object.keys(record.fired.byModel),
|
|
390
|
+
firedUnder: (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record) ? undefined : Object.keys(record.fired.byModel),
|
|
378
391
|
recentLog: record.fired.log,
|
|
379
392
|
};
|
|
380
393
|
}
|
|
@@ -388,7 +401,10 @@ function toReportEntry(record) {
|
|
|
388
401
|
function buildUsageReport(store, options) {
|
|
389
402
|
const now = options.now ?? new Date();
|
|
390
403
|
const limit = options.limit ?? 10;
|
|
391
|
-
const
|
|
404
|
+
const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
|
|
405
|
+
const allRecords = Object.values(store.entries).filter((r) => r.family === 'learning');
|
|
406
|
+
const learningRecords = allRecords.filter(r => !excluded.has(r.key));
|
|
407
|
+
const complete = excluded.size === 0 && (!store.evidence || store.evidence.status === 'reliable');
|
|
392
408
|
const offered = learningRecords.filter((r) => r.offered.total > 0);
|
|
393
409
|
const fired = learningRecords.filter((r) => firedCount(r) > 0);
|
|
394
410
|
const neverFired = offered.filter((r) => firedCount(r) === 0);
|
|
@@ -397,11 +413,11 @@ function buildUsageReport(store, options) {
|
|
|
397
413
|
generatedAt: now.toISOString(),
|
|
398
414
|
windowDays: options.windowDays,
|
|
399
415
|
currentModel: (0, learning_usage_projection_1.resolveCurrentModel)(options.currentModel),
|
|
400
|
-
integrityFlagged:
|
|
416
|
+
integrityFlagged: excluded.size,
|
|
401
417
|
totals: {
|
|
402
|
-
entriesOffered: offered.length,
|
|
403
|
-
entriesWithFirings: fired.length,
|
|
404
|
-
entriesNeverFired: neverFired.length,
|
|
418
|
+
entriesOffered: complete ? offered.length : null,
|
|
419
|
+
entriesWithFirings: complete ? fired.length : null,
|
|
420
|
+
entriesNeverFired: complete ? neverFired.length : null,
|
|
405
421
|
},
|
|
406
422
|
workingHardest: fired
|
|
407
423
|
.sort((a, b) => {
|
|
@@ -277,7 +277,8 @@ const reportCommand = addModelOption(addCommonOptions(new commander_1.Command('r
|
|
|
277
277
|
});
|
|
278
278
|
emit(report, options.json, () => {
|
|
279
279
|
const lines = [
|
|
280
|
-
|
|
280
|
+
report.totals.entriesOffered === null ? 'Corpus totals are unavailable; listed entries have independent usable evidence.'
|
|
281
|
+
: `${report.totals.entriesOffered} entries were offered. ${report.totals.entriesWithFirings} fired. ${report.totals.entriesNeverFired} have never fired.`,
|
|
281
282
|
];
|
|
282
283
|
if (report.workingHardest.length) {
|
|
283
284
|
lines.push('', 'Working hardest');
|
|
@@ -295,7 +296,7 @@ const reportCommand = addModelOption(addCommonOptions(new commander_1.Command('r
|
|
|
295
296
|
report.ignoredWhileInContext.forEach((e, i) => lines.push(`${i + 1}. ${e.title}. Attested ignored ${e.ignored} time${e.ignored === 1 ? '' : 's'} (fired ${e.fired}). In context and the work went against it, so the entry needs work rather than another recurrence.`));
|
|
296
297
|
}
|
|
297
298
|
if (report.integrityFlagged > 0) {
|
|
298
|
-
lines.push('', `${report.integrityFlagged}
|
|
299
|
+
lines.push('', `${report.integrityFlagged} entries were excluded from usage conclusions. Run \`learning-usage verify\` for diagnostic details.`);
|
|
299
300
|
}
|
|
300
301
|
lines.push('', 'Nothing is retired from this view. Run sleep-on-learnings to decide.');
|
|
301
302
|
return lines.join('\n');
|
|
@@ -450,7 +451,8 @@ const pruneCommand = addCommonOptions(new commander_1.Command('prune'))
|
|
|
450
451
|
const workspaceRoot = resolveWorkspaceRoot(options);
|
|
451
452
|
const limits = (0, learning_usage_store_1.resolveUsageLimits)(workspaceRoot);
|
|
452
453
|
const result = (0, learning_usage_store_1.pruneUsageStore)(limits);
|
|
453
|
-
emit({ ...result, storePath: (0, learning_usage_store_1.resolveUsageStorePath)(), limits }, options.json, () =>
|
|
454
|
+
emit({ ...result, storePath: (0, learning_usage_store_1.resolveUsageStorePath)(), limits }, options.json, () => result.maintenanceStatus === 'deferred' ? `Maintenance deferred; history unchanged. ${result.evidence?.observations.join(', ') ?? ''}`
|
|
455
|
+
: `Pruned ${result.prunedOfferRecords} offer records and ${result.prunedFiringRecords} firing records across ${result.entries} entries. Aggregates unchanged.`);
|
|
454
456
|
});
|
|
455
457
|
function mb(bytes) {
|
|
456
458
|
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
@@ -472,9 +474,11 @@ const verifyCommand = addModelOption(addCommonOptions(new commander_1.Command('v
|
|
|
472
474
|
'',
|
|
473
475
|
];
|
|
474
476
|
const finding = (count, singular, plural, remedy) => count === 0 ? ` none: ${singular}` : ` ${count} ${count === 1 ? singular : plural}. ${remedy}`;
|
|
477
|
+
if (report.evidence)
|
|
478
|
+
lines.push(`Evidence: ${report.evidence.status}. ${report.evidence.observations.join(', ')}.`, ...(report.evidence.recovery ? [`Recovered from ${report.evidence.recovery}; primary replacement was not persisted by this read.`] : []), '');
|
|
475
479
|
lines.push('Identity', finding(report.twinPairs.length, 'lesson tracked under two keys', 'lessons tracked under two keys', 'Its offers and its firings are on different records, so every count about it is partial.'), report.unmatchedTitleKeys.length
|
|
476
480
|
? ` ${report.unmatchedTitleKeys.length} title-keyed record${report.unmatchedTitleKeys.length === 1 ? ' has' : 's have'} no id twin. Expected for entries no tier has stamped; run \`stamp-ids\` to reduce it.`
|
|
477
|
-
: ' every title-keyed record has been matched or has no twin', '', 'Counters', finding(report.invariantViolations.length, 'entry holds more attested firings than recorded offers', 'entries hold more attested firings than recorded offers', '
|
|
481
|
+
: ' every title-keyed record has been matched or has no twin', '', 'Counters', finding(report.invariantViolations.length, 'entry holds more attested firings than recorded offers', 'entries hold more attested firings than recorded offers', 'The counts are inconsistent. This observation does not establish their cause.'), finding(report.caseVariantModelBuckets.length, 'model bucket differs only by case', 'model buckets differ only by case', 'One model spelled two ways splits its own denominator, so every per-model conclusion about it is computed on a fraction of its traffic.'), finding(report.unknownModelAttribution.length, 'entry has incomplete model attribution', 'entries have incomplete model attribution', 'Affected model conclusions are withheld. The cause is not established; unknown historical attribution cannot be inferred from the current model.'), '', 'Storage');
|
|
478
482
|
if (report.storeCopies.length === 0) {
|
|
479
483
|
lines.push(' one store file, no unmerged copies');
|
|
480
484
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.modelAttributionIncomplete = modelAttributionIncomplete;
|
|
4
|
+
exports.countEvidenceProblem = countEvidenceProblem;
|
|
5
|
+
exports.usageEvidenceExclusions = usageEvidenceExclusions;
|
|
6
|
+
const learning_usage_store_1 = require("./learning-usage-store");
|
|
7
|
+
const validCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
8
|
+
/** Missing attribution is unknown, including a known bucket beside unattributed history. */
|
|
9
|
+
function modelAttributionIncomplete(record) {
|
|
10
|
+
const buckets = [record.offered.byModel, record.fired.byModel];
|
|
11
|
+
if (buckets.some(map => !map || Object.entries(map).some(([name, n]) => !validCount(n) || !(0, learning_usage_store_1.normalizeModelName)(name) || (0, learning_usage_store_1.normalizeModelName)(name) !== name)))
|
|
12
|
+
return true;
|
|
13
|
+
return Object.values(record.offered.byModel).reduce((a, b) => a + b, 0) !== record.offered.total
|
|
14
|
+
|| Object.values(record.fired.byModel).reduce((a, b) => a + b, 0)
|
|
15
|
+
!== record.fired.byOutcome.applied + record.fired.byOutcome.prevented;
|
|
16
|
+
}
|
|
17
|
+
/** Validate counts independently of a cached integrity flag. Never alter the record. */
|
|
18
|
+
function countEvidenceProblem(record, offersEraStart) {
|
|
19
|
+
const counts = [record.offered.total, record.offered.aboveThreshold, record.fired.total,
|
|
20
|
+
...Object.values(record.offered.byModel), ...Object.values(record.fired.byModel),
|
|
21
|
+
...Object.values(record.fired.byOutcome)];
|
|
22
|
+
if (counts.some(n => !validCount(n)) || learning_usage_store_1.FIRING_OUTCOMES.some(outcome => !validCount(record.fired.byOutcome[outcome]))
|
|
23
|
+
|| record.offered.aboveThreshold > record.offered.total
|
|
24
|
+
|| Object.values(record.fired.byOutcome).reduce((a, b) => a + b, 0) !== record.fired.total) {
|
|
25
|
+
return 'Usage counters are inconsistent.';
|
|
26
|
+
}
|
|
27
|
+
const violation = record.integrity ?? (0, learning_usage_store_1.integrityViolation)(record, offersEraStart);
|
|
28
|
+
if (!offersEraStart && record.offered.total > 0 && record.fired.total > record.offered.total) {
|
|
29
|
+
return 'Excess uses lack provenance establishing the offer-recording era.';
|
|
30
|
+
}
|
|
31
|
+
return violation ? violation.detail : null;
|
|
32
|
+
}
|
|
33
|
+
/** Read-time exclusions follow the actual record, never ambient UI or model state. */
|
|
34
|
+
function usageEvidenceExclusions(store) {
|
|
35
|
+
const excluded = new Map();
|
|
36
|
+
for (const key of store.evidence?.excludedKeys ?? [])
|
|
37
|
+
excluded.set(key, 'Unreadable usage record.');
|
|
38
|
+
const byTitle = new Map();
|
|
39
|
+
for (const record of Object.values(store.entries)) {
|
|
40
|
+
const problem = countEvidenceProblem(record, store.offersEraStart);
|
|
41
|
+
if (store.evidence?.status === 'unavailable')
|
|
42
|
+
excluded.set(record.key, 'Usage history is unavailable.');
|
|
43
|
+
else if (problem)
|
|
44
|
+
excluded.set(record.key, problem);
|
|
45
|
+
if (!record.fileType)
|
|
46
|
+
continue;
|
|
47
|
+
for (const title of record.titles) {
|
|
48
|
+
const titleKey = (0, learning_usage_store_1.titleUsageKey)(record.fileType, title);
|
|
49
|
+
byTitle.set(titleKey, [...new Set([...(byTitle.get(titleKey) ?? []), record.key])]);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
for (const keys of byTitle.values()) {
|
|
53
|
+
if (keys.length > 1 && keys.some(key => key.startsWith('T:'))) {
|
|
54
|
+
for (const key of keys)
|
|
55
|
+
excluded.set(key, 'Ambiguous duplicate usage identity.');
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return excluded;
|
|
59
|
+
}
|
|
@@ -15,6 +15,7 @@ exports.resolveCurrentModel = resolveCurrentModel;
|
|
|
15
15
|
* Nothing here writes.
|
|
16
16
|
*/
|
|
17
17
|
const learning_usage_store_1 = require("./learning-usage-store");
|
|
18
|
+
const learning_usage_eligibility_1 = require("./learning-usage-eligibility");
|
|
18
19
|
function toLookupValue(record) {
|
|
19
20
|
return {
|
|
20
21
|
key: record.key,
|
|
@@ -24,7 +25,7 @@ function toLookupValue(record) {
|
|
|
24
25
|
firedTotalAttestations: record.fired.total,
|
|
25
26
|
lastFired: record.fired.lastFiredAt,
|
|
26
27
|
lastOffered: record.offered.lastOfferedAt,
|
|
27
|
-
firedUnder: Object.keys(record.fired.byModel),
|
|
28
|
+
firedUnder: (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record) ? undefined : Object.keys(record.fired.byModel),
|
|
28
29
|
byOutcome: record.fired.byOutcome,
|
|
29
30
|
log: record.fired.log,
|
|
30
31
|
};
|
|
@@ -36,7 +37,10 @@ function toLookupValue(record) {
|
|
|
36
37
|
*/
|
|
37
38
|
function buildUsageLookup(store) {
|
|
38
39
|
const lookup = new Map();
|
|
40
|
+
const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
|
|
39
41
|
for (const record of Object.values(store.entries)) {
|
|
42
|
+
if (excluded.has(record.key))
|
|
43
|
+
continue;
|
|
40
44
|
const value = toLookupValue(record);
|
|
41
45
|
lookup.set(record.key, value);
|
|
42
46
|
if (record.fileType) {
|
|
@@ -230,6 +230,13 @@ function normalizeModelName(model) {
|
|
|
230
230
|
function emptyStore() {
|
|
231
231
|
return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {}, foldedThroughBytes: {}, offersEraStart: null };
|
|
232
232
|
}
|
|
233
|
+
function withEvidence(store, evidence) {
|
|
234
|
+
Object.defineProperty(store, 'evidence', { value: evidence, enumerable: false, configurable: true });
|
|
235
|
+
return store;
|
|
236
|
+
}
|
|
237
|
+
function unavailableStore(observation) {
|
|
238
|
+
return withEvidence(emptyStore(), { status: 'unavailable', excludedKeys: [], observations: [observation] });
|
|
239
|
+
}
|
|
233
240
|
/**
|
|
234
241
|
* Is this parsed value a record every consumer can read without crashing?
|
|
235
242
|
*
|
|
@@ -254,14 +261,21 @@ function isUsableRecord(value) {
|
|
|
254
261
|
const fired = record.fired;
|
|
255
262
|
if (!offered || typeof offered !== 'object' || !fired || typeof fired !== 'object')
|
|
256
263
|
return false;
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
&&
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
&&
|
|
263
|
-
&&
|
|
264
|
-
&&
|
|
264
|
+
const count = (v) => Number.isSafeInteger(v) && v >= 0;
|
|
265
|
+
const counts = (v) => !!v && typeof v === 'object' && !Array.isArray(v)
|
|
266
|
+
&& Object.values(v).every(count);
|
|
267
|
+
const nullableString = (v) => v === null || typeof v === 'string';
|
|
268
|
+
return typeof record.key === 'string' && (record.family === 'learning' || record.family === 'rule')
|
|
269
|
+
&& Array.isArray(offered.log) && offered.log.every(r => r && isValidDate(r.date)
|
|
270
|
+
&& typeof r.job === 'string' && nullableString(r.model))
|
|
271
|
+
&& count(offered.total) && count(offered.aboveThreshold)
|
|
272
|
+
&& counts(offered.byModel)
|
|
273
|
+
&& Array.isArray(fired.log) && fired.log.every(r => r && isValidDate(r.date)
|
|
274
|
+
&& typeof r.job === 'string' && nullableString(r.model) && exports.FIRING_OUTCOMES.includes(r.outcome))
|
|
275
|
+
&& count(fired.total) && counts(fired.byOutcome) && counts(fired.byModel)
|
|
276
|
+
&& exports.FIRING_OUTCOMES.every(outcome => count(fired.byOutcome[outcome]))
|
|
277
|
+
&& Array.isArray(fired.seenSources) && fired.seenSources.every(s => typeof s === 'string')
|
|
278
|
+
&& Array.isArray(record.titles) && record.titles.every(t => typeof t === 'string');
|
|
265
279
|
}
|
|
266
280
|
/**
|
|
267
281
|
* The earliest offer date a pre-#1752 snapshot can evidence.
|
|
@@ -301,15 +315,23 @@ function readCheckpoints(value) {
|
|
|
301
315
|
}
|
|
302
316
|
/** Read the normal snapshot, recovering a missing or torn copy from its checkpoint. */
|
|
303
317
|
function readUsageSnapshot(storePath = resolveUsageStorePath()) {
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
318
|
+
const primary = parseUsageSnapshot(storePath);
|
|
319
|
+
if (primary && primary.evidence?.status !== 'unavailable')
|
|
320
|
+
return primary;
|
|
321
|
+
// An unknown schema is not evidence that an older checkpoint can replace it.
|
|
322
|
+
if (primary?.evidence?.observations.includes('unsupported-schema'))
|
|
323
|
+
return primary;
|
|
324
|
+
const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
|
|
325
|
+
if (backup && !backup.evidence)
|
|
326
|
+
return withEvidence(backup, {
|
|
327
|
+
status: 'reliable', excludedKeys: [],
|
|
328
|
+
observations: [(0, fs_1.existsSync)(storePath) ? 'primary-unreadable' : 'primary-missing'],
|
|
329
|
+
recovery: 'checkpoint', persisted: false,
|
|
330
|
+
});
|
|
309
331
|
if ((0, fs_1.existsSync)(resolveUsageCheckpointPath(storePath))) {
|
|
310
332
|
throw new Error('Usage snapshot and checkpoint backup are unreadable. Restore a matching backup; do not delete the usage directory.');
|
|
311
333
|
}
|
|
312
|
-
return emptyStore();
|
|
334
|
+
return primary ?? emptyStore();
|
|
313
335
|
}
|
|
314
336
|
function resolveUsageCheckpointPath(storePath) {
|
|
315
337
|
return storePath + '.checkpoint';
|
|
@@ -324,19 +346,29 @@ function parseUsageSnapshot(storePath) {
|
|
|
324
346
|
// `entry.offered` throws on it.
|
|
325
347
|
if (!parsed || typeof parsed !== 'object' || !parsed.entries
|
|
326
348
|
|| typeof parsed.entries !== 'object' || Array.isArray(parsed.entries)) {
|
|
327
|
-
return
|
|
349
|
+
return unavailableStore('malformed-snapshot');
|
|
350
|
+
}
|
|
351
|
+
if (parsed.version !== undefined && parsed.version !== STORE_VERSION)
|
|
352
|
+
return unavailableStore('unsupported-schema');
|
|
353
|
+
if (parsed.foldedThroughBytes !== undefined && (!parsed.foldedThroughBytes
|
|
354
|
+
|| typeof parsed.foldedThroughBytes !== 'object' || Array.isArray(parsed.foldedThroughBytes)
|
|
355
|
+
|| Object.values(parsed.foldedThroughBytes).some(v => !Number.isSafeInteger(v) || v < 0))) {
|
|
356
|
+
return unavailableStore('malformed-checkpoints');
|
|
328
357
|
}
|
|
329
358
|
// Drop any value that is not a usable record rather than handing a caller
|
|
330
359
|
// something it will crash on. A partially corrupt store degrades to the records
|
|
331
360
|
// that are still readable before consolidation. Consolidated snapshots must be complete.
|
|
332
361
|
const entries = {};
|
|
362
|
+
const excludedKeys = [];
|
|
333
363
|
for (const [key, value] of Object.entries(parsed.entries)) {
|
|
334
|
-
if (isUsableRecord(value))
|
|
364
|
+
if (isUsableRecord(value) && value.key === key)
|
|
335
365
|
entries[key] = value;
|
|
336
366
|
else if (parsed.consolidated)
|
|
337
|
-
return
|
|
367
|
+
return unavailableStore('malformed-consolidated-snapshot');
|
|
368
|
+
else
|
|
369
|
+
excludedKeys.push(key);
|
|
338
370
|
}
|
|
339
|
-
|
|
371
|
+
const store = {
|
|
340
372
|
consolidated: parsed.consolidated === true,
|
|
341
373
|
version: parsed.version ?? STORE_VERSION,
|
|
342
374
|
updatedAt: parsed.updatedAt ?? emptyStore().updatedAt,
|
|
@@ -346,9 +378,12 @@ function parseUsageSnapshot(storePath) {
|
|
|
346
378
|
? parsed.offersEraStart
|
|
347
379
|
: deriveOffersEraStart(entries),
|
|
348
380
|
};
|
|
381
|
+
return excludedKeys.length ? withEvidence(store, {
|
|
382
|
+
status: 'partial', excludedKeys, observations: ['malformed-records'],
|
|
383
|
+
}) : store;
|
|
349
384
|
}
|
|
350
385
|
catch {
|
|
351
|
-
return
|
|
386
|
+
return unavailableStore('unreadable-snapshot');
|
|
352
387
|
}
|
|
353
388
|
}
|
|
354
389
|
/**
|
|
@@ -357,9 +392,12 @@ function parseUsageSnapshot(storePath) {
|
|
|
357
392
|
* journals, so a torn ordinary refresh can recover without replaying old history.
|
|
358
393
|
*/
|
|
359
394
|
function writeUsageSnapshot(store, storePath = resolveUsageStorePath()) {
|
|
395
|
+
if (store.evidence && store.evidence.status !== 'reliable')
|
|
396
|
+
return false;
|
|
360
397
|
try {
|
|
361
398
|
(0, fs_1.mkdirSync)((0, path_1.dirname)(storePath), { recursive: true });
|
|
362
|
-
|
|
399
|
+
const { evidence: _evidence, ...snapshot } = store;
|
|
400
|
+
(0, fs_1.writeFileSync)(storePath, `${JSON.stringify(snapshot, null, 2)}\n`, { encoding: 'utf8', flush: true });
|
|
363
401
|
return true;
|
|
364
402
|
}
|
|
365
403
|
catch {
|
|
@@ -385,7 +423,10 @@ function foldUsageEvents(options) {
|
|
|
385
423
|
throw new Error('Usage journal no longer matches its consolidated snapshot. Restore matching checkpoint files before continuing.');
|
|
386
424
|
if (tail.restarted) {
|
|
387
425
|
const cold = (0, learning_usage_events_1.readEventsSince)(options.eventDir, {});
|
|
388
|
-
return applyTail(emptyStore(),
|
|
426
|
+
return applyTail(withEvidence(emptyStore(), {
|
|
427
|
+
status: 'unavailable', excludedKeys: base.evidence?.excludedKeys ?? [],
|
|
428
|
+
observations: [...(base.evidence?.observations ?? []), 'journal-restarted'],
|
|
429
|
+
}), cold, limits, now, true);
|
|
389
430
|
}
|
|
390
431
|
return applyTail(base, tail, limits, now, false);
|
|
391
432
|
}
|
|
@@ -398,12 +439,19 @@ function applyTail(base, tail, limits, now, restarted) {
|
|
|
398
439
|
foldedThroughBytes: { ...(base.foldedThroughBytes ?? {}) },
|
|
399
440
|
offersEraStart: base.offersEraStart ?? null,
|
|
400
441
|
};
|
|
442
|
+
if (base.evidence)
|
|
443
|
+
withEvidence(store, base.evidence);
|
|
401
444
|
let tailBytes = 0;
|
|
445
|
+
let unsupportedEvents = false;
|
|
402
446
|
const touched = new Set();
|
|
403
447
|
for (const read of tail.reads) {
|
|
404
448
|
const from = restarted ? 0 : (store.foldedThroughBytes[read.machineId] ?? 0);
|
|
405
449
|
tailBytes += Math.max(0, read.through - from);
|
|
406
450
|
for (const event of read.records) {
|
|
451
|
+
if (!isSupportedUsageEvent(event)) {
|
|
452
|
+
unsupportedEvents = true;
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
407
455
|
const key = applyEvent(store, event, limits, now);
|
|
408
456
|
if (key)
|
|
409
457
|
touched.add(key);
|
|
@@ -418,17 +466,63 @@ function applyTail(base, tail, limits, now, restarted) {
|
|
|
418
466
|
// E: the invariant is evaluated on the read path, where it is always current
|
|
419
467
|
// and costs the write path nothing.
|
|
420
468
|
flagIntegrity(store);
|
|
469
|
+
const incompleteJournal = tail.skippedLines > 0
|
|
470
|
+
|| tail.reads.some(read => read.through < read.baseOffset + read.size - read.headerBytes);
|
|
471
|
+
if (incompleteJournal || unsupportedEvents) {
|
|
472
|
+
withEvidence(store, {
|
|
473
|
+
status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [],
|
|
474
|
+
observations: [...(store.evidence?.observations ?? []),
|
|
475
|
+
...(incompleteJournal ? ['incomplete-journal'] : []),
|
|
476
|
+
...(unsupportedEvents ? ['unsupported-journal-events'] : [])],
|
|
477
|
+
});
|
|
478
|
+
}
|
|
421
479
|
return { store, skippedLines: tail.skippedLines, restarted, tailBytes, logs: tail.reads };
|
|
422
480
|
}
|
|
423
481
|
/**
|
|
424
482
|
* The checkpointed read every consumer uses. Snapshot plus the bytes appended
|
|
425
483
|
* past it, republished once the tail is worth re-checkpointing.
|
|
426
484
|
*/
|
|
485
|
+
/** Metadata detects observed writes/sync during a read; it is not a writer lock. */
|
|
486
|
+
function usageReadState(storePath, eventDir) {
|
|
487
|
+
const dir = (0, path_1.dirname)(storePath);
|
|
488
|
+
const copies = (0, fs_1.existsSync)(dir) ? (0, fs_1.readdirSync)(dir)
|
|
489
|
+
.filter(name => /^learning-usage(-.+)?\.json$/.test(name) && name !== STORE_FILE_NAME).sort() : [];
|
|
490
|
+
const paths = [storePath, resolveUsageCheckpointPath(storePath),
|
|
491
|
+
...(0, learning_usage_events_1.listEventLogs)(eventDir).map(log => log.path), ...copies.map(name => (0, path_1.join)(dir, name))];
|
|
492
|
+
const fingerprint = JSON.stringify(paths.map(file => {
|
|
493
|
+
try {
|
|
494
|
+
const stat = (0, fs_1.statSync)(file);
|
|
495
|
+
return [file, stat.size, stat.mtimeMs, stat.ctimeMs, stat.ino];
|
|
496
|
+
}
|
|
497
|
+
catch {
|
|
498
|
+
return [file, 'missing'];
|
|
499
|
+
}
|
|
500
|
+
}));
|
|
501
|
+
return { fingerprint, copies };
|
|
502
|
+
}
|
|
427
503
|
function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
|
|
428
|
-
const snapshot = readUsageSnapshot(storePath);
|
|
429
504
|
const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
|
|
430
|
-
|
|
431
|
-
|
|
505
|
+
const before = usageReadState(storePath, eventDir);
|
|
506
|
+
const snapshot = readUsageSnapshot(storePath);
|
|
507
|
+
const logs = (0, learning_usage_events_1.listEventLogs)(eventDir);
|
|
508
|
+
const finish = (store) => {
|
|
509
|
+
const after = usageReadState(storePath, eventDir);
|
|
510
|
+
const observations = [...(store.evidence?.observations ?? [])];
|
|
511
|
+
if (before.fingerprint !== after.fingerprint)
|
|
512
|
+
observations.push('inputs-changed-during-read');
|
|
513
|
+
if (after.copies.length)
|
|
514
|
+
observations.push('unmerged-store-copies');
|
|
515
|
+
if (store.consolidated && Object.keys(store.foldedThroughBytes ?? {}).some(machineId => !logs.some(log => log.machineId === machineId)))
|
|
516
|
+
observations.push('consolidated-journal-missing');
|
|
517
|
+
if (observations.length > (store.evidence?.observations.length ?? 0))
|
|
518
|
+
withEvidence(store, {
|
|
519
|
+
status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [], observations,
|
|
520
|
+
});
|
|
521
|
+
flagIntegrity(store);
|
|
522
|
+
return store;
|
|
523
|
+
};
|
|
524
|
+
if (logs.length === 0 || snapshot.evidence?.observations.includes('unsupported-schema'))
|
|
525
|
+
return finish(snapshot);
|
|
432
526
|
const fold = (base) => foldUsageEvents({
|
|
433
527
|
eventDir,
|
|
434
528
|
limits: options.limits,
|
|
@@ -442,14 +536,20 @@ function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
|
|
|
442
536
|
}
|
|
443
537
|
catch (error) {
|
|
444
538
|
const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
|
|
445
|
-
if (!backup)
|
|
539
|
+
if (!backup || backup.evidence)
|
|
446
540
|
throw error;
|
|
447
|
-
folded = fold(backup
|
|
541
|
+
folded = fold(withEvidence(backup, {
|
|
542
|
+
status: 'reliable', excludedKeys: [], observations: ['primary-journal-mismatch'],
|
|
543
|
+
recovery: 'checkpoint', persisted: false,
|
|
544
|
+
}));
|
|
448
545
|
}
|
|
546
|
+
finish(folded.store);
|
|
449
547
|
const mode = options.refreshSnapshot ?? 'auto';
|
|
450
548
|
const shouldRefresh = mode === 'always'
|
|
451
549
|
|| (mode === 'auto' && (folded.tailBytes >= SNAPSHOT_REFRESH_BYTES || !(0, fs_1.existsSync)(storePath) || folded.restarted));
|
|
452
|
-
|
|
550
|
+
// Recovery is usable in memory. Persisting a primary replacement requires the
|
|
551
|
+
// explicit maintenance interval; an ordinary reader never establishes it.
|
|
552
|
+
if (shouldRefresh && !folded.store.evidence) {
|
|
453
553
|
folded.store.updatedAt = (options.now ?? new Date()).toISOString();
|
|
454
554
|
writeUsageSnapshot(folded.store, storePath);
|
|
455
555
|
}
|
|
@@ -636,6 +736,20 @@ function applyEvent(store, event, limits, now) {
|
|
|
636
736
|
void now;
|
|
637
737
|
return null;
|
|
638
738
|
}
|
|
739
|
+
function isSupportedUsageEvent(event) {
|
|
740
|
+
if (typeof event.key !== 'string' || !event.key || typeof event.date !== 'string' || !isValidDate(event.date))
|
|
741
|
+
return false;
|
|
742
|
+
if (event.family !== 'learning' && event.family !== 'rule')
|
|
743
|
+
return false;
|
|
744
|
+
if (typeof event.job !== 'string' || (event.model !== null && typeof event.model !== 'string'))
|
|
745
|
+
return false;
|
|
746
|
+
if (event.kind === 'offer')
|
|
747
|
+
return typeof event.aboveThreshold === 'boolean';
|
|
748
|
+
if (event.kind === 'firing')
|
|
749
|
+
return exports.FIRING_OUTCOMES.includes(event.outcome)
|
|
750
|
+
&& (event.source === undefined || event.source === 'attested' || event.source === 'backfill-legacy');
|
|
751
|
+
return false;
|
|
752
|
+
}
|
|
639
753
|
function applyOfferEvent(store, event, common) {
|
|
640
754
|
const record = ensureRecord(store, common.key, common.family, common.fileType, common.level);
|
|
641
755
|
rememberTitle(record, typeof event.title === 'string' ? event.title : undefined);
|
|
@@ -883,19 +997,17 @@ function noteTitleChange(key, newTitle, options) {
|
|
|
883
997
|
store.updatedAt = (options.now ?? new Date()).toISOString();
|
|
884
998
|
writeUsageSnapshot(store, storePath);
|
|
885
999
|
}
|
|
886
|
-
/**
|
|
887
|
-
* Apply retention and log bounds across the whole store, and compact the event
|
|
888
|
-
* log behind it.
|
|
889
|
-
*
|
|
890
|
-
* Compaction is a requirement rather than an optimization: S4 measured a cold
|
|
891
|
-
* full fold at 84 ms on today's volume but 794 ms at ten times it, and the
|
|
892
|
-
* checkpoint only makes a cold fold rare, not bounded.
|
|
893
|
-
*/
|
|
894
1000
|
function pruneUsageStore(limits, options = {}) {
|
|
895
1001
|
const storePath = options.storePath ?? resolveUsageStorePath();
|
|
896
1002
|
const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
|
|
897
1003
|
const now = options.now ?? new Date();
|
|
898
1004
|
const store = readUsageStore(storePath, { limits, now, eventDir });
|
|
1005
|
+
if (store.evidence && store.evidence.status !== 'reliable')
|
|
1006
|
+
return {
|
|
1007
|
+
maintenanceStatus: 'deferred', evidence: store.evidence,
|
|
1008
|
+
prunedOfferRecords: 0, prunedFiringRecords: 0, entries: Object.keys(store.entries).length,
|
|
1009
|
+
compaction: compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store }),
|
|
1010
|
+
};
|
|
899
1011
|
let prunedOfferRecords = 0;
|
|
900
1012
|
let prunedFiringRecords = 0;
|
|
901
1013
|
for (const record of Object.values(store.entries)) {
|
|
@@ -906,7 +1018,10 @@ function pruneUsageStore(limits, options = {}) {
|
|
|
906
1018
|
prunedFiringRecords += firingsBefore - record.fired.log.length;
|
|
907
1019
|
}
|
|
908
1020
|
const compaction = compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store });
|
|
909
|
-
return {
|
|
1021
|
+
return {
|
|
1022
|
+
maintenanceStatus: compaction.maintenanceStatus, evidence: compaction.evidence,
|
|
1023
|
+
prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length, compaction,
|
|
1024
|
+
};
|
|
910
1025
|
}
|
|
911
1026
|
/** Save totals and source bookmarks in both copies, then clear processed journals.
|
|
912
1027
|
* Requires an interval without concurrent writers or sync activity.
|
|
@@ -915,6 +1030,14 @@ function compactUsageEvents(options) {
|
|
|
915
1030
|
const now = options.now ?? new Date();
|
|
916
1031
|
const store = options.store
|
|
917
1032
|
?? readUsageStore(options.snapshotPath, { limits: options.limits, now, eventDir: options.eventDir });
|
|
1033
|
+
// Do not turn a partial read into a durable replacement or advance bookmarks
|
|
1034
|
+
// over evidence this reader could not understand. Independent review can run.
|
|
1035
|
+
if (store.evidence && store.evidence.status !== 'reliable')
|
|
1036
|
+
return {
|
|
1037
|
+
maintenanceStatus: 'deferred', evidence: store.evidence,
|
|
1038
|
+
scanned: 0, kept: 0, dropped: 0, skippedLines: 0, logs: [],
|
|
1039
|
+
checkpoints: { ...(store.foldedThroughBytes ?? {}) },
|
|
1040
|
+
};
|
|
918
1041
|
for (const log of (0, learning_usage_events_1.listEventLogs)(options.eventDir)) {
|
|
919
1042
|
if ((store.foldedThroughBytes?.[log.machineId] ?? 0) < log.baseOffset) {
|
|
920
1043
|
throw new Error('Cannot consolidate from a snapshot older than the journal checkpoint.');
|
|
@@ -931,7 +1054,8 @@ function compactUsageEvents(options) {
|
|
|
931
1054
|
const temp = checkpoint + '.tmp';
|
|
932
1055
|
try {
|
|
933
1056
|
(0, fs_1.mkdirSync)((0, path_1.dirname)(checkpoint), { recursive: true });
|
|
934
|
-
|
|
1057
|
+
const { evidence: _evidence, ...snapshot } = store;
|
|
1058
|
+
(0, fs_1.writeFileSync)(temp, JSON.stringify(snapshot, null, 2) + '\n', { encoding: 'utf8', flush: true });
|
|
935
1059
|
(0, fs_1.renameSync)(temp, checkpoint);
|
|
936
1060
|
}
|
|
937
1061
|
catch {
|
|
@@ -944,7 +1068,7 @@ function compactUsageEvents(options) {
|
|
|
944
1068
|
if (!writeUsageSnapshot(store, options.snapshotPath)) {
|
|
945
1069
|
throw new Error('Cannot consolidate usage events: aggregate snapshot could not be saved.');
|
|
946
1070
|
}
|
|
947
|
-
|
|
1071
|
+
const compaction = (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
|
|
948
1072
|
if (event.kind !== 'offer' && event.kind !== 'firing')
|
|
949
1073
|
return true;
|
|
950
1074
|
const date = typeof event.date === 'string' ? event.date : '';
|
|
@@ -952,4 +1076,5 @@ function compactUsageEvents(options) {
|
|
|
952
1076
|
return true;
|
|
953
1077
|
return event.kind === 'firing' && !exports.FIRING_OUTCOMES.includes(event.outcome);
|
|
954
1078
|
}, store.foldedThroughBytes ?? {});
|
|
1079
|
+
return { ...compaction, maintenanceStatus: 'completed', evidence: store.evidence };
|
|
955
1080
|
}
|