fraim 2.0.340 → 2.0.341

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,6 +27,7 @@ const fs_1 = require("fs");
27
27
  const path_1 = require("path");
28
28
  const learning_usage_store_1 = require("./learning-usage-store");
29
29
  const learning_usage_projection_1 = require("./learning-usage-projection");
30
+ const learning_usage_eligibility_1 = require("./learning-usage-eligibility");
30
31
  function daysBetween(from, to) {
31
32
  const a = new Date(from).getTime();
32
33
  const b = new Date(to).getTime();
@@ -39,6 +40,7 @@ function firedCount(record) {
39
40
  }
40
41
  function classifyRecurrence(store, options) {
41
42
  const record = store.entries[options.key];
43
+ const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
42
44
  const base = {
43
45
  key: options.key,
44
46
  title: record?.titles[record.titles.length - 1] ?? null,
@@ -51,6 +53,14 @@ function classifyRecurrence(store, options) {
51
53
  firedTotal: record ? firedCount(record) : 0,
52
54
  },
53
55
  };
56
+ if (record && excluded.has(record.key)) {
57
+ return {
58
+ ...base, classification: 'undetermined',
59
+ diagnosis: 'The available usage evidence cannot establish this recurrence.',
60
+ recommendation: 'Review independent source evidence without attributing this recurrence to usage.',
61
+ recurrenceBump: false, targetOfFix: 'none',
62
+ };
63
+ }
54
64
  if (!record) {
55
65
  return {
56
66
  ...base,
@@ -94,7 +104,7 @@ function classifyRecurrence(store, options) {
94
104
  // per entry per session and a plain newest-N bound evicted the job-labelled
95
105
  // ones within about a day. Asking about a job the store has never retained an
96
106
  // offer for is a gap in the record, not a delivery failure.
97
- const jobCoverage = Object.values(store.entries).some((entry) => entry.offered.log.some((offer) => offer.job === options.job && daysBetween(offer.date, options.date) <= options.windowDays));
107
+ const jobCoverage = Object.values(store.entries).some((entry) => !excluded.has(entry.key) && entry.offered.log.some((offer) => offer.job === options.job && daysBetween(offer.date, options.date) <= options.windowDays));
98
108
  if (matchingOffers.length > 0) {
99
109
  return {
100
110
  ...base,
@@ -231,7 +241,7 @@ function firingsUnderModel(record, model) {
231
241
  * unrecorded.
232
242
  */
233
243
  function modelAttributionUnknown(record) {
234
- return firedCount(record) > 0 && Object.keys(record.fired.byModel).length === 0;
244
+ return (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record);
235
245
  }
236
246
  function earlierModelFirings(record, currentModel) {
237
247
  return Object.entries(record.fired.byModel)
@@ -261,24 +271,22 @@ function findRetirementCandidates(store, options) {
261
271
  }
262
272
  const candidates = [];
263
273
  const notCandidates = [];
274
+ const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
264
275
  for (const record of Object.values(store.entries)) {
265
276
  if (record.family !== 'learning')
266
277
  continue;
267
278
  const title = latestTitle(record);
279
+ if (excluded.has(record.key) || modelAttributionUnknown(record)) {
280
+ notCandidates.push({ key: record.key, title,
281
+ reason: excluded.get(record.key) ?? 'Model attribution is incomplete; unknown use is not zero use.' });
282
+ continue;
283
+ }
268
284
  const offered = offersUnderModel(record, currentModel);
269
285
  const fired = firingsUnderModel(record, currentModel);
270
286
  if (fired > 0) {
271
287
  notCandidates.push({ key: record.key, title, reason: `It fired ${fired} time${fired === 1 ? '' : 's'} under the current model, so it is still earning its place.` });
272
288
  continue;
273
289
  }
274
- if (modelAttributionUnknown(record)) {
275
- notCandidates.push({
276
- key: record.key,
277
- title,
278
- reason: `Per-model attribution is unknown for this entry: it fired ${firedCount(record)} time${firedCount(record) === 1 ? '' : 's'} and no firing carries a model, so how hard it is working under ${currentModel} cannot be measured. Held back rather than counted as zero.`,
279
- });
280
- continue;
281
- }
282
290
  if (offered < threshold) {
283
291
  notCandidates.push({
284
292
  key: record.key,
@@ -326,6 +334,10 @@ function deriveStanding(record, options) {
326
334
  const offered = record.offered.aboveThreshold;
327
335
  const title = latestTitle(record);
328
336
  const threshold = options.limits.retirementOfferThreshold;
337
+ if ((0, learning_usage_eligibility_1.countEvidenceProblem)(record, options.offersEraStart)) {
338
+ return { key: record.key, title, standing: options.currentStanding?.(record.key) ?? 'standard',
339
+ reason: 'No usage-based change is supported.', offered: null, fired: null, recommendation: null };
340
+ }
329
341
  let standing;
330
342
  let reason;
331
343
  if (fired > 0 && daysSince(record.fired.lastFiredAt, now) <= STANDING_RECENT_DAYS) {
@@ -356,10 +368,11 @@ function deriveStanding(record, options) {
356
368
  */
357
369
  function assessStanding(store, options) {
358
370
  const now = options.now ?? new Date();
371
+ const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
359
372
  const assessments = Object.values(store.entries)
360
- .filter((r) => r.family === 'learning')
361
- .map((r) => deriveStanding(r, { ...options, now }))
362
- .sort((a, b) => b.fired - a.fired || b.offered - a.offered);
373
+ .filter((r) => r.family === 'learning' && !excluded.has(r.key))
374
+ .map((r) => deriveStanding(r, { ...options, now, offersEraStart: store.offersEraStart }))
375
+ .sort((a, b) => (b.fired ?? 0) - (a.fired ?? 0) || (b.offered ?? 0) - (a.offered ?? 0));
363
376
  return {
364
377
  generatedAt: now.toISOString(),
365
378
  assessments,
@@ -374,7 +387,7 @@ function toReportEntry(record) {
374
387
  fired: firedCount(record),
375
388
  ignored: record.fired.byOutcome.ignored,
376
389
  lastFired: record.fired.lastFiredAt,
377
- firedUnder: Object.keys(record.fired.byModel),
390
+ firedUnder: (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record) ? undefined : Object.keys(record.fired.byModel),
378
391
  recentLog: record.fired.log,
379
392
  };
380
393
  }
@@ -388,7 +401,10 @@ function toReportEntry(record) {
388
401
  function buildUsageReport(store, options) {
389
402
  const now = options.now ?? new Date();
390
403
  const limit = options.limit ?? 10;
391
- const learningRecords = Object.values(store.entries).filter((r) => r.family === 'learning');
404
+ const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
405
+ const allRecords = Object.values(store.entries).filter((r) => r.family === 'learning');
406
+ const learningRecords = allRecords.filter(r => !excluded.has(r.key));
407
+ const complete = excluded.size === 0 && (!store.evidence || store.evidence.status === 'reliable');
392
408
  const offered = learningRecords.filter((r) => r.offered.total > 0);
393
409
  const fired = learningRecords.filter((r) => firedCount(r) > 0);
394
410
  const neverFired = offered.filter((r) => firedCount(r) === 0);
@@ -397,11 +413,11 @@ function buildUsageReport(store, options) {
397
413
  generatedAt: now.toISOString(),
398
414
  windowDays: options.windowDays,
399
415
  currentModel: (0, learning_usage_projection_1.resolveCurrentModel)(options.currentModel),
400
- integrityFlagged: learningRecords.filter((r) => Boolean(r.integrity)).length,
416
+ integrityFlagged: excluded.size,
401
417
  totals: {
402
- entriesOffered: offered.length,
403
- entriesWithFirings: fired.length,
404
- entriesNeverFired: neverFired.length,
418
+ entriesOffered: complete ? offered.length : null,
419
+ entriesWithFirings: complete ? fired.length : null,
420
+ entriesNeverFired: complete ? neverFired.length : null,
405
421
  },
406
422
  workingHardest: fired
407
423
  .sort((a, b) => {
@@ -277,7 +277,8 @@ const reportCommand = addModelOption(addCommonOptions(new commander_1.Command('r
277
277
  });
278
278
  emit(report, options.json, () => {
279
279
  const lines = [
280
- `${report.totals.entriesOffered} entries were offered. ${report.totals.entriesWithFirings} fired. ${report.totals.entriesNeverFired} have never fired.`,
280
+ report.totals.entriesOffered === null ? 'Corpus totals are unavailable; listed entries have independent usable evidence.'
281
+ : `${report.totals.entriesOffered} entries were offered. ${report.totals.entriesWithFirings} fired. ${report.totals.entriesNeverFired} have never fired.`,
281
282
  ];
282
283
  if (report.workingHardest.length) {
283
284
  lines.push('', 'Working hardest');
@@ -295,7 +296,7 @@ const reportCommand = addModelOption(addCommonOptions(new commander_1.Command('r
295
296
  report.ignoredWhileInContext.forEach((e, i) => lines.push(`${i + 1}. ${e.title}. Attested ignored ${e.ignored} time${e.ignored === 1 ? '' : 's'} (fired ${e.fired}). In context and the work went against it, so the entry needs work rather than another recurrence.`));
296
297
  }
297
298
  if (report.integrityFlagged > 0) {
298
- lines.push('', `${report.integrityFlagged} entr${report.integrityFlagged === 1 ? 'y has' : 'ies have'} an integrity flag: more attested firings than recorded offers. Run \`learning-usage verify\` before acting on anything below.`);
299
+ lines.push('', `${report.integrityFlagged} entries were excluded from usage conclusions. Run \`learning-usage verify\` for diagnostic details.`);
299
300
  }
300
301
  lines.push('', 'Nothing is retired from this view. Run sleep-on-learnings to decide.');
301
302
  return lines.join('\n');
@@ -450,7 +451,8 @@ const pruneCommand = addCommonOptions(new commander_1.Command('prune'))
450
451
  const workspaceRoot = resolveWorkspaceRoot(options);
451
452
  const limits = (0, learning_usage_store_1.resolveUsageLimits)(workspaceRoot);
452
453
  const result = (0, learning_usage_store_1.pruneUsageStore)(limits);
453
- emit({ ...result, storePath: (0, learning_usage_store_1.resolveUsageStorePath)(), limits }, options.json, () => `Pruned ${result.prunedOfferRecords} offer records and ${result.prunedFiringRecords} firing records across ${result.entries} entries. Aggregates unchanged.`);
454
+ emit({ ...result, storePath: (0, learning_usage_store_1.resolveUsageStorePath)(), limits }, options.json, () => result.maintenanceStatus === 'deferred' ? `Maintenance deferred; history unchanged. ${result.evidence?.observations.join(', ') ?? ''}`
455
+ : `Pruned ${result.prunedOfferRecords} offer records and ${result.prunedFiringRecords} firing records across ${result.entries} entries. Aggregates unchanged.`);
454
456
  });
455
457
  function mb(bytes) {
456
458
  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
@@ -472,9 +474,11 @@ const verifyCommand = addModelOption(addCommonOptions(new commander_1.Command('v
472
474
  '',
473
475
  ];
474
476
  const finding = (count, singular, plural, remedy) => count === 0 ? ` none: ${singular}` : ` ${count} ${count === 1 ? singular : plural}. ${remedy}`;
477
+ if (report.evidence)
478
+ lines.push(`Evidence: ${report.evidence.status}. ${report.evidence.observations.join(', ')}.`, ...(report.evidence.recovery ? [`Recovered from ${report.evidence.recovery}; primary replacement was not persisted by this read.`] : []), '');
475
479
  lines.push('Identity', finding(report.twinPairs.length, 'lesson tracked under two keys', 'lessons tracked under two keys', 'Its offers and its firings are on different records, so every count about it is partial.'), report.unmatchedTitleKeys.length
476
480
  ? ` ${report.unmatchedTitleKeys.length} title-keyed record${report.unmatchedTitleKeys.length === 1 ? ' has' : 's have'} no id twin. Expected for entries no tier has stamped; run \`stamp-ids\` to reduce it.`
477
- : ' every title-keyed record has been matched or has no twin', '', 'Counters', finding(report.invariantViolations.length, 'entry holds more attested firings than recorded offers', 'entries hold more attested firings than recorded offers', 'An entry cannot fire without being offered, so one of the two counters is wrong. History stranded in a store copy nothing reads is the usual cause.'), finding(report.caseVariantModelBuckets.length, 'model bucket differs only by case', 'model buckets differ only by case', 'One model spelled two ways splits its own denominator, so every per-model conclusion about it is computed on a fraction of its traffic.'), finding(report.unknownModelAttribution.length, 'entry fired with no model recorded', 'entries fired with no model recorded', 'Per-model retirement cannot be answered for these, and they are held back rather than counted as zero. These were attested before record-firings carried --agent and --model, so the model was never recorded against them and a rebuild cannot invent it; attestations recorded from now on carry both.'), '', 'Storage');
481
+ : ' every title-keyed record has been matched or has no twin', '', 'Counters', finding(report.invariantViolations.length, 'entry holds more attested firings than recorded offers', 'entries hold more attested firings than recorded offers', 'The counts are inconsistent. This observation does not establish their cause.'), finding(report.caseVariantModelBuckets.length, 'model bucket differs only by case', 'model buckets differ only by case', 'One model spelled two ways splits its own denominator, so every per-model conclusion about it is computed on a fraction of its traffic.'), finding(report.unknownModelAttribution.length, 'entry has incomplete model attribution', 'entries have incomplete model attribution', 'Affected model conclusions are withheld. The cause is not established; unknown historical attribution cannot be inferred from the current model.'), '', 'Storage');
478
482
  if (report.storeCopies.length === 0) {
479
483
  lines.push(' one store file, no unmerged copies');
480
484
  }
@@ -0,0 +1,59 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.modelAttributionIncomplete = modelAttributionIncomplete;
4
+ exports.countEvidenceProblem = countEvidenceProblem;
5
+ exports.usageEvidenceExclusions = usageEvidenceExclusions;
6
+ const learning_usage_store_1 = require("./learning-usage-store");
7
+ const validCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
8
+ /** Missing attribution is unknown, including a known bucket beside unattributed history. */
9
+ function modelAttributionIncomplete(record) {
10
+ const buckets = [record.offered.byModel, record.fired.byModel];
11
+ if (buckets.some(map => !map || Object.entries(map).some(([name, n]) => !validCount(n) || !(0, learning_usage_store_1.normalizeModelName)(name) || (0, learning_usage_store_1.normalizeModelName)(name) !== name)))
12
+ return true;
13
+ return Object.values(record.offered.byModel).reduce((a, b) => a + b, 0) !== record.offered.total
14
+ || Object.values(record.fired.byModel).reduce((a, b) => a + b, 0)
15
+ !== record.fired.byOutcome.applied + record.fired.byOutcome.prevented;
16
+ }
17
+ /** Validate counts independently of a cached integrity flag. Never alter the record. */
18
+ function countEvidenceProblem(record, offersEraStart) {
19
+ const counts = [record.offered.total, record.offered.aboveThreshold, record.fired.total,
20
+ ...Object.values(record.offered.byModel), ...Object.values(record.fired.byModel),
21
+ ...Object.values(record.fired.byOutcome)];
22
+ if (counts.some(n => !validCount(n)) || learning_usage_store_1.FIRING_OUTCOMES.some(outcome => !validCount(record.fired.byOutcome[outcome]))
23
+ || record.offered.aboveThreshold > record.offered.total
24
+ || Object.values(record.fired.byOutcome).reduce((a, b) => a + b, 0) !== record.fired.total) {
25
+ return 'Usage counters are inconsistent.';
26
+ }
27
+ const violation = record.integrity ?? (0, learning_usage_store_1.integrityViolation)(record, offersEraStart);
28
+ if (!offersEraStart && record.offered.total > 0 && record.fired.total > record.offered.total) {
29
+ return 'Excess uses lack provenance establishing the offer-recording era.';
30
+ }
31
+ return violation ? violation.detail : null;
32
+ }
33
+ /** Read-time exclusions follow the actual record, never ambient UI or model state. */
34
+ function usageEvidenceExclusions(store) {
35
+ const excluded = new Map();
36
+ for (const key of store.evidence?.excludedKeys ?? [])
37
+ excluded.set(key, 'Unreadable usage record.');
38
+ const byTitle = new Map();
39
+ for (const record of Object.values(store.entries)) {
40
+ const problem = countEvidenceProblem(record, store.offersEraStart);
41
+ if (store.evidence?.status === 'unavailable')
42
+ excluded.set(record.key, 'Usage history is unavailable.');
43
+ else if (problem)
44
+ excluded.set(record.key, problem);
45
+ if (!record.fileType)
46
+ continue;
47
+ for (const title of record.titles) {
48
+ const titleKey = (0, learning_usage_store_1.titleUsageKey)(record.fileType, title);
49
+ byTitle.set(titleKey, [...new Set([...(byTitle.get(titleKey) ?? []), record.key])]);
50
+ }
51
+ }
52
+ for (const keys of byTitle.values()) {
53
+ if (keys.length > 1 && keys.some(key => key.startsWith('T:'))) {
54
+ for (const key of keys)
55
+ excluded.set(key, 'Ambiguous duplicate usage identity.');
56
+ }
57
+ }
58
+ return excluded;
59
+ }
@@ -15,6 +15,7 @@ exports.resolveCurrentModel = resolveCurrentModel;
15
15
  * Nothing here writes.
16
16
  */
17
17
  const learning_usage_store_1 = require("./learning-usage-store");
18
+ const learning_usage_eligibility_1 = require("./learning-usage-eligibility");
18
19
  function toLookupValue(record) {
19
20
  return {
20
21
  key: record.key,
@@ -24,7 +25,7 @@ function toLookupValue(record) {
24
25
  firedTotalAttestations: record.fired.total,
25
26
  lastFired: record.fired.lastFiredAt,
26
27
  lastOffered: record.offered.lastOfferedAt,
27
- firedUnder: Object.keys(record.fired.byModel),
28
+ firedUnder: (0, learning_usage_eligibility_1.modelAttributionIncomplete)(record) ? undefined : Object.keys(record.fired.byModel),
28
29
  byOutcome: record.fired.byOutcome,
29
30
  log: record.fired.log,
30
31
  };
@@ -36,7 +37,10 @@ function toLookupValue(record) {
36
37
  */
37
38
  function buildUsageLookup(store) {
38
39
  const lookup = new Map();
40
+ const excluded = (0, learning_usage_eligibility_1.usageEvidenceExclusions)(store);
39
41
  for (const record of Object.values(store.entries)) {
42
+ if (excluded.has(record.key))
43
+ continue;
40
44
  const value = toLookupValue(record);
41
45
  lookup.set(record.key, value);
42
46
  if (record.fileType) {
@@ -230,6 +230,13 @@ function normalizeModelName(model) {
230
230
  function emptyStore() {
231
231
  return { version: STORE_VERSION, updatedAt: new Date(0).toISOString(), entries: {}, foldedThroughBytes: {}, offersEraStart: null };
232
232
  }
233
+ function withEvidence(store, evidence) {
234
+ Object.defineProperty(store, 'evidence', { value: evidence, enumerable: false, configurable: true });
235
+ return store;
236
+ }
237
+ function unavailableStore(observation) {
238
+ return withEvidence(emptyStore(), { status: 'unavailable', excludedKeys: [], observations: [observation] });
239
+ }
233
240
  /**
234
241
  * Is this parsed value a record every consumer can read without crashing?
235
242
  *
@@ -254,14 +261,21 @@ function isUsableRecord(value) {
254
261
  const fired = record.fired;
255
262
  if (!offered || typeof offered !== 'object' || !fired || typeof fired !== 'object')
256
263
  return false;
257
- return Array.isArray(offered.log)
258
- && typeof offered.total === 'number'
259
- && typeof offered.aboveThreshold === 'number'
260
- && !!offered.byModel && typeof offered.byModel === 'object'
261
- && Array.isArray(fired.log)
262
- && typeof fired.total === 'number'
263
- && !!fired.byOutcome && typeof fired.byOutcome === 'object'
264
- && Array.isArray(record.titles);
264
+ const count = (v) => Number.isSafeInteger(v) && v >= 0;
265
+ const counts = (v) => !!v && typeof v === 'object' && !Array.isArray(v)
266
+ && Object.values(v).every(count);
267
+ const nullableString = (v) => v === null || typeof v === 'string';
268
+ return typeof record.key === 'string' && (record.family === 'learning' || record.family === 'rule')
269
+ && Array.isArray(offered.log) && offered.log.every(r => r && isValidDate(r.date)
270
+ && typeof r.job === 'string' && nullableString(r.model))
271
+ && count(offered.total) && count(offered.aboveThreshold)
272
+ && counts(offered.byModel)
273
+ && Array.isArray(fired.log) && fired.log.every(r => r && isValidDate(r.date)
274
+ && typeof r.job === 'string' && nullableString(r.model) && exports.FIRING_OUTCOMES.includes(r.outcome))
275
+ && count(fired.total) && counts(fired.byOutcome) && counts(fired.byModel)
276
+ && exports.FIRING_OUTCOMES.every(outcome => count(fired.byOutcome[outcome]))
277
+ && Array.isArray(fired.seenSources) && fired.seenSources.every(s => typeof s === 'string')
278
+ && Array.isArray(record.titles) && record.titles.every(t => typeof t === 'string');
265
279
  }
266
280
  /**
267
281
  * The earliest offer date a pre-#1752 snapshot can evidence.
@@ -301,15 +315,23 @@ function readCheckpoints(value) {
301
315
  }
302
316
  /** Read the normal snapshot, recovering a missing or torn copy from its checkpoint. */
303
317
  function readUsageSnapshot(storePath = resolveUsageStorePath()) {
304
- for (const candidate of [storePath, resolveUsageCheckpointPath(storePath)]) {
305
- const snapshot = parseUsageSnapshot(candidate);
306
- if (snapshot)
307
- return snapshot;
308
- }
318
+ const primary = parseUsageSnapshot(storePath);
319
+ if (primary && primary.evidence?.status !== 'unavailable')
320
+ return primary;
321
+ // An unknown schema is not evidence that an older checkpoint can replace it.
322
+ if (primary?.evidence?.observations.includes('unsupported-schema'))
323
+ return primary;
324
+ const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
325
+ if (backup && !backup.evidence)
326
+ return withEvidence(backup, {
327
+ status: 'reliable', excludedKeys: [],
328
+ observations: [(0, fs_1.existsSync)(storePath) ? 'primary-unreadable' : 'primary-missing'],
329
+ recovery: 'checkpoint', persisted: false,
330
+ });
309
331
  if ((0, fs_1.existsSync)(resolveUsageCheckpointPath(storePath))) {
310
332
  throw new Error('Usage snapshot and checkpoint backup are unreadable. Restore a matching backup; do not delete the usage directory.');
311
333
  }
312
- return emptyStore();
334
+ return primary ?? emptyStore();
313
335
  }
314
336
  function resolveUsageCheckpointPath(storePath) {
315
337
  return storePath + '.checkpoint';
@@ -324,19 +346,29 @@ function parseUsageSnapshot(storePath) {
324
346
  // `entry.offered` throws on it.
325
347
  if (!parsed || typeof parsed !== 'object' || !parsed.entries
326
348
  || typeof parsed.entries !== 'object' || Array.isArray(parsed.entries)) {
327
- return null;
349
+ return unavailableStore('malformed-snapshot');
350
+ }
351
+ if (parsed.version !== undefined && parsed.version !== STORE_VERSION)
352
+ return unavailableStore('unsupported-schema');
353
+ if (parsed.foldedThroughBytes !== undefined && (!parsed.foldedThroughBytes
354
+ || typeof parsed.foldedThroughBytes !== 'object' || Array.isArray(parsed.foldedThroughBytes)
355
+ || Object.values(parsed.foldedThroughBytes).some(v => !Number.isSafeInteger(v) || v < 0))) {
356
+ return unavailableStore('malformed-checkpoints');
328
357
  }
329
358
  // Drop any value that is not a usable record rather than handing a caller
330
359
  // something it will crash on. A partially corrupt store degrades to the records
331
360
  // that are still readable before consolidation. Consolidated snapshots must be complete.
332
361
  const entries = {};
362
+ const excludedKeys = [];
333
363
  for (const [key, value] of Object.entries(parsed.entries)) {
334
- if (isUsableRecord(value))
364
+ if (isUsableRecord(value) && value.key === key)
335
365
  entries[key] = value;
336
366
  else if (parsed.consolidated)
337
- return null;
367
+ return unavailableStore('malformed-consolidated-snapshot');
368
+ else
369
+ excludedKeys.push(key);
338
370
  }
339
- return {
371
+ const store = {
340
372
  consolidated: parsed.consolidated === true,
341
373
  version: parsed.version ?? STORE_VERSION,
342
374
  updatedAt: parsed.updatedAt ?? emptyStore().updatedAt,
@@ -346,9 +378,12 @@ function parseUsageSnapshot(storePath) {
346
378
  ? parsed.offersEraStart
347
379
  : deriveOffersEraStart(entries),
348
380
  };
381
+ return excludedKeys.length ? withEvidence(store, {
382
+ status: 'partial', excludedKeys, observations: ['malformed-records'],
383
+ }) : store;
349
384
  }
350
385
  catch {
351
- return null;
386
+ return unavailableStore('unreadable-snapshot');
352
387
  }
353
388
  }
354
389
  /**
@@ -357,9 +392,12 @@ function parseUsageSnapshot(storePath) {
357
392
  * journals, so a torn ordinary refresh can recover without replaying old history.
358
393
  */
359
394
  function writeUsageSnapshot(store, storePath = resolveUsageStorePath()) {
395
+ if (store.evidence && store.evidence.status !== 'reliable')
396
+ return false;
360
397
  try {
361
398
  (0, fs_1.mkdirSync)((0, path_1.dirname)(storePath), { recursive: true });
362
- (0, fs_1.writeFileSync)(storePath, `${JSON.stringify(store, null, 2)}\n`, { encoding: 'utf8', flush: true });
399
+ const { evidence: _evidence, ...snapshot } = store;
400
+ (0, fs_1.writeFileSync)(storePath, `${JSON.stringify(snapshot, null, 2)}\n`, { encoding: 'utf8', flush: true });
363
401
  return true;
364
402
  }
365
403
  catch {
@@ -385,7 +423,10 @@ function foldUsageEvents(options) {
385
423
  throw new Error('Usage journal no longer matches its consolidated snapshot. Restore matching checkpoint files before continuing.');
386
424
  if (tail.restarted) {
387
425
  const cold = (0, learning_usage_events_1.readEventsSince)(options.eventDir, {});
388
- return applyTail(emptyStore(), cold, limits, now, true);
426
+ return applyTail(withEvidence(emptyStore(), {
427
+ status: 'unavailable', excludedKeys: base.evidence?.excludedKeys ?? [],
428
+ observations: [...(base.evidence?.observations ?? []), 'journal-restarted'],
429
+ }), cold, limits, now, true);
389
430
  }
390
431
  return applyTail(base, tail, limits, now, false);
391
432
  }
@@ -398,12 +439,19 @@ function applyTail(base, tail, limits, now, restarted) {
398
439
  foldedThroughBytes: { ...(base.foldedThroughBytes ?? {}) },
399
440
  offersEraStart: base.offersEraStart ?? null,
400
441
  };
442
+ if (base.evidence)
443
+ withEvidence(store, base.evidence);
401
444
  let tailBytes = 0;
445
+ let unsupportedEvents = false;
402
446
  const touched = new Set();
403
447
  for (const read of tail.reads) {
404
448
  const from = restarted ? 0 : (store.foldedThroughBytes[read.machineId] ?? 0);
405
449
  tailBytes += Math.max(0, read.through - from);
406
450
  for (const event of read.records) {
451
+ if (!isSupportedUsageEvent(event)) {
452
+ unsupportedEvents = true;
453
+ continue;
454
+ }
407
455
  const key = applyEvent(store, event, limits, now);
408
456
  if (key)
409
457
  touched.add(key);
@@ -418,17 +466,63 @@ function applyTail(base, tail, limits, now, restarted) {
418
466
  // E: the invariant is evaluated on the read path, where it is always current
419
467
  // and costs the write path nothing.
420
468
  flagIntegrity(store);
469
+ const incompleteJournal = tail.skippedLines > 0
470
+ || tail.reads.some(read => read.through < read.baseOffset + read.size - read.headerBytes);
471
+ if (incompleteJournal || unsupportedEvents) {
472
+ withEvidence(store, {
473
+ status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [],
474
+ observations: [...(store.evidence?.observations ?? []),
475
+ ...(incompleteJournal ? ['incomplete-journal'] : []),
476
+ ...(unsupportedEvents ? ['unsupported-journal-events'] : [])],
477
+ });
478
+ }
421
479
  return { store, skippedLines: tail.skippedLines, restarted, tailBytes, logs: tail.reads };
422
480
  }
423
481
  /**
424
482
  * The checkpointed read every consumer uses. Snapshot plus the bytes appended
425
483
  * past it, republished once the tail is worth re-checkpointing.
426
484
  */
485
+ /** Metadata detects observed writes/sync during a read; it is not a writer lock. */
486
+ function usageReadState(storePath, eventDir) {
487
+ const dir = (0, path_1.dirname)(storePath);
488
+ const copies = (0, fs_1.existsSync)(dir) ? (0, fs_1.readdirSync)(dir)
489
+ .filter(name => /^learning-usage(-.+)?\.json$/.test(name) && name !== STORE_FILE_NAME).sort() : [];
490
+ const paths = [storePath, resolveUsageCheckpointPath(storePath),
491
+ ...(0, learning_usage_events_1.listEventLogs)(eventDir).map(log => log.path), ...copies.map(name => (0, path_1.join)(dir, name))];
492
+ const fingerprint = JSON.stringify(paths.map(file => {
493
+ try {
494
+ const stat = (0, fs_1.statSync)(file);
495
+ return [file, stat.size, stat.mtimeMs, stat.ctimeMs, stat.ino];
496
+ }
497
+ catch {
498
+ return [file, 'missing'];
499
+ }
500
+ }));
501
+ return { fingerprint, copies };
502
+ }
427
503
  function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
428
- const snapshot = readUsageSnapshot(storePath);
429
504
  const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
430
- if ((0, learning_usage_events_1.listEventLogs)(eventDir).length === 0)
431
- return snapshot;
505
+ const before = usageReadState(storePath, eventDir);
506
+ const snapshot = readUsageSnapshot(storePath);
507
+ const logs = (0, learning_usage_events_1.listEventLogs)(eventDir);
508
+ const finish = (store) => {
509
+ const after = usageReadState(storePath, eventDir);
510
+ const observations = [...(store.evidence?.observations ?? [])];
511
+ if (before.fingerprint !== after.fingerprint)
512
+ observations.push('inputs-changed-during-read');
513
+ if (after.copies.length)
514
+ observations.push('unmerged-store-copies');
515
+ if (store.consolidated && Object.keys(store.foldedThroughBytes ?? {}).some(machineId => !logs.some(log => log.machineId === machineId)))
516
+ observations.push('consolidated-journal-missing');
517
+ if (observations.length > (store.evidence?.observations.length ?? 0))
518
+ withEvidence(store, {
519
+ status: 'unavailable', excludedKeys: store.evidence?.excludedKeys ?? [], observations,
520
+ });
521
+ flagIntegrity(store);
522
+ return store;
523
+ };
524
+ if (logs.length === 0 || snapshot.evidence?.observations.includes('unsupported-schema'))
525
+ return finish(snapshot);
432
526
  const fold = (base) => foldUsageEvents({
433
527
  eventDir,
434
528
  limits: options.limits,
@@ -442,14 +536,20 @@ function readUsageStore(storePath = resolveUsageStorePath(), options = {}) {
442
536
  }
443
537
  catch (error) {
444
538
  const backup = parseUsageSnapshot(resolveUsageCheckpointPath(storePath));
445
- if (!backup)
539
+ if (!backup || backup.evidence)
446
540
  throw error;
447
- folded = fold(backup);
541
+ folded = fold(withEvidence(backup, {
542
+ status: 'reliable', excludedKeys: [], observations: ['primary-journal-mismatch'],
543
+ recovery: 'checkpoint', persisted: false,
544
+ }));
448
545
  }
546
+ finish(folded.store);
449
547
  const mode = options.refreshSnapshot ?? 'auto';
450
548
  const shouldRefresh = mode === 'always'
451
549
  || (mode === 'auto' && (folded.tailBytes >= SNAPSHOT_REFRESH_BYTES || !(0, fs_1.existsSync)(storePath) || folded.restarted));
452
- if (shouldRefresh) {
550
+ // Recovery is usable in memory. Persisting a primary replacement requires the
551
+ // explicit maintenance interval; an ordinary reader never establishes it.
552
+ if (shouldRefresh && !folded.store.evidence) {
453
553
  folded.store.updatedAt = (options.now ?? new Date()).toISOString();
454
554
  writeUsageSnapshot(folded.store, storePath);
455
555
  }
@@ -636,6 +736,20 @@ function applyEvent(store, event, limits, now) {
636
736
  void now;
637
737
  return null;
638
738
  }
739
+ function isSupportedUsageEvent(event) {
740
+ if (typeof event.key !== 'string' || !event.key || typeof event.date !== 'string' || !isValidDate(event.date))
741
+ return false;
742
+ if (event.family !== 'learning' && event.family !== 'rule')
743
+ return false;
744
+ if (typeof event.job !== 'string' || (event.model !== null && typeof event.model !== 'string'))
745
+ return false;
746
+ if (event.kind === 'offer')
747
+ return typeof event.aboveThreshold === 'boolean';
748
+ if (event.kind === 'firing')
749
+ return exports.FIRING_OUTCOMES.includes(event.outcome)
750
+ && (event.source === undefined || event.source === 'attested' || event.source === 'backfill-legacy');
751
+ return false;
752
+ }
639
753
  function applyOfferEvent(store, event, common) {
640
754
  const record = ensureRecord(store, common.key, common.family, common.fileType, common.level);
641
755
  rememberTitle(record, typeof event.title === 'string' ? event.title : undefined);
@@ -883,19 +997,17 @@ function noteTitleChange(key, newTitle, options) {
883
997
  store.updatedAt = (options.now ?? new Date()).toISOString();
884
998
  writeUsageSnapshot(store, storePath);
885
999
  }
886
- /**
887
- * Apply retention and log bounds across the whole store, and compact the event
888
- * log behind it.
889
- *
890
- * Compaction is a requirement rather than an optimization: S4 measured a cold
891
- * full fold at 84 ms on today's volume but 794 ms at ten times it, and the
892
- * checkpoint only makes a cold fold rare, not bounded.
893
- */
894
1000
  function pruneUsageStore(limits, options = {}) {
895
1001
  const storePath = options.storePath ?? resolveUsageStorePath();
896
1002
  const eventDir = options.eventDir ?? (0, learning_usage_events_1.resolveEventDir)(storePath);
897
1003
  const now = options.now ?? new Date();
898
1004
  const store = readUsageStore(storePath, { limits, now, eventDir });
1005
+ if (store.evidence && store.evidence.status !== 'reliable')
1006
+ return {
1007
+ maintenanceStatus: 'deferred', evidence: store.evidence,
1008
+ prunedOfferRecords: 0, prunedFiringRecords: 0, entries: Object.keys(store.entries).length,
1009
+ compaction: compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store }),
1010
+ };
899
1011
  let prunedOfferRecords = 0;
900
1012
  let prunedFiringRecords = 0;
901
1013
  for (const record of Object.values(store.entries)) {
@@ -906,7 +1018,10 @@ function pruneUsageStore(limits, options = {}) {
906
1018
  prunedFiringRecords += firingsBefore - record.fired.log.length;
907
1019
  }
908
1020
  const compaction = compactUsageEvents({ eventDir, snapshotPath: storePath, limits, now, store });
909
- return { prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length, compaction };
1021
+ return {
1022
+ maintenanceStatus: compaction.maintenanceStatus, evidence: compaction.evidence,
1023
+ prunedOfferRecords, prunedFiringRecords, entries: Object.keys(store.entries).length, compaction,
1024
+ };
910
1025
  }
911
1026
  /** Save totals and source bookmarks in both copies, then clear processed journals.
912
1027
  * Requires an interval without concurrent writers or sync activity.
@@ -915,6 +1030,14 @@ function compactUsageEvents(options) {
915
1030
  const now = options.now ?? new Date();
916
1031
  const store = options.store
917
1032
  ?? readUsageStore(options.snapshotPath, { limits: options.limits, now, eventDir: options.eventDir });
1033
+ // Do not turn a partial read into a durable replacement or advance bookmarks
1034
+ // over evidence this reader could not understand. Independent review can run.
1035
+ if (store.evidence && store.evidence.status !== 'reliable')
1036
+ return {
1037
+ maintenanceStatus: 'deferred', evidence: store.evidence,
1038
+ scanned: 0, kept: 0, dropped: 0, skippedLines: 0, logs: [],
1039
+ checkpoints: { ...(store.foldedThroughBytes ?? {}) },
1040
+ };
918
1041
  for (const log of (0, learning_usage_events_1.listEventLogs)(options.eventDir)) {
919
1042
  if ((store.foldedThroughBytes?.[log.machineId] ?? 0) < log.baseOffset) {
920
1043
  throw new Error('Cannot consolidate from a snapshot older than the journal checkpoint.');
@@ -931,7 +1054,8 @@ function compactUsageEvents(options) {
931
1054
  const temp = checkpoint + '.tmp';
932
1055
  try {
933
1056
  (0, fs_1.mkdirSync)((0, path_1.dirname)(checkpoint), { recursive: true });
934
- (0, fs_1.writeFileSync)(temp, JSON.stringify(store, null, 2) + '\n', { encoding: 'utf8', flush: true });
1057
+ const { evidence: _evidence, ...snapshot } = store;
1058
+ (0, fs_1.writeFileSync)(temp, JSON.stringify(snapshot, null, 2) + '\n', { encoding: 'utf8', flush: true });
935
1059
  (0, fs_1.renameSync)(temp, checkpoint);
936
1060
  }
937
1061
  catch {
@@ -944,7 +1068,7 @@ function compactUsageEvents(options) {
944
1068
  if (!writeUsageSnapshot(store, options.snapshotPath)) {
945
1069
  throw new Error('Cannot consolidate usage events: aggregate snapshot could not be saved.');
946
1070
  }
947
- return (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
1071
+ const compaction = (0, learning_usage_events_1.compactEventLogs)(options.eventDir, (event) => {
948
1072
  if (event.kind !== 'offer' && event.kind !== 'firing')
949
1073
  return true;
950
1074
  const date = typeof event.date === 'string' ? event.date : '';
@@ -952,4 +1076,5 @@ function compactUsageEvents(options) {
952
1076
  return true;
953
1077
  return event.kind === 'firing' && !exports.FIRING_OUTCOMES.includes(event.outcome);
954
1078
  }, store.foldedThroughBytes ?? {});
1079
+ return { ...compaction, maintenanceStatus: 'completed', evidence: store.evidence };
955
1080
  }