agent-recall-core 3.4.38 → 3.4.39

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +1 -1
  2. package/dist/host-profile.d.ts +69 -0
  3. package/dist/host-profile.d.ts.map +1 -0
  4. package/dist/host-profile.js +107 -0
  5. package/dist/host-profile.js.map +1 -0
  6. package/dist/index.d.ts +7 -2
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +11 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/storage/corrections.d.ts +128 -0
  11. package/dist/storage/corrections.d.ts.map +1 -1
  12. package/dist/storage/corrections.js +562 -147
  13. package/dist/storage/corrections.js.map +1 -1
  14. package/dist/storage/lifecycle-telemetry.d.ts +56 -0
  15. package/dist/storage/lifecycle-telemetry.d.ts.map +1 -0
  16. package/dist/storage/lifecycle-telemetry.js +134 -0
  17. package/dist/storage/lifecycle-telemetry.js.map +1 -0
  18. package/dist/storage/sanitize.d.ts +2 -1
  19. package/dist/storage/sanitize.d.ts.map +1 -1
  20. package/dist/storage/sanitize.js +19 -2
  21. package/dist/storage/sanitize.js.map +1 -1
  22. package/dist/storage/session.d.ts +25 -1
  23. package/dist/storage/session.d.ts.map +1 -1
  24. package/dist/storage/session.js +60 -1
  25. package/dist/storage/session.js.map +1 -1
  26. package/dist/tools-logic/check-action.d.ts.map +1 -1
  27. package/dist/tools-logic/check-action.js +97 -3
  28. package/dist/tools-logic/check-action.js.map +1 -1
  29. package/dist/tools-logic/check.d.ts +22 -0
  30. package/dist/tools-logic/check.d.ts.map +1 -1
  31. package/dist/tools-logic/check.js +31 -1
  32. package/dist/tools-logic/check.js.map +1 -1
  33. package/dist/tools-logic/session-end.d.ts.map +1 -1
  34. package/dist/tools-logic/session-end.js +49 -4
  35. package/dist/tools-logic/session-end.js.map +1 -1
  36. package/dist/tools-logic/session-start.d.ts.map +1 -1
  37. package/dist/tools-logic/session-start.js +29 -6
  38. package/dist/tools-logic/session-start.js.map +1 -1
  39. package/dist/tools-logic/smart-recall.d.ts +93 -1
  40. package/dist/tools-logic/smart-recall.d.ts.map +1 -1
  41. package/dist/tools-logic/smart-recall.js +199 -11
  42. package/dist/tools-logic/smart-recall.js.map +1 -1
  43. package/dist/tools-logic/smart-remember.d.ts.map +1 -1
  44. package/dist/tools-logic/smart-remember.js +12 -0
  45. package/dist/tools-logic/smart-remember.js.map +1 -1
  46. package/dist/tools-logic/store-doctor.d.ts +1 -1
  47. package/dist/tools-logic/store-doctor.d.ts.map +1 -1
  48. package/dist/tools-logic/store-doctor.js +64 -0
  49. package/dist/tools-logic/store-doctor.js.map +1 -1
  50. package/dist/types.d.ts +1 -1
  51. package/dist/types.js +1 -1
  52. package/package.json +4 -3
  53. package/dist/tools-logic/brief.d.ts +0 -84
  54. package/dist/tools-logic/brief.d.ts.map +0 -1
  55. package/dist/tools-logic/brief.js +0 -193
  56. package/dist/tools-logic/brief.js.map +0 -1
  57. package/dist/tools-logic/dashboard-export.d.ts +0 -130
  58. package/dist/tools-logic/dashboard-export.d.ts.map +0 -1
  59. package/dist/tools-logic/dashboard-export.js +0 -485
  60. package/dist/tools-logic/dashboard-export.js.map +0 -1
  61. package/dist/tools-logic/memory-query.d.ts +0 -43
  62. package/dist/tools-logic/memory-query.d.ts.map +0 -1
  63. package/dist/tools-logic/memory-query.js +0 -77
  64. package/dist/tools-logic/memory-query.js.map +0 -1
  65. package/dist/tools-logic/project-status.d.ts +0 -24
  66. package/dist/tools-logic/project-status.d.ts.map +0 -1
  67. package/dist/tools-logic/project-status.js +0 -155
  68. package/dist/tools-logic/project-status.js.map +0 -1
@@ -8,8 +8,9 @@ import * as fs from "node:fs";
8
8
  import * as path from "node:path";
9
9
  import * as crypto from "node:crypto";
10
10
  import { ensureDir } from "./fs-utils.js";
11
- import { sanitizeName } from "./sanitize.js";
11
+ import { byteCap, sanitizeName } from "./sanitize.js";
12
12
  import { journalDir, projectSubPath } from "./paths.js";
13
+ import { withLock } from "./filelock.js";
13
14
  // ---------------------------------------------------------------------------
14
15
  // Internal helpers
15
16
  // ---------------------------------------------------------------------------
@@ -117,6 +118,13 @@ function slugify(text) {
117
118
  * don't dedupe by id — every *.json file is a distinct record). Returns null
118
119
  * when no matching file is found (defensive fallback only; the record was
119
120
  * just read from this same directory by every current caller).
121
+ *
122
+ * KNOWN LIMITATION of the null-fallback recompute at the call sites: it
123
+ * produces the PLAIN `${date}--${slugify(rule)}.json` name and does NOT carry
124
+ * the id-hash suffix a collision-disambiguated record was originally written
125
+ * under (see writeCorrection's brand-new branch). Reachable only if a
126
+ * record's backing file vanishes from disk mid-call, inside the lock —
127
+ * defensive-only today, flagged by review 2026-07-27.
120
128
  */
121
129
  function findExistingCorrectionFile(dir, id) {
122
130
  if (!fs.existsSync(dir))
@@ -551,64 +559,98 @@ export function writeCorrection(project, correction) {
551
559
  // Auto-detect severity if not already set
552
560
  const severity = correction.severity ?? detectSeverity(`${correction.rule} ${correction.context}`);
553
561
  const record = applyCorrectionDefaults({ ...correction, severity }, todayDate());
554
- // ── P1: on-write consolidation (refine-not-overwrite) ─────────────────────
555
- // Borrow Hindsight's consolidation idea, AR-native: instead of accumulating a
556
- // new dated file for a re-stated rule, fold it into the most similar ACTIVE
557
- // correction of the SAME kind and bump that record's proof_count. The matched
558
- // record keeps its id/date (stable document_id) and absorbs the new tags +
559
- // higher severity/authority/weight. High-precision LOCAL gate — no key, no
560
- // network — so this never runs an LLM on the storage hot path.
561
- const normNew = normalizeRule(record.rule);
562
- for (const existing of readActiveCorrections(project)) {
563
- if (existing.id === record.id)
564
- continue; // never merge into self (same-day re-slug)
565
- if ((existing.kind ?? "correction") !== (record.kind ?? "correction"))
566
- continue;
567
- if (normalizeRule(existing.rule) !== normNew)
568
- continue;
569
- const merged = {
570
- ...existing,
571
- proof_count: (existing.proof_count ?? 1) + 1,
572
- merged_from: [...(existing.merged_from ?? []), record.id],
573
- tags: Array.from(new Set([...(existing.tags ?? []), ...(record.tags ?? [])])),
574
- // keep the STRONGER signal on every axis
575
- severity: existing.severity === "p0" || record.severity === "p0" ? "p0" : "p1",
576
- weight: Math.max(existing.weight ?? 0, record.weight ?? 0),
577
- authoritative: Boolean(existing.authoritative || record.authoritative),
578
- last_outcome: new Date().toISOString(),
579
- // RD-1: keep the existing classification; adopt the incoming capture's
580
- // class only when the existing record predates the field AND the
581
- // incoming class is a REAL class. Review fix MEDIUM-1 (2026-07-14):
582
- // writing "other" into a pre-RD-1 file changes nothing at read time
583
- // (absent already reads as other) but permanently forecloses a future
584
- // real classification — stored-wins would keep "other" forever. Note the
585
- // merge gate is rule-text equality while classification runs on the full
586
- // context text, so incoming classes CAN differ across merges; stored
587
- // still wins in that case by design (durable classification).
588
- ...(existing.failure_class
589
- ? { failure_class: existing.failure_class }
590
- : record.failure_class && record.failure_class !== "other"
591
- ? { failure_class: record.failure_class }
592
- : {}),
593
- };
594
- // Rewrite: reuse the EXISTING file's name (never recompute via slugify —
595
- // see findExistingCorrectionFile doc). Falls back to a fresh v2 filename
596
- // only in the defensive case where the file has vanished mid-call.
597
- const mfile = findExistingCorrectionFile(dir, existing.id)
598
- ?? `${merged.date}--${slugify(merged.rule || merged.id)}.json`;
599
- writeRecordAtomic(path.join(dir, mfile), merged);
562
+ // ── LOCKED critical section (P0 data-loss fix, 2026-07-25) ────────────────
563
+ // From here on this is an unlocked read-all→find→mutate→atomic-rewrite→
564
+ // index-regen sequence: the consolidation scan below reads every *.json
565
+ // file, the merge branch rewrites the ONE matched record, and either branch
566
+ // ends by calling regenerateCorrectionsIndex, which re-reads the whole store
567
+ // and rewrites the ONE shared _index.md. Unlocked, two concurrent
568
+ // writeCorrection calls for the same project can (a) both miss each other's
569
+ // match and create duplicate un-merged records instead of consolidating, or
570
+ // (b) race on the shared index rewrite. Locked with `corrections-${project}`
571
+ // — PROJECT-scoped, not per-correction-id, because the index file is shared
572
+ // across every correction in the project (a per-id lock would still let two
573
+ // DIFFERENT corrections' index writes race). Mirrors retractCorrection /
574
+ // recordOutcome below and the existing `palace-index-${project}` /
575
+ // `digest-${project}` lock pattern elsewhere in this codebase.
576
+ return withLock(`corrections-${project}`, () => {
577
+ // ── P1: on-write consolidation (refine-not-overwrite) ─────────────────────
578
+ // Borrow Hindsight's consolidation idea, AR-native: instead of accumulating a
579
+ // new dated file for a re-stated rule, fold it into the most similar ACTIVE
580
+ // correction of the SAME kind and bump that record's proof_count. The matched
581
+ // record keeps its id/date (stable document_id) and absorbs the new tags +
582
+ // higher severity/authority/weight. High-precision LOCAL gate — no key, no
583
+ // network — so this never runs an LLM on the storage hot path.
584
+ const normNew = normalizeRule(record.rule);
585
+ for (const existing of readActiveCorrections(project)) {
586
+ if (existing.id === record.id)
587
+ continue; // never merge into self (same-day re-slug)
588
+ if ((existing.kind ?? "correction") !== (record.kind ?? "correction"))
589
+ continue;
590
+ if (normalizeRule(existing.rule) !== normNew)
591
+ continue;
592
+ const merged = {
593
+ ...existing,
594
+ proof_count: (existing.proof_count ?? 1) + 1,
595
+ merged_from: [...(existing.merged_from ?? []), record.id],
596
+ tags: Array.from(new Set([...(existing.tags ?? []), ...(record.tags ?? [])])),
597
+ // keep the STRONGER signal on every axis
598
+ severity: existing.severity === "p0" || record.severity === "p0" ? "p0" : "p1",
599
+ weight: Math.max(existing.weight ?? 0, record.weight ?? 0),
600
+ authoritative: Boolean(existing.authoritative || record.authoritative),
601
+ last_outcome: new Date().toISOString(),
602
+ // RD-1: keep the existing classification; adopt the incoming capture's
603
+ // class only when the existing record predates the field AND the
604
+ // incoming class is a REAL class. Review fix MEDIUM-1 (2026-07-14):
605
+ // writing "other" into a pre-RD-1 file changes nothing at read time
606
+ // (absent already reads as other) but permanently forecloses a future
607
+ // real classification — stored-wins would keep "other" forever. Note the
608
+ // merge gate is rule-text equality while classification runs on the full
609
+ // context text, so incoming classes CAN differ across merges; stored
610
+ // still wins in that case by design (durable classification).
611
+ ...(existing.failure_class
612
+ ? { failure_class: existing.failure_class }
613
+ : record.failure_class && record.failure_class !== "other"
614
+ ? { failure_class: record.failure_class }
615
+ : {}),
616
+ };
617
+ // Rewrite: reuse the EXISTING file's name (never recompute via slugify —
618
+ // see findExistingCorrectionFile doc). Falls back to a fresh v2 filename
619
+ // only in the defensive case where the file has vanished mid-call.
620
+ const mfile = findExistingCorrectionFile(dir, existing.id)
621
+ ?? `${merged.date}--${slugify(merged.rule || merged.id)}.json`;
622
+ writeRecordAtomic(path.join(dir, mfile), merged);
623
+ // W2-1: regenerate the materialized index on every corrections mutation.
624
+ regenerateCorrectionsIndex(project);
625
+ return { written: true, merged: true, id: merged.id };
626
+ }
627
+ // Brand-new record — no existing file to preserve. v2 delimiter ("--").
628
+ let filename = `${record.date}--${slugify(record.rule || record.id)}.json`;
629
+ let filepath = path.join(dir, filename);
630
+ if (fs.existsSync(filepath)) {
631
+ // Same-day slug collision with a DIFFERENT record: the active-rule merge
632
+ // loop above already claimed every same-rule case, so whatever lives at
633
+ // this path is a distinct rule whose slug happens to coincide (two
634
+ // CJK-heavy rules sharing one surviving Latin word, or a RETRACTED
635
+ // record the active-only merge scan skipped). Overwriting would silently
636
+ // destroy it — both callers would still see { written: true }. See
637
+ // test/cjk-slug-collision.test.mjs. Disambiguate inside the slug field
638
+ // with a short deterministic hash of this record's id: hex + single "-"
639
+ // join keeps the "--" field-delimiter grammar intact, and re-running the
640
+ // same write lands on the same name.
641
+ const idHash = crypto.createHash("sha256").update(record.id).digest("hex").slice(0, 8);
642
+ // 39 = 48 (corrections' slug byte budget) - 1 ("-" join) - 8 (hex hash):
643
+ // the disambiguated slug field stays inside the same 48-byte envelope.
644
+ const baseSlug = byteCap(slugify(record.rule || record.id), 39).replace(/-+$/g, "");
645
+ filename = `${record.date}--${baseSlug}-${idHash}.json`;
646
+ filepath = path.join(dir, filename);
647
+ }
648
+ // Atomic write — tmp + rename, mode 0600
649
+ writeRecordAtomic(filepath, record);
600
650
  // W2-1: regenerate the materialized index on every corrections mutation.
601
651
  regenerateCorrectionsIndex(project);
602
- return { written: true, merged: true, id: merged.id };
603
- }
604
- // Brand-new record — no existing file to preserve. v2 delimiter ("--").
605
- const filename = `${record.date}--${slugify(record.rule || record.id)}.json`;
606
- const filepath = path.join(dir, filename);
607
- // Atomic write — tmp + rename, mode 0600
608
- writeRecordAtomic(filepath, record);
609
- // W2-1: regenerate the materialized index on every corrections mutation.
610
- regenerateCorrectionsIndex(project);
611
- return { written: true, merged: false, id: record.id };
652
+ return { written: true, merged: false, id: record.id };
653
+ });
612
654
  }
613
655
  /**
614
656
  * Read all corrections for a project, sorted newest first.
@@ -654,30 +696,38 @@ export function readP0Corrections(project) {
654
696
  */
655
697
  export function retractCorrection(project, id, reason, supersededBy) {
656
698
  const dir = correctionsDir(project);
657
- // Find the correction record by id
658
- const all = readCorrections(project);
659
- const target = all.find((r) => r.id === id);
660
- if (!target) {
661
- return { success: false, error: `correction not found: ${id}` };
662
- }
663
- const updated = {
664
- ...target,
665
- active: false,
666
- retracted_at: new Date().toISOString(),
667
- ...(reason !== undefined ? { retract_reason: reason } : {}),
668
- // P2: forward pointer to the correction that replaced this one (audit trail).
669
- ...(supersededBy !== undefined ? { superseded_by: supersededBy } : {}),
670
- };
671
- // Rewrite — reuse the EXISTING file's name (see findExistingCorrectionFile
672
- // doc); never recompute via slugify, or a v1-named file would orphan.
673
- const filename = findExistingCorrectionFile(dir, updated.id)
674
- ?? `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
675
- const filepath = path.join(dir, filename);
676
- // Atomic rewrite — tmp + rename, mode 0600
677
- writeRecordAtomic(filepath, updated);
678
- // W2-1: regenerate the materialized index on every corrections mutation.
679
- regenerateCorrectionsIndex(project);
680
- return { success: true, id };
699
+ // ── LOCKED critical section (P0 data-loss fix, 2026-07-25) ────────────────
700
+ // read-all→find→mutate→atomic-rewrite→index-regen, unlocked, is exactly the
701
+ // race class this fix closes — see the matching note in writeCorrection
702
+ // above and recordOutcome below. Locked with `corrections-${project}`
703
+ // (PROJECT-scoped, matching regenerateCorrectionsIndex's one-shared-file-
704
+ // per-project shape; not per-correction-id — see writeCorrection's note).
705
+ return withLock(`corrections-${project}`, () => {
706
+ // Find the correction record by id
707
+ const all = readCorrections(project);
708
+ const target = all.find((r) => r.id === id);
709
+ if (!target) {
710
+ return { success: false, error: `correction not found: ${id}` };
711
+ }
712
+ const updated = {
713
+ ...target,
714
+ active: false,
715
+ retracted_at: new Date().toISOString(),
716
+ ...(reason !== undefined ? { retract_reason: reason } : {}),
717
+ // P2: forward pointer to the correction that replaced this one (audit trail).
718
+ ...(supersededBy !== undefined ? { superseded_by: supersededBy } : {}),
719
+ };
720
+ // Rewrite — reuse the EXISTING file's name (see findExistingCorrectionFile
721
+ // doc); never recompute via slugify, or a v1-named file would orphan.
722
+ const filename = findExistingCorrectionFile(dir, updated.id)
723
+ ?? `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
724
+ const filepath = path.join(dir, filename);
725
+ // Atomic rewrite — tmp + rename, mode 0600
726
+ writeRecordAtomic(filepath, updated);
727
+ // W2-1: regenerate the materialized index on every corrections mutation.
728
+ regenerateCorrectionsIndex(project);
729
+ return { success: true, id };
730
+ });
681
731
  }
682
732
  /**
683
733
  * Record an outcome event for a correction (retrieved / heeded / recurred).
@@ -720,78 +770,443 @@ export function recordOutcome(outcome) {
720
770
  outcome.kind === "unknown") {
721
771
  return;
722
772
  }
723
- // Update the per-correction file's counters.
724
- const target = readCorrections(outcome.project).find((r) => r.id === outcome.correction_id);
725
- if (!target)
726
- return; // outcome can still be replayed later if record is restored
727
- const updated = {
728
- ...target,
729
- retrieved_count: target.retrieved_count ?? 0,
730
- heeded_count: target.heeded_count ?? 0,
731
- recurrence_count: target.recurrence_count ?? 0,
773
+ // ── LOCKED critical section (P0 data-loss fix, 2026-07-25) ────────────────
774
+ // This is the exact read-all→find→mutate→atomic-rewrite→index-regen span the
775
+ // audit reproduced (24 procs × 20 calls losing 66-78% of retrieved_count
776
+ // increments): readCorrections() below re-reads every *.json file, so two
777
+ // concurrent processes can both read the SAME stale counter value, both
778
+ // increment it by one, and the last atomic rewrite to land silently discards
779
+ // the other's increment. Locked with `corrections-${project}` — PROJECT-
780
+ // scoped, not per-correction-id, because regenerateCorrectionsIndex rewrites
781
+ // ONE shared _index.md per project (a per-id lock would still let two
782
+ // DIFFERENT corrections' index writes race). See the matching note in
783
+ // writeCorrection/retractCorrection above; same lock name, same pattern as
784
+ // `palace-index-${project}` / `digest-${project}` elsewhere in this codebase.
785
+ withLock(`corrections-${outcome.project}`, () => {
786
+ // Update the per-correction file's counters.
787
+ const target = readCorrections(outcome.project).find((r) => r.id === outcome.correction_id);
788
+ if (!target)
789
+ return; // outcome can still be replayed later if record is restored
790
+ const updated = {
791
+ ...target,
792
+ retrieved_count: target.retrieved_count ?? 0,
793
+ heeded_count: target.heeded_count ?? 0,
794
+ recurrence_count: target.recurrence_count ?? 0,
795
+ };
796
+ if (outcome.kind === "retrieved") {
797
+ updated.retrieved_count = (updated.retrieved_count ?? 0) + 1;
798
+ updated.last_retrieved = outcome.at;
799
+ }
800
+ else if (outcome.kind === "heeded") {
801
+ updated.heeded_count = (updated.heeded_count ?? 0) + 1;
802
+ updated.last_outcome = outcome.at;
803
+ }
804
+ else if (outcome.kind === "recurred") {
805
+ updated.recurrence_count = (updated.recurrence_count ?? 0) + 1;
806
+ updated.last_outcome = outcome.at;
807
+ }
808
+ else if (outcome.kind === "predicted") {
809
+ // Wave 5: prediction fired — instrument the predict-the-correction loop.
810
+ updated.predicted_count = (updated.predicted_count ?? 0) + 1;
811
+ updated.last_predicted = outcome.at;
812
+ }
813
+ else if (outcome.kind === "predict_hit") {
814
+ updated.predict_hits = (updated.predict_hits ?? 0) + 1;
815
+ }
816
+ const r = updated.retrieved_count ?? 0;
817
+ // Clamp to [0,1]: `retrieved` is guarded 1/day but `heeded` can fire on every
818
+ // session_end, so raw heeded/retrieved can exceed 1.0 ("150% heeded" is
819
+ // nonsense). min(1, …) keeps the metric honest. (Root-cause follow-up: apply
820
+ // the same 1/day guard to heeded as retrieved has, for finer resolution.)
821
+ // NB (Wave 5): `precision` is heeded/retrieved ONLY — predict_* never touch it.
822
+ updated.precision = r > 0 ? Math.min(1, Number(((updated.heeded_count ?? 0) / r).toFixed(3))) : undefined;
823
+ // Wave 5: predict_precision = predict_hits / predicted_count, kept SEPARATE
824
+ // from `precision`. Undefined until at least one prediction has fired.
825
+ // A predict_hit implies a prior prediction. If data is inconsistent (hits
826
+ // recorded without a matching predicted_count — e.g. migrated/corrupt records),
827
+ // floor the denominator at predict_hits so the metric stays VISIBLE and bounded
828
+ // rather than silently undefined while hits exist.
829
+ const pc = updated.predicted_count ?? 0;
830
+ const ph = updated.predict_hits ?? 0;
831
+ const predictDenom = Math.max(pc, ph);
832
+ updated.predict_precision = predictDenom > 0
833
+ ? Math.min(1, Number((ph / predictDenom).toFixed(3)))
834
+ : undefined;
835
+ // P3: evidence-grounded proof_confidence. With NO outcome evidence yet, keep the
836
+ // authority prior (weight); once heeded/recurrence accrue, move to the Beta
837
+ // posterior so a rule that keeps being honored strengthens and one whose bug
838
+ // keeps recurring weakens. Kept SEPARATE from `precision` (heeded/retrieved) and
839
+ // from `weight` (static authority) — this is the evidence axis.
840
+ const heededC = updated.heeded_count ?? 0;
841
+ const recurC = updated.recurrence_count ?? 0;
842
+ updated.proof_confidence = (heededC + recurC) > 0
843
+ ? Number(betaPosterior(heededC, recurC).toFixed(3))
844
+ : (updated.weight ?? defaultWeight(updated.severity));
845
+ // Re-write the JSON file atomically (tmp + rename — prevents truncation on
846
+ // SIGTERM). Reuse the EXISTING filename (see findExistingCorrectionFile
847
+ // doc) — never recompute via slugify, or a v1-named file would orphan.
848
+ const filename = findExistingCorrectionFile(dir, updated.id)
849
+ ?? `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
850
+ const filepath = path.join(dir, filename);
851
+ writeRecordAtomic(filepath, updated);
852
+ // W2-1: regenerate the materialized index on every corrections mutation.
853
+ // (The ledger-only early-returns above for triggered/not_triggered/unknown
854
+ // touch no per-record field the index renders — severity/status/date/rule/
855
+ // failure_class are all unchanged — so they deliberately skip this call,
856
+ // preserving the existing hot-path optimization documented above.)
857
+ regenerateCorrectionsIndex(outcome.project);
858
+ });
859
+ }
860
+ /** Outcome kinds that affect per-correction counters (mirrors recordOutcome's early-return for triggered/not_triggered/unknown — those are ledger-only). */
861
+ const COUNTER_KINDS = new Set([
862
+ "retrieved",
863
+ "heeded",
864
+ "recurred",
865
+ "predicted",
866
+ "predict_hit",
867
+ ]);
868
+ /**
869
+ * Replay-order key for one ledger event.
870
+ *
871
+ * `recorded_at` (forensic, C3b) is the wall-clock instant recordOutcome() was
872
+ * actually CALLED — i.e. it IS the live call order. `at` (semantic) can be
873
+ * backdated by the dream-audit path, so two events can have `at` values out of
874
+ * order relative to when they were really recorded (a nightly dream-audit call
875
+ * backdates `at` to yesterday but is physically called well AFTER today's live
876
+ * calls). recordOutcome has NO "is this newer" guard — every relevant call
877
+ * unconditionally OVERWRITES last_retrieved/last_outcome/last_predicted with
878
+ * `outcome.at`. So the only way a replay reproduces bit-identical results to a
879
+ * lossless LIVE call sequence is to apply events in the order the CALLS
880
+ * happened (recorded_at), each one stamping its own `.at` value — NOT in
881
+ * semantic-day order. Falls back to `at` only for pre-C3b lines that lack
882
+ * `recorded_at` entirely: those lines predate the dream-audit backdating
883
+ * feature (C3b introduced both `recorded_at` AND backdating together), so for
884
+ * them `at` already WAS the wall-clock time — a safe, non-arbitrary fallback.
885
+ */
886
+ function replayOrderKey(evt) {
887
+ const ts = evt.recorded_at ?? evt.at;
888
+ const t = new Date(ts).getTime();
889
+ return Number.isNaN(t) ? 0 : t;
890
+ }
891
+ /**
892
+ * Sort events into live-call order for replay. Array.prototype.sort is stable
893
+ * (guaranteed since ES2019 / all supported Node versions), so ties — identical
894
+ * recorded_at, or both timestamps missing/invalid — keep original ledger line
895
+ * order, the next-best tie-break absent finer-grained sequencing.
896
+ */
897
+ function sortForReplay(events) {
898
+ return [...events]
899
+ .map((evt, idx) => ({ evt, idx }))
900
+ .sort((a, b) => replayOrderKey(a.evt) - replayOrderKey(b.evt) || a.idx - b.idx)
901
+ .map(({ evt }) => evt);
902
+ }
903
+ /**
904
+ * Pure replay of one correction's counter-affecting ledger events, already
905
+ * ordered by `sortForReplay`. Recomputes every field recordOutcome derives,
906
+ * applying the EXACT SAME per-event formulas (see recordOutcome, this file,
907
+ * ~L1032-1084) one event at a time — this is what makes the rebuild produce
908
+ * IDENTICAL results to a lossless live call sequence.
909
+ *
910
+ * `record` supplies ONLY the weight/severity fallback `proof_confidence` needs
911
+ * when no heeded/recurrence evidence exists yet (recordOutcome's own
912
+ * `updated.weight ?? defaultWeight(updated.severity)` fallback, L1084) — its
913
+ * counter fields are never read here; those are recomputed from scratch.
914
+ *
915
+ * Mirrors recordOutcome's asymmetry precisely: retrieved_count/heeded_count/
916
+ * recurrence_count are seeded to 0 as soon as ANY counter-kind event fires
917
+ * (recordOutcome's `{...target, retrieved_count: target.retrieved_count ?? 0, …}`
918
+ * runs unconditionally before the per-kind branch, L1032-1037), while
919
+ * predicted_count/predict_hits stay `undefined` until their OWN specific kind
920
+ * fires at least once (they are only ever seeded inline at their own
921
+ * increment site, L1049/L1052).
922
+ */
923
+ export function recomputeCorrectionCounters(events, record) {
924
+ let retrieved;
925
+ let heeded;
926
+ let recurrence;
927
+ let predicted;
928
+ let predictHits;
929
+ let lastRetrieved;
930
+ let lastOutcome;
931
+ let lastPredicted;
932
+ for (const evt of events) {
933
+ if (!COUNTER_KINDS.has(evt.kind))
934
+ continue; // ledger-only — mirror recordOutcome's early return (L1007-1013)
935
+ // Unconditional baseline seed on EVERY counter-kind event (recordOutcome L1032-1037).
936
+ retrieved = retrieved ?? 0;
937
+ heeded = heeded ?? 0;
938
+ recurrence = recurrence ?? 0;
939
+ if (evt.kind === "retrieved") {
940
+ retrieved = retrieved + 1;
941
+ lastRetrieved = evt.at;
942
+ }
943
+ else if (evt.kind === "heeded") {
944
+ heeded = heeded + 1;
945
+ lastOutcome = evt.at;
946
+ }
947
+ else if (evt.kind === "recurred") {
948
+ recurrence = recurrence + 1;
949
+ lastOutcome = evt.at;
950
+ }
951
+ else if (evt.kind === "predicted") {
952
+ predicted = (predicted ?? 0) + 1;
953
+ lastPredicted = evt.at;
954
+ }
955
+ else if (evt.kind === "predict_hit") {
956
+ predictHits = (predictHits ?? 0) + 1;
957
+ }
958
+ }
959
+ // precision — recordOutcome L1054/L1060.
960
+ const r = retrieved ?? 0;
961
+ const precision = r > 0 ? Math.min(1, Number(((heeded ?? 0) / r).toFixed(3))) : undefined;
962
+ // predict_precision — recordOutcome L1068-1073 (denominator floored at
963
+ // predict_hits when predicted_count is inconsistent/missing).
964
+ const pc = predicted ?? 0;
965
+ const ph = predictHits ?? 0;
966
+ const predictDenom = Math.max(pc, ph);
967
+ const predictPrecision = predictDenom > 0 ? Math.min(1, Number((ph / predictDenom).toFixed(3))) : undefined;
968
+ // proof_confidence — recordOutcome L1080-1084.
969
+ const heededC = heeded ?? 0;
970
+ const recurC = recurrence ?? 0;
971
+ const proofConfidence = heededC + recurC > 0
972
+ ? Number(betaPosterior(heededC, recurC).toFixed(3))
973
+ : (record.weight ?? defaultWeight(record.severity));
974
+ return {
975
+ retrieved_count: retrieved,
976
+ heeded_count: heeded,
977
+ recurrence_count: recurrence,
978
+ predicted_count: predicted,
979
+ predict_hits: predictHits,
980
+ last_retrieved: lastRetrieved,
981
+ last_outcome: lastOutcome,
982
+ last_predicted: lastPredicted,
983
+ precision,
984
+ predict_precision: predictPrecision,
985
+ proof_confidence: proofConfidence,
732
986
  };
733
- if (outcome.kind === "retrieved") {
734
- updated.retrieved_count = (updated.retrieved_count ?? 0) + 1;
735
- updated.last_retrieved = outcome.at;
987
+ }
988
+ /**
989
+ * Bulk-read + split _outcomes.jsonl into per-correction event lists, quarantining
990
+ * malformed/shapeless lines instead of silently dropping them (unlike
991
+ * bucketOutcomesBy above, whose callers only need coarse kind-presence and
992
+ * already tolerate silent skips).
993
+ *
994
+ * BULK READ, not streamed: this mirrors bucketOutcomesBy's existing pattern in
995
+ * this exact file (fs.readFileSync + split("\n")) and keeps the doctor check
996
+ * that shares this function fully synchronous — store-doctor.ts's public API
997
+ * (runStoreDoctor) is synchronous today and is called from the session_start
998
+ * hot path; switching to a streaming reader would force it (and every caller)
999
+ * async, a much wider blast radius than this repair tool's scope. Acceptable
1000
+ * per-project: _outcomes.jsonl is one project's correction-outcome history,
1001
+ * not a global log.
1002
+ */
1003
+ function parseOutcomesLedger(project) {
1004
+ const malformedRows = [];
1005
+ const eventsByCorrectionId = new Map();
1006
+ const p = outcomesPath(project);
1007
+ if (!fs.existsSync(p))
1008
+ return { malformedRows, eventsByCorrectionId };
1009
+ let raw;
1010
+ try {
1011
+ raw = fs.readFileSync(p, "utf-8");
736
1012
  }
737
- else if (outcome.kind === "heeded") {
738
- updated.heeded_count = (updated.heeded_count ?? 0) + 1;
739
- updated.last_outcome = outcome.at;
1013
+ catch (err) {
1014
+ malformedRows.push({
1015
+ line: 0,
1016
+ raw: "",
1017
+ error: `failed to read ledger: ${err instanceof Error ? err.message : String(err)}`,
1018
+ });
1019
+ return { malformedRows, eventsByCorrectionId };
740
1020
  }
741
- else if (outcome.kind === "recurred") {
742
- updated.recurrence_count = (updated.recurrence_count ?? 0) + 1;
743
- updated.last_outcome = outcome.at;
1021
+ const lines = raw.split("\n");
1022
+ for (let i = 0; i < lines.length; i++) {
1023
+ const lineNo = i + 1;
1024
+ const trimmed = lines[i].trim();
1025
+ if (!trimmed)
1026
+ continue; // blank line — not malformed, just skip
1027
+ let evt;
1028
+ try {
1029
+ evt = JSON.parse(trimmed);
1030
+ }
1031
+ catch (err) {
1032
+ malformedRows.push({
1033
+ line: lineNo,
1034
+ raw: lines[i],
1035
+ error: err instanceof Error ? err.message : String(err),
1036
+ });
1037
+ continue;
1038
+ }
1039
+ if (!evt || typeof evt !== "object" || !evt.correction_id || !evt.kind || !evt.at) {
1040
+ malformedRows.push({
1041
+ line: lineNo,
1042
+ raw: lines[i],
1043
+ error: "parsed but missing required field(s): correction_id, kind, at",
1044
+ });
1045
+ continue;
1046
+ }
1047
+ let arr = eventsByCorrectionId.get(evt.correction_id);
1048
+ if (!arr) {
1049
+ arr = [];
1050
+ eventsByCorrectionId.set(evt.correction_id, arr);
1051
+ }
1052
+ arr.push(evt);
744
1053
  }
745
- else if (outcome.kind === "predicted") {
746
- // Wave 5: prediction fired — instrument the predict-the-correction loop.
747
- updated.predicted_count = (updated.predicted_count ?? 0) + 1;
748
- updated.last_predicted = outcome.at;
1054
+ return { malformedRows, eventsByCorrectionId };
1055
+ }
1056
+ /** Snapshot of a correction's CURRENT on-disk counter/derived fields (before any rebuild). */
1057
+ function currentCountersOf(r) {
1058
+ return {
1059
+ retrieved_count: r.retrieved_count,
1060
+ heeded_count: r.heeded_count,
1061
+ recurrence_count: r.recurrence_count,
1062
+ predicted_count: r.predicted_count,
1063
+ predict_hits: r.predict_hits,
1064
+ last_retrieved: r.last_retrieved,
1065
+ last_outcome: r.last_outcome,
1066
+ last_predicted: r.last_predicted,
1067
+ precision: r.precision,
1068
+ predict_precision: r.predict_precision,
1069
+ proof_confidence: r.proof_confidence,
1070
+ };
1071
+ }
1072
+ const COUNTER_FIELD_NAMES = [
1073
+ "retrieved_count",
1074
+ "heeded_count",
1075
+ "recurrence_count",
1076
+ "predicted_count",
1077
+ "predict_hits",
1078
+ "last_retrieved",
1079
+ "last_outcome",
1080
+ "last_predicted",
1081
+ "precision",
1082
+ "predict_precision",
1083
+ "proof_confidence",
1084
+ ];
1085
+ function countersEqual(a, b) {
1086
+ return COUNTER_FIELD_NAMES.every((k) => a[k] === b[k]);
1087
+ }
1088
+ /**
1089
+ * Shared replay-and-compare core for a single project. Read-only — never
1090
+ * mutates, never locks. For every on-disk correction that has at least one
1091
+ * counter-affecting ledger event, replays that event history from scratch
1092
+ * (sortForReplay + recomputeCorrectionCounters) and compares the result
1093
+ * against what's CURRENTLY materialized on disk.
1094
+ *
1095
+ * Corrections with ZERO ledger events for their id are deliberately EXCLUDED
1096
+ * from the result — there is no ledger evidence to rebuild FROM, so "recompute
1097
+ * from scratch" would mean zeroing out counters that might legitimately have
1098
+ * come from a source other than this project's _outcomes.jsonl (e.g. a very
1099
+ * old pre-ledger record). Rebuild only ever touches ids the lossless ledger
1100
+ * can actually vouch for.
1101
+ *
1102
+ * This is the ONE implementation shared by `runOutcomesRebuild` (write-side,
1103
+ * corrections.ts) and store-doctor.ts's `outcomes_ledger_divergence` check
1104
+ * (read-only) — factored here so the two can never disagree.
1105
+ */
1106
+ export function computeLedgerDivergence(project) {
1107
+ const { malformedRows, eventsByCorrectionId } = parseOutcomesLedger(project);
1108
+ const all = readCorrections(project);
1109
+ const entries = [];
1110
+ for (const record of all) {
1111
+ const raw = eventsByCorrectionId.get(record.id);
1112
+ if (!raw || raw.length === 0)
1113
+ continue;
1114
+ const relevant = raw.filter((e) => COUNTER_KINDS.has(e.kind));
1115
+ if (relevant.length === 0)
1116
+ continue;
1117
+ const ordered = sortForReplay(relevant);
1118
+ const after = recomputeCorrectionCounters(ordered, record);
1119
+ const before = currentCountersOf(record);
1120
+ entries.push({ id: record.id, record, before, after, changed: !countersEqual(before, after) });
749
1121
  }
750
- else if (outcome.kind === "predict_hit") {
751
- updated.predict_hits = (updated.predict_hits ?? 0) + 1;
1122
+ return { malformedRows, entries };
1123
+ }
1124
+ function toPublicDiffs(entries) {
1125
+ return entries.map(({ id, before, after, changed }) => ({ id, before, after, changed }));
1126
+ }
1127
+ /**
1128
+ * Rebuild a project's materialized outcome counters from a full replay of the
1129
+ * (lossless) _outcomes.jsonl ledger — the WRITE-side repair for records
1130
+ * corrupted by the pre-05b3699 unlocked read-modify-write in recordOutcome.
1131
+ *
1132
+ * SAFETY INVARIANTS (mirrors store-repair.ts's StoreRepairOptions contract):
1133
+ * 1. DRY-RUN BY DEFAULT. `opts.apply` must be EXPLICITLY true to mutate.
1134
+ * 2. IDEMPOTENT. A second apply run recomputes from the SAME lossless ledger
1135
+ * and disk now matches it, so every entry's `changed` is false and
1136
+ * nothing is rewritten (not even the index).
1137
+ * 3. LOCK SAFETY. The apply pass runs inside `withLock(\`corrections-${project}\`,
1138
+ * …)` — the SAME lock name recordOutcome/retractCorrection/writeCorrection
1139
+ * use (05b3699) — so a rebuild-apply can never race a live recordOutcome
1140
+ * call for the same project.
1141
+ * 4. Only per-correction JSON files that actually differ are rewritten; the
1142
+ * shared _index.md is regenerated ONCE at the end, only if anything wrote.
1143
+ */
1144
+ export function runOutcomesRebuild(project, opts = {}) {
1145
+ const apply = opts.apply === true;
1146
+ if (!apply) {
1147
+ const plan = computeLedgerDivergence(project);
1148
+ const corrections = toPublicDiffs(plan.entries);
1149
+ return {
1150
+ apply: false,
1151
+ malformedRows: plan.malformedRows,
1152
+ corrections,
1153
+ summary: {
1154
+ totalCorrections: corrections.length,
1155
+ changed: corrections.filter((c) => c.changed).length,
1156
+ malformed: plan.malformedRows.length,
1157
+ },
1158
+ };
752
1159
  }
753
- const r = updated.retrieved_count ?? 0;
754
- // Clamp to [0,1]: `retrieved` is guarded 1/day but `heeded` can fire on every
755
- // session_end, so raw heeded/retrieved can exceed 1.0 ("150% heeded" is
756
- // nonsense). min(1, …) keeps the metric honest. (Root-cause follow-up: apply
757
- // the same 1/day guard to heeded as retrieved has, for finer resolution.)
758
- // NB (Wave 5): `precision` is heeded/retrieved ONLY — predict_* never touch it.
759
- updated.precision = r > 0 ? Math.min(1, Number(((updated.heeded_count ?? 0) / r).toFixed(3))) : undefined;
760
- // Wave 5: predict_precision = predict_hits / predicted_count, kept SEPARATE
761
- // from `precision`. Undefined until at least one prediction has fired.
762
- // A predict_hit implies a prior prediction. If data is inconsistent (hits
763
- // recorded without a matching predicted_count — e.g. migrated/corrupt records),
764
- // floor the denominator at predict_hits so the metric stays VISIBLE and bounded
765
- // rather than silently undefined while hits exist.
766
- const pc = updated.predicted_count ?? 0;
767
- const ph = updated.predict_hits ?? 0;
768
- const predictDenom = Math.max(pc, ph);
769
- updated.predict_precision = predictDenom > 0
770
- ? Math.min(1, Number((ph / predictDenom).toFixed(3)))
771
- : undefined;
772
- // P3: evidence-grounded proof_confidence. With NO outcome evidence yet, keep the
773
- // authority prior (weight); once heeded/recurrence accrue, move to the Beta
774
- // posterior so a rule that keeps being honored strengthens and one whose bug
775
- // keeps recurring weakens. Kept SEPARATE from `precision` (heeded/retrieved) and
776
- // from `weight` (static authority) — this is the evidence axis.
777
- const heededC = updated.heeded_count ?? 0;
778
- const recurC = updated.recurrence_count ?? 0;
779
- updated.proof_confidence = (heededC + recurC) > 0
780
- ? Number(betaPosterior(heededC, recurC).toFixed(3))
781
- : (updated.weight ?? defaultWeight(updated.severity));
782
- // Re-write the JSON file atomically (tmp + rename — prevents truncation on
783
- // SIGTERM). Reuse the EXISTING filename (see findExistingCorrectionFile
784
- // doc) — never recompute via slugify, or a v1-named file would orphan.
785
- const filename = findExistingCorrectionFile(dir, updated.id)
786
- ?? `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
787
- const filepath = path.join(dir, filename);
788
- writeRecordAtomic(filepath, updated);
789
- // W2-1: regenerate the materialized index on every corrections mutation.
790
- // (The ledger-only early-returns above for triggered/not_triggered/unknown
791
- // touch no per-record field the index renders — severity/status/date/rule/
792
- // failure_class are all unchanged — so they deliberately skip this call,
793
- // preserving the existing hot-path optimization documented above.)
794
- regenerateCorrectionsIndex(outcome.project);
1160
+ // ── LOCKED apply pass ──────────────────────────────────────────────────────
1161
+ // Re-derive the plan INSIDE the lock: a concurrent live recordOutcome call
1162
+ // could append new ledger lines or rewrite a correction file between any
1163
+ // earlier unlocked plan computation and acquiring this lock. Same lock name
1164
+ // as writeCorrection/retractCorrection/recordOutcome above.
1165
+ return withLock(`corrections-${project}`, () => {
1166
+ const dir = correctionsDir(project);
1167
+ const plan = computeLedgerDivergence(project);
1168
+ let wrote = 0;
1169
+ for (const entry of plan.entries) {
1170
+ if (!entry.changed)
1171
+ continue;
1172
+ const updated = {
1173
+ ...entry.record,
1174
+ retrieved_count: entry.after.retrieved_count,
1175
+ heeded_count: entry.after.heeded_count,
1176
+ recurrence_count: entry.after.recurrence_count,
1177
+ predicted_count: entry.after.predicted_count,
1178
+ predict_hits: entry.after.predict_hits,
1179
+ last_retrieved: entry.after.last_retrieved,
1180
+ last_outcome: entry.after.last_outcome,
1181
+ last_predicted: entry.after.last_predicted,
1182
+ precision: entry.after.precision,
1183
+ predict_precision: entry.after.predict_precision,
1184
+ proof_confidence: entry.after.proof_confidence,
1185
+ };
1186
+ // Reuse the EXISTING file's name (see findExistingCorrectionFile doc) —
1187
+ // never recompute via slugify, matching every other rewrite in this file.
1188
+ const filename = findExistingCorrectionFile(dir, updated.id) ??
1189
+ `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
1190
+ writeRecordAtomic(path.join(dir, filename), updated);
1191
+ wrote++;
1192
+ }
1193
+ // Regenerate the shared index ONCE, only if something actually changed —
1194
+ // keeps a clean second apply-run a true no-op (idempotency invariant #2).
1195
+ if (wrote > 0) {
1196
+ regenerateCorrectionsIndex(project);
1197
+ }
1198
+ const corrections = toPublicDiffs(plan.entries);
1199
+ return {
1200
+ apply: true,
1201
+ malformedRows: plan.malformedRows,
1202
+ corrections,
1203
+ summary: {
1204
+ totalCorrections: corrections.length,
1205
+ changed: wrote,
1206
+ malformed: plan.malformedRows.length,
1207
+ },
1208
+ };
1209
+ });
795
1210
  }
796
1211
  /**
797
1212
  * Best-effort: append one row to corrections/_rejected.jsonl recording a