agent-recall-core 3.4.37 → 3.4.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -53
- package/dist/helpers/activity-feed.d.ts.map +1 -1
- package/dist/helpers/activity-feed.js +12 -7
- package/dist/helpers/activity-feed.js.map +1 -1
- package/dist/helpers/alignment-patterns.d.ts.map +1 -1
- package/dist/helpers/alignment-patterns.js +8 -13
- package/dist/helpers/alignment-patterns.js.map +1 -1
- package/dist/helpers/handoff.d.ts.map +1 -1
- package/dist/helpers/handoff.js +5 -8
- package/dist/helpers/handoff.js.map +1 -1
- package/dist/helpers/journal-files.d.ts +17 -0
- package/dist/helpers/journal-files.d.ts.map +1 -1
- package/dist/helpers/journal-files.js +61 -0
- package/dist/helpers/journal-files.js.map +1 -1
- package/dist/helpers/journal-filter.d.ts +22 -0
- package/dist/helpers/journal-filter.d.ts.map +1 -1
- package/dist/helpers/journal-filter.js +23 -0
- package/dist/helpers/journal-filter.js.map +1 -1
- package/dist/helpers/journal-name-parser.d.ts +13 -2
- package/dist/helpers/journal-name-parser.d.ts.map +1 -1
- package/dist/helpers/journal-name-parser.js +46 -2
- package/dist/helpers/journal-name-parser.js.map +1 -1
- package/dist/helpers/journal-sig-theme.d.ts +11 -0
- package/dist/helpers/journal-sig-theme.d.ts.map +1 -1
- package/dist/helpers/journal-sig-theme.js +18 -0
- package/dist/helpers/journal-sig-theme.js.map +1 -1
- package/dist/host-profile.d.ts +69 -0
- package/dist/host-profile.d.ts.map +1 -0
- package/dist/host-profile.js +107 -0
- package/dist/host-profile.js.map +1 -0
- package/dist/index.d.ts +7 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/dist/naming.d.ts +10 -3
- package/dist/naming.d.ts.map +1 -1
- package/dist/naming.js +19 -15
- package/dist/naming.js.map +1 -1
- package/dist/palace/pipeline.d.ts +6 -0
- package/dist/palace/pipeline.d.ts.map +1 -1
- package/dist/palace/pipeline.js +15 -2
- package/dist/palace/pipeline.js.map +1 -1
- package/dist/palace/rooms.d.ts +20 -0
- package/dist/palace/rooms.d.ts.map +1 -1
- package/dist/palace/rooms.js +90 -0
- package/dist/palace/rooms.js.map +1 -1
- package/dist/palace/skills.d.ts +13 -0
- package/dist/palace/skills.d.ts.map +1 -1
- package/dist/palace/skills.js +53 -6
- package/dist/palace/skills.js.map +1 -1
- package/dist/storage/ab-experiment.d.ts.map +1 -1
- package/dist/storage/ab-experiment.js +8 -12
- package/dist/storage/ab-experiment.js.map +1 -1
- package/dist/storage/corrections.d.ts +146 -0
- package/dist/storage/corrections.d.ts.map +1 -1
- package/dist/storage/corrections.js +746 -151
- package/dist/storage/corrections.js.map +1 -1
- package/dist/storage/cwd-allowlist.d.ts.map +1 -1
- package/dist/storage/cwd-allowlist.js +2 -4
- package/dist/storage/cwd-allowlist.js.map +1 -1
- package/dist/storage/lifecycle-telemetry.d.ts +56 -0
- package/dist/storage/lifecycle-telemetry.d.ts.map +1 -0
- package/dist/storage/lifecycle-telemetry.js +134 -0
- package/dist/storage/lifecycle-telemetry.js.map +1 -0
- package/dist/storage/memory-protocol.d.ts.map +1 -1
- package/dist/storage/memory-protocol.js +6 -10
- package/dist/storage/memory-protocol.js.map +1 -1
- package/dist/storage/paths.d.ts +120 -1
- package/dist/storage/paths.d.ts.map +1 -1
- package/dist/storage/paths.js +145 -31
- package/dist/storage/paths.js.map +1 -1
- package/dist/storage/project.d.ts.map +1 -1
- package/dist/storage/project.js +12 -4
- package/dist/storage/project.js.map +1 -1
- package/dist/storage/sanitize.d.ts +55 -0
- package/dist/storage/sanitize.d.ts.map +1 -0
- package/dist/storage/sanitize.js +114 -0
- package/dist/storage/sanitize.js.map +1 -0
- package/dist/storage/session.d.ts +37 -4
- package/dist/storage/session.d.ts.map +1 -1
- package/dist/storage/session.js +105 -9
- package/dist/storage/session.js.map +1 -1
- package/dist/tools-logic/bootstrap.d.ts.map +1 -1
- package/dist/tools-logic/bootstrap.js +2 -4
- package/dist/tools-logic/bootstrap.js.map +1 -1
- package/dist/tools-logic/check-action.d.ts.map +1 -1
- package/dist/tools-logic/check-action.js +97 -3
- package/dist/tools-logic/check-action.js.map +1 -1
- package/dist/tools-logic/check.d.ts +22 -0
- package/dist/tools-logic/check.d.ts.map +1 -1
- package/dist/tools-logic/check.js +36 -9
- package/dist/tools-logic/check.js.map +1 -1
- package/dist/tools-logic/export-corrections.js +2 -2
- package/dist/tools-logic/export-corrections.js.map +1 -1
- package/dist/tools-logic/journal-capture.d.ts.map +1 -1
- package/dist/tools-logic/journal-capture.js +53 -36
- package/dist/tools-logic/journal-capture.js.map +1 -1
- package/dist/tools-logic/journal-write.d.ts.map +1 -1
- package/dist/tools-logic/journal-write.js +51 -28
- package/dist/tools-logic/journal-write.js.map +1 -1
- package/dist/tools-logic/knowledge-read.d.ts.map +1 -1
- package/dist/tools-logic/knowledge-read.js +8 -6
- package/dist/tools-logic/knowledge-read.js.map +1 -1
- package/dist/tools-logic/knowledge-write.d.ts.map +1 -1
- package/dist/tools-logic/knowledge-write.js +5 -8
- package/dist/tools-logic/knowledge-write.js.map +1 -1
- package/dist/tools-logic/palace-write.d.ts.map +1 -1
- package/dist/tools-logic/palace-write.js +5 -1
- package/dist/tools-logic/palace-write.js.map +1 -1
- package/dist/tools-logic/project-board.js +2 -2
- package/dist/tools-logic/project-board.js.map +1 -1
- package/dist/tools-logic/session-end.d.ts.map +1 -1
- package/dist/tools-logic/session-end.js +58 -8
- package/dist/tools-logic/session-end.js.map +1 -1
- package/dist/tools-logic/session-start.d.ts.map +1 -1
- package/dist/tools-logic/session-start.js +34 -10
- package/dist/tools-logic/session-start.js.map +1 -1
- package/dist/tools-logic/smart-recall.d.ts +93 -1
- package/dist/tools-logic/smart-recall.d.ts.map +1 -1
- package/dist/tools-logic/smart-recall.js +199 -11
- package/dist/tools-logic/smart-recall.js.map +1 -1
- package/dist/tools-logic/smart-remember.d.ts.map +1 -1
- package/dist/tools-logic/smart-remember.js +12 -0
- package/dist/tools-logic/smart-remember.js.map +1 -1
- package/dist/tools-logic/store-doctor.d.ts +1 -1
- package/dist/tools-logic/store-doctor.d.ts.map +1 -1
- package/dist/tools-logic/store-doctor.js +64 -0
- package/dist/tools-logic/store-doctor.js.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/vector/local-vector-store.d.ts.map +1 -1
- package/dist/vector/local-vector-store.js +5 -4
- package/dist/vector/local-vector-store.js.map +1 -1
- package/package.json +4 -3
- package/dist/tools-logic/brief.d.ts +0 -84
- package/dist/tools-logic/brief.d.ts.map +0 -1
- package/dist/tools-logic/brief.js +0 -193
- package/dist/tools-logic/brief.js.map +0 -1
- package/dist/tools-logic/dashboard-export.d.ts +0 -130
- package/dist/tools-logic/dashboard-export.d.ts.map +0 -1
- package/dist/tools-logic/dashboard-export.js +0 -485
- package/dist/tools-logic/dashboard-export.js.map +0 -1
- package/dist/tools-logic/memory-query.d.ts +0 -43
- package/dist/tools-logic/memory-query.d.ts.map +0 -1
- package/dist/tools-logic/memory-query.js +0 -77
- package/dist/tools-logic/memory-query.js.map +0 -1
- package/dist/tools-logic/project-status.d.ts +0 -24
- package/dist/tools-logic/project-status.d.ts.map +0 -1
- package/dist/tools-logic/project-status.js +0 -155
- package/dist/tools-logic/project-status.js.map +0 -1
|
@@ -6,24 +6,23 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "node:fs";
|
|
8
8
|
import * as path from "node:path";
|
|
9
|
-
import
|
|
9
|
+
import * as crypto from "node:crypto";
|
|
10
10
|
import { ensureDir } from "./fs-utils.js";
|
|
11
|
+
import { byteCap, sanitizeName } from "./sanitize.js";
|
|
12
|
+
import { journalDir, projectSubPath } from "./paths.js";
|
|
13
|
+
import { withLock } from "./filelock.js";
|
|
11
14
|
// ---------------------------------------------------------------------------
|
|
12
15
|
// Internal helpers
|
|
13
16
|
// ---------------------------------------------------------------------------
|
|
14
17
|
function correctionsDir(project) {
|
|
15
|
-
//
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if (!resolved.startsWith(rootSep)) {
|
|
24
|
-
throw new Error(`Invalid project (path escape): ${project}`);
|
|
25
|
-
}
|
|
26
|
-
return resolved;
|
|
18
|
+
// v2: shared sanitizer + EXISTING-DIR REUSE RULE (naming-v2 spec §2) — was a
|
|
19
|
+
// local duplicate of the old char-preserving sanitizer, which was exactly
|
|
20
|
+
// the call-site divergence the spec calls out: corrections/ could resolve
|
|
21
|
+
// to a DIFFERENT case-folded directory than journal/palace for the same
|
|
22
|
+
// project. F2 fix (independent review, 2026-07-20): now routes through
|
|
23
|
+
// paths.ts's projectSubPath (was a hand-rolled path.join + inline escape
|
|
24
|
+
// check that duplicated, and could drift from, paths.ts's own guard).
|
|
25
|
+
return projectSubPath(project, "corrections");
|
|
27
26
|
}
|
|
28
27
|
function outcomesPath(project) {
|
|
29
28
|
return path.join(correctionsDir(project), "_outcomes.jsonl");
|
|
@@ -60,13 +59,97 @@ function detectSeverity(text) {
|
|
|
60
59
|
const p0Patterns = /\bnever\b|\balways\b|\bdon'?t\b|\bdo not\b|\bmust not\b|\bforbid\b|\bprohibit\b/i;
|
|
61
60
|
return p0Patterns.test(text) ? "p0" : "p1";
|
|
62
61
|
}
|
|
63
|
-
/**
|
|
62
|
+
/**
|
|
63
|
+
* Leading interjection / stop-phrase strip (naming-v2 spec §3, corrections
|
|
64
|
+
* row). A correction's `rule` text is frequently the raw human utterance
|
|
65
|
+
* ("No, that's wrong. Never publish without approval") — the slug should
|
|
66
|
+
* describe the RULE, not the acknowledgment it opens with. Repeats until no
|
|
67
|
+
* further match (so "No, wait, actually never publish..." fully strips).
|
|
68
|
+
*/
|
|
69
|
+
// F4 fix (independent review, 2026-07-20): the trailing separator class was
|
|
70
|
+
// ASCII-only ([,.!?:;\s]), so a CJK interjection followed by full-width
|
|
71
|
+
// punctuation (,。!?:;、 — the normal punctuation after "你错了"/"不对" in
|
|
72
|
+
// real usage) never matched at all — the CJK branch of this regex was
|
|
73
|
+
// effectively dead code. Added the full-width punctuation marks to the
|
|
74
|
+
// separator class; ASCII behavior (and the "no rescue without a separator"
|
|
75
|
+
// property that keeps "notification..." from being stripped) is unchanged.
|
|
76
|
+
const INTERJECTION_PREFIX = /^(no|yes|ok|okay|nope|yeah|wait|stop|hmm|but|actually|你错了|不对|不是|对|好的?)[,.!?:;\s,。!?:;、]+/i;
|
|
77
|
+
// Exported for direct unit testing of the F4 fix (independent review,
|
|
78
|
+
// 2026-07-20) — the full writeCorrection() pipeline runs this text through
|
|
79
|
+
// the capture-quality gate first (isLikelyRealCorrection), which requires an
|
|
80
|
+
// actionable-signal marker; that gate is orthogonal to (and would otherwise
|
|
81
|
+
// obscure) the interjection-stripping behavior under test here. Not part of
|
|
82
|
+
// the public index.ts barrel — internal-use export, same pattern as the
|
|
83
|
+
// other test-only exports already in this file (splitSentences, dropHardNoise).
|
|
84
|
+
export function stripInterjections(text) {
|
|
85
|
+
let s = text;
|
|
86
|
+
// Bounded by construction: each pass strips a short, fixed prefix or makes
|
|
87
|
+
// no change (loop exits) — a correction rule is never long enough for this
|
|
88
|
+
// to run more than a handful of iterations.
|
|
89
|
+
for (;;) {
|
|
90
|
+
const next = s.replace(INTERJECTION_PREFIX, "");
|
|
91
|
+
if (next === s)
|
|
92
|
+
return s;
|
|
93
|
+
s = next;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Slugify text for use in filenames — v2 grammar (naming-v2 spec §3, §6):
|
|
98
|
+
* strip leading interjections from the RULE text first, then sanitize via
|
|
99
|
+
* the shared v2 sanitizer, byte-capped at 48 bytes (corrections' slug
|
|
100
|
+
* budget). Falls back to sanitizing the ORIGINAL text when stripping leaves
|
|
101
|
+
* nothing behind.
|
|
102
|
+
*/
|
|
64
103
|
function slugify(text) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
104
|
+
const stripped = stripInterjections(text).trim();
|
|
105
|
+
return sanitizeName(stripped || text, 48);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Find the on-disk filename for an EXISTING correction record by id, by
|
|
109
|
+
* scanning and parsing every *.json file in the corrections dir.
|
|
110
|
+
*
|
|
111
|
+
* Rewrite paths (retractCorrection, recordOutcome, on-write consolidation)
|
|
112
|
+
* MUST reuse the file the record already lives at — recomputing a filename
|
|
113
|
+
* via `slugify(record.rule)` would drift the moment slugify's grammar
|
|
114
|
+
* changes (as it just did, v1 → v2): a record written under the OLD
|
|
115
|
+
* single-dash grammar would get rewritten to a NEW double-dash filename,
|
|
116
|
+
* leaving the original untouched on disk (violates "never rename/rewrite an
|
|
117
|
+
* existing file") AND creating a duplicate record with the same id (readers
|
|
118
|
+
* don't dedupe by id — every *.json file is a distinct record). Returns null
|
|
119
|
+
* when no matching file is found (defensive fallback only; the record was
|
|
120
|
+
* just read from this same directory by every current caller).
|
|
121
|
+
*
|
|
122
|
+
* KNOWN LIMITATION of the null-fallback recompute at the call sites: it
|
|
123
|
+
* produces the PLAIN `${date}--${slugify(rule)}.json` name and does NOT carry
|
|
124
|
+
* the id-hash suffix a collision-disambiguated record was originally written
|
|
125
|
+
* under (see writeCorrection's brand-new branch). Reachable only if a
|
|
126
|
+
* record's backing file vanishes from disk mid-call, inside the lock —
|
|
127
|
+
* defensive-only today, flagged by review 2026-07-27.
|
|
128
|
+
*/
|
|
129
|
+
function findExistingCorrectionFile(dir, id) {
|
|
130
|
+
if (!fs.existsSync(dir))
|
|
131
|
+
return null;
|
|
132
|
+
let files;
|
|
133
|
+
try {
|
|
134
|
+
files = fs.readdirSync(dir);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
for (const file of files) {
|
|
140
|
+
if (!file.endsWith(".json") || file.startsWith("_"))
|
|
141
|
+
continue;
|
|
142
|
+
try {
|
|
143
|
+
const raw = fs.readFileSync(path.join(dir, file), "utf-8");
|
|
144
|
+
const parsed = JSON.parse(raw);
|
|
145
|
+
if (parsed && parsed.id === id)
|
|
146
|
+
return file;
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
// skip malformed/unreadable — never throw from a lookup helper
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return null;
|
|
70
153
|
}
|
|
71
154
|
function defaultWeight(severity) {
|
|
72
155
|
return severity === "p0" ? 1.0 : 0.7;
|
|
@@ -85,6 +168,91 @@ function writeRecordAtomic(filepath, record) {
|
|
|
85
168
|
fs.writeFileSync(tmp, JSON.stringify(record, null, 2), { encoding: "utf-8", mode: 0o600 });
|
|
86
169
|
fs.renameSync(tmp, filepath);
|
|
87
170
|
}
|
|
171
|
+
/** Sort rank for the corrections index — p0 first, unknown severities last. */
|
|
172
|
+
function severityRank(severity) {
|
|
173
|
+
switch (severity) {
|
|
174
|
+
case "p0": return 0;
|
|
175
|
+
case "p1": return 1;
|
|
176
|
+
case "p2": return 2;
|
|
177
|
+
case "p3": return 3;
|
|
178
|
+
default: return 4;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* One-line, table-safe rendering of a correction's rule text (W2-1, naming-v2
|
|
183
|
+
* spec §4): collapse newlines/whitespace, escape markdown table pipes, then
|
|
184
|
+
* byte/char-cap at 80 chars with an ellipsis. Never throws on odd input.
|
|
185
|
+
*/
|
|
186
|
+
function ruleOneLiner(rule, maxLen = 80) {
|
|
187
|
+
const oneLine = (rule ?? "").replace(/\s+/g, " ").trim().replace(/\|/g, "\\|");
|
|
188
|
+
if (oneLine.length <= maxLen)
|
|
189
|
+
return oneLine || "(no rule text)";
|
|
190
|
+
return oneLine.slice(0, maxLen - 1).trimEnd() + "…";
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* W2-1 (naming-v2 spec §4) — regenerate corrections/_index.md, the
|
|
194
|
+
* materialized machine fast-path over the corrections store: a severity-first
|
|
195
|
+
* sorted table (`| severity | failure_class | status | date | rule |`),
|
|
196
|
+
* serving "show me my worst active pattern" via one `ls`+`cat` instead of
|
|
197
|
+
* reading every JSON file.
|
|
198
|
+
*
|
|
199
|
+
* ATOMIC (write-temp + rename) and re-derived from a FULL re-read of the
|
|
200
|
+
* store's *.json files on every call — source of truth is the files, never
|
|
201
|
+
* incremental in-memory state, so the index always matches disk regardless
|
|
202
|
+
* of caller.
|
|
203
|
+
*
|
|
204
|
+
* NEVER throws: regenerating the index must never fail the write that
|
|
205
|
+
* triggered it (writeCorrection / retractCorrection / recordOutcome). Any
|
|
206
|
+
* error is logged as a one-line stderr message and swallowed.
|
|
207
|
+
*/
|
|
208
|
+
export function regenerateCorrectionsIndex(project) {
|
|
209
|
+
try {
|
|
210
|
+
const dir = correctionsDir(project);
|
|
211
|
+
ensureDir(dir);
|
|
212
|
+
const all = readCorrections(project); // full re-read of every *.json file
|
|
213
|
+
const activeCount = all.filter((r) => r.active !== false).length;
|
|
214
|
+
const retractedCount = all.length - activeCount;
|
|
215
|
+
const p0ActiveCount = all.filter((r) => r.severity === "p0" && r.active !== false).length;
|
|
216
|
+
// severity (p0>p1>p2>p3) → status (active first) → date desc
|
|
217
|
+
const rows = [...all].sort((a, b) => {
|
|
218
|
+
const sevDiff = severityRank(a.severity) - severityRank(b.severity);
|
|
219
|
+
if (sevDiff !== 0)
|
|
220
|
+
return sevDiff;
|
|
221
|
+
const aActive = a.active !== false ? 0 : 1;
|
|
222
|
+
const bActive = b.active !== false ? 0 : 1;
|
|
223
|
+
if (aActive !== bActive)
|
|
224
|
+
return aActive - bActive;
|
|
225
|
+
return (b.date ?? "").localeCompare(a.date ?? "");
|
|
226
|
+
});
|
|
227
|
+
const lines = [];
|
|
228
|
+
lines.push("# Corrections Index — regenerated on write; do not edit");
|
|
229
|
+
lines.push("");
|
|
230
|
+
lines.push(`Generated: ${new Date().toISOString()}`);
|
|
231
|
+
lines.push(`${activeCount} active / ${retractedCount} retracted / ${p0ActiveCount} p0-active`);
|
|
232
|
+
lines.push("");
|
|
233
|
+
lines.push("| severity | failure_class | status | date | rule |");
|
|
234
|
+
lines.push("|---|---|---|---|---|");
|
|
235
|
+
for (const r of rows) {
|
|
236
|
+
const status = r.active !== false ? "active" : "retracted";
|
|
237
|
+
const failureClass = r.failure_class ?? "other";
|
|
238
|
+
lines.push(`| ${r.severity} | ${failureClass} | ${status} | ${r.date} | ${ruleOneLiner(r.rule)} |`);
|
|
239
|
+
}
|
|
240
|
+
const content = lines.join("\n") + "\n";
|
|
241
|
+
const indexPath = path.join(dir, "_index.md");
|
|
242
|
+
const tmp = `${indexPath}.tmp-${crypto.randomBytes(4).toString("hex")}`;
|
|
243
|
+
fs.writeFileSync(tmp, content, "utf-8");
|
|
244
|
+
fs.renameSync(tmp, indexPath);
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
try {
|
|
248
|
+
process.stderr.write(`[agent-recall] corrections index regeneration failed for "${project}": ` +
|
|
249
|
+
`${err instanceof Error ? err.message : String(err)}\n`);
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
/* a diagnostic write must never throw into the caller */
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
88
256
|
/**
|
|
89
257
|
* Beta posterior mean E[Beta(α,β)] with α=heeded+1, β=recurrence+1 (Laplace).
|
|
90
258
|
* Mirrors the canonical `betaUtility` in tools-logic/smart-recall.ts — kept INLINE
|
|
@@ -391,55 +559,98 @@ export function writeCorrection(project, correction) {
|
|
|
391
559
|
// Auto-detect severity if not already set
|
|
392
560
|
const severity = correction.severity ?? detectSeverity(`${correction.rule} ${correction.context}`);
|
|
393
561
|
const record = applyCorrectionDefaults({ ...correction, severity }, todayDate());
|
|
394
|
-
// ──
|
|
395
|
-
//
|
|
396
|
-
//
|
|
397
|
-
//
|
|
398
|
-
//
|
|
399
|
-
//
|
|
400
|
-
//
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
:
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
562
|
+
// ── LOCKED critical section (P0 data-loss fix, 2026-07-25) ────────────────
|
|
563
|
+
// From here on this is an unlocked read-all→find→mutate→atomic-rewrite→
|
|
564
|
+
// index-regen sequence: the consolidation scan below reads every *.json
|
|
565
|
+
// file, the merge branch rewrites the ONE matched record, and either branch
|
|
566
|
+
// ends by calling regenerateCorrectionsIndex, which re-reads the whole store
|
|
567
|
+
// and rewrites the ONE shared _index.md. Unlocked, two concurrent
|
|
568
|
+
// writeCorrection calls for the same project can (a) both miss each other's
|
|
569
|
+
// match and create duplicate un-merged records instead of consolidating, or
|
|
570
|
+
// (b) race on the shared index rewrite. Locked with `corrections-${project}`
|
|
571
|
+
// — PROJECT-scoped, not per-correction-id, because the index file is shared
|
|
572
|
+
// across every correction in the project (a per-id lock would still let two
|
|
573
|
+
// DIFFERENT corrections' index writes race). Mirrors retractCorrection /
|
|
574
|
+
// recordOutcome below and the existing `palace-index-${project}` /
|
|
575
|
+
// `digest-${project}` lock pattern elsewhere in this codebase.
|
|
576
|
+
return withLock(`corrections-${project}`, () => {
|
|
577
|
+
// ── P1: on-write consolidation (refine-not-overwrite) ─────────────────────
|
|
578
|
+
// Borrow Hindsight's consolidation idea, AR-native: instead of accumulating a
|
|
579
|
+
// new dated file for a re-stated rule, fold it into the most similar ACTIVE
|
|
580
|
+
// correction of the SAME kind and bump that record's proof_count. The matched
|
|
581
|
+
// record keeps its id/date (stable document_id) and absorbs the new tags +
|
|
582
|
+
// higher severity/authority/weight. High-precision LOCAL gate — no key, no
|
|
583
|
+
// network — so this never runs an LLM on the storage hot path.
|
|
584
|
+
const normNew = normalizeRule(record.rule);
|
|
585
|
+
for (const existing of readActiveCorrections(project)) {
|
|
586
|
+
if (existing.id === record.id)
|
|
587
|
+
continue; // never merge into self (same-day re-slug)
|
|
588
|
+
if ((existing.kind ?? "correction") !== (record.kind ?? "correction"))
|
|
589
|
+
continue;
|
|
590
|
+
if (normalizeRule(existing.rule) !== normNew)
|
|
591
|
+
continue;
|
|
592
|
+
const merged = {
|
|
593
|
+
...existing,
|
|
594
|
+
proof_count: (existing.proof_count ?? 1) + 1,
|
|
595
|
+
merged_from: [...(existing.merged_from ?? []), record.id],
|
|
596
|
+
tags: Array.from(new Set([...(existing.tags ?? []), ...(record.tags ?? [])])),
|
|
597
|
+
// keep the STRONGER signal on every axis
|
|
598
|
+
severity: existing.severity === "p0" || record.severity === "p0" ? "p0" : "p1",
|
|
599
|
+
weight: Math.max(existing.weight ?? 0, record.weight ?? 0),
|
|
600
|
+
authoritative: Boolean(existing.authoritative || record.authoritative),
|
|
601
|
+
last_outcome: new Date().toISOString(),
|
|
602
|
+
// RD-1: keep the existing classification; adopt the incoming capture's
|
|
603
|
+
// class only when the existing record predates the field AND the
|
|
604
|
+
// incoming class is a REAL class. Review fix MEDIUM-1 (2026-07-14):
|
|
605
|
+
// writing "other" into a pre-RD-1 file changes nothing at read time
|
|
606
|
+
// (absent already reads as other) but permanently forecloses a future
|
|
607
|
+
// real classification — stored-wins would keep "other" forever. Note the
|
|
608
|
+
// merge gate is rule-text equality while classification runs on the full
|
|
609
|
+
// context text, so incoming classes CAN differ across merges; stored
|
|
610
|
+
// still wins in that case by design (durable classification).
|
|
611
|
+
...(existing.failure_class
|
|
612
|
+
? { failure_class: existing.failure_class }
|
|
613
|
+
: record.failure_class && record.failure_class !== "other"
|
|
614
|
+
? { failure_class: record.failure_class }
|
|
615
|
+
: {}),
|
|
616
|
+
};
|
|
617
|
+
// Rewrite: reuse the EXISTING file's name (never recompute via slugify —
|
|
618
|
+
// see findExistingCorrectionFile doc). Falls back to a fresh v2 filename
|
|
619
|
+
// only in the defensive case where the file has vanished mid-call.
|
|
620
|
+
const mfile = findExistingCorrectionFile(dir, existing.id)
|
|
621
|
+
?? `${merged.date}--${slugify(merged.rule || merged.id)}.json`;
|
|
622
|
+
writeRecordAtomic(path.join(dir, mfile), merged);
|
|
623
|
+
// W2-1: regenerate the materialized index on every corrections mutation.
|
|
624
|
+
regenerateCorrectionsIndex(project);
|
|
625
|
+
return { written: true, merged: true, id: merged.id };
|
|
626
|
+
}
|
|
627
|
+
// Brand-new record — no existing file to preserve. v2 delimiter ("--").
|
|
628
|
+
let filename = `${record.date}--${slugify(record.rule || record.id)}.json`;
|
|
629
|
+
let filepath = path.join(dir, filename);
|
|
630
|
+
if (fs.existsSync(filepath)) {
|
|
631
|
+
// Same-day slug collision with a DIFFERENT record: the active-rule merge
|
|
632
|
+
// loop above already claimed every same-rule case, so whatever lives at
|
|
633
|
+
// this path is a distinct rule whose slug happens to coincide (two
|
|
634
|
+
// CJK-heavy rules sharing one surviving Latin word, or a RETRACTED
|
|
635
|
+
// record the active-only merge scan skipped). Overwriting would silently
|
|
636
|
+
// destroy it — both callers would still see { written: true }. See
|
|
637
|
+
// test/cjk-slug-collision.test.mjs. Disambiguate inside the slug field
|
|
638
|
+
// with a short deterministic hash of this record's id: hex + single "-"
|
|
639
|
+
// join keeps the "--" field-delimiter grammar intact, and re-running the
|
|
640
|
+
// same write lands on the same name.
|
|
641
|
+
const idHash = crypto.createHash("sha256").update(record.id).digest("hex").slice(0, 8);
|
|
642
|
+
// 39 = 48 (corrections' slug byte budget) - 1 ("-" join) - 8 (hex hash):
|
|
643
|
+
// the disambiguated slug field stays inside the same 48-byte envelope.
|
|
644
|
+
const baseSlug = byteCap(slugify(record.rule || record.id), 39).replace(/-+$/g, "");
|
|
645
|
+
filename = `${record.date}--${baseSlug}-${idHash}.json`;
|
|
646
|
+
filepath = path.join(dir, filename);
|
|
647
|
+
}
|
|
648
|
+
// Atomic write — tmp + rename, mode 0600
|
|
649
|
+
writeRecordAtomic(filepath, record);
|
|
650
|
+
// W2-1: regenerate the materialized index on every corrections mutation.
|
|
651
|
+
regenerateCorrectionsIndex(project);
|
|
652
|
+
return { written: true, merged: false, id: record.id };
|
|
653
|
+
});
|
|
443
654
|
}
|
|
444
655
|
/**
|
|
445
656
|
* Read all corrections for a project, sorted newest first.
|
|
@@ -485,25 +696,38 @@ export function readP0Corrections(project) {
|
|
|
485
696
|
*/
|
|
486
697
|
export function retractCorrection(project, id, reason, supersededBy) {
|
|
487
698
|
const dir = correctionsDir(project);
|
|
488
|
-
//
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
699
|
+
// ── LOCKED critical section (P0 data-loss fix, 2026-07-25) ────────────────
|
|
700
|
+
// read-all→find→mutate→atomic-rewrite→index-regen, unlocked, is exactly the
|
|
701
|
+
// race class this fix closes — see the matching note in writeCorrection
|
|
702
|
+
// above and recordOutcome below. Locked with `corrections-${project}`
|
|
703
|
+
// (PROJECT-scoped, matching regenerateCorrectionsIndex's one-shared-file-
|
|
704
|
+
// per-project shape; not per-correction-id — see writeCorrection's note).
|
|
705
|
+
return withLock(`corrections-${project}`, () => {
|
|
706
|
+
// Find the correction record by id
|
|
707
|
+
const all = readCorrections(project);
|
|
708
|
+
const target = all.find((r) => r.id === id);
|
|
709
|
+
if (!target) {
|
|
710
|
+
return { success: false, error: `correction not found: ${id}` };
|
|
711
|
+
}
|
|
712
|
+
const updated = {
|
|
713
|
+
...target,
|
|
714
|
+
active: false,
|
|
715
|
+
retracted_at: new Date().toISOString(),
|
|
716
|
+
...(reason !== undefined ? { retract_reason: reason } : {}),
|
|
717
|
+
// P2: forward pointer to the correction that replaced this one (audit trail).
|
|
718
|
+
...(supersededBy !== undefined ? { superseded_by: supersededBy } : {}),
|
|
719
|
+
};
|
|
720
|
+
// Rewrite — reuse the EXISTING file's name (see findExistingCorrectionFile
|
|
721
|
+
// doc); never recompute via slugify, or a v1-named file would orphan.
|
|
722
|
+
const filename = findExistingCorrectionFile(dir, updated.id)
|
|
723
|
+
?? `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
|
|
724
|
+
const filepath = path.join(dir, filename);
|
|
725
|
+
// Atomic rewrite — tmp + rename, mode 0600
|
|
726
|
+
writeRecordAtomic(filepath, updated);
|
|
727
|
+
// W2-1: regenerate the materialized index on every corrections mutation.
|
|
728
|
+
regenerateCorrectionsIndex(project);
|
|
729
|
+
return { success: true, id };
|
|
730
|
+
});
|
|
507
731
|
}
|
|
508
732
|
/**
|
|
509
733
|
* Record an outcome event for a correction (retrieved / heeded / recurred).
|
|
@@ -546,69 +770,443 @@ export function recordOutcome(outcome) {
|
|
|
546
770
|
outcome.kind === "unknown") {
|
|
547
771
|
return;
|
|
548
772
|
}
|
|
549
|
-
//
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
773
|
+
// ── LOCKED critical section (P0 data-loss fix, 2026-07-25) ────────────────
|
|
774
|
+
// This is the exact read-all→find→mutate→atomic-rewrite→index-regen span the
|
|
775
|
+
// audit reproduced (24 procs × 20 calls losing 66-78% of retrieved_count
|
|
776
|
+
// increments): readCorrections() below re-reads every *.json file, so two
|
|
777
|
+
// concurrent processes can both read the SAME stale counter value, both
|
|
778
|
+
// increment it by one, and the last atomic rewrite to land silently discards
|
|
779
|
+
// the other's increment. Locked with `corrections-${project}` — PROJECT-
|
|
780
|
+
// scoped, not per-correction-id, because regenerateCorrectionsIndex rewrites
|
|
781
|
+
// ONE shared _index.md per project (a per-id lock would still let two
|
|
782
|
+
// DIFFERENT corrections' index writes race). See the matching note in
|
|
783
|
+
// writeCorrection/retractCorrection above; same lock name, same pattern as
|
|
784
|
+
// `palace-index-${project}` / `digest-${project}` elsewhere in this codebase.
|
|
785
|
+
withLock(`corrections-${outcome.project}`, () => {
|
|
786
|
+
// Update the per-correction file's counters.
|
|
787
|
+
const target = readCorrections(outcome.project).find((r) => r.id === outcome.correction_id);
|
|
788
|
+
if (!target)
|
|
789
|
+
return; // outcome can still be replayed later if record is restored
|
|
790
|
+
const updated = {
|
|
791
|
+
...target,
|
|
792
|
+
retrieved_count: target.retrieved_count ?? 0,
|
|
793
|
+
heeded_count: target.heeded_count ?? 0,
|
|
794
|
+
recurrence_count: target.recurrence_count ?? 0,
|
|
795
|
+
};
|
|
796
|
+
if (outcome.kind === "retrieved") {
|
|
797
|
+
updated.retrieved_count = (updated.retrieved_count ?? 0) + 1;
|
|
798
|
+
updated.last_retrieved = outcome.at;
|
|
799
|
+
}
|
|
800
|
+
else if (outcome.kind === "heeded") {
|
|
801
|
+
updated.heeded_count = (updated.heeded_count ?? 0) + 1;
|
|
802
|
+
updated.last_outcome = outcome.at;
|
|
803
|
+
}
|
|
804
|
+
else if (outcome.kind === "recurred") {
|
|
805
|
+
updated.recurrence_count = (updated.recurrence_count ?? 0) + 1;
|
|
806
|
+
updated.last_outcome = outcome.at;
|
|
807
|
+
}
|
|
808
|
+
else if (outcome.kind === "predicted") {
|
|
809
|
+
// Wave 5: prediction fired — instrument the predict-the-correction loop.
|
|
810
|
+
updated.predicted_count = (updated.predicted_count ?? 0) + 1;
|
|
811
|
+
updated.last_predicted = outcome.at;
|
|
812
|
+
}
|
|
813
|
+
else if (outcome.kind === "predict_hit") {
|
|
814
|
+
updated.predict_hits = (updated.predict_hits ?? 0) + 1;
|
|
815
|
+
}
|
|
816
|
+
const r = updated.retrieved_count ?? 0;
|
|
817
|
+
// Clamp to [0,1]: `retrieved` is guarded 1/day but `heeded` can fire on every
|
|
818
|
+
// session_end, so raw heeded/retrieved can exceed 1.0 ("150% heeded" is
|
|
819
|
+
// nonsense). min(1, …) keeps the metric honest. (Root-cause follow-up: apply
|
|
820
|
+
// the same 1/day guard to heeded as retrieved has, for finer resolution.)
|
|
821
|
+
// NB (Wave 5): `precision` is heeded/retrieved ONLY — predict_* never touch it.
|
|
822
|
+
updated.precision = r > 0 ? Math.min(1, Number(((updated.heeded_count ?? 0) / r).toFixed(3))) : undefined;
|
|
823
|
+
// Wave 5: predict_precision = predict_hits / predicted_count, kept SEPARATE
|
|
824
|
+
// from `precision`. Undefined until at least one prediction has fired.
|
|
825
|
+
// A predict_hit implies a prior prediction. If data is inconsistent (hits
|
|
826
|
+
// recorded without a matching predicted_count — e.g. migrated/corrupt records),
|
|
827
|
+
// floor the denominator at predict_hits so the metric stays VISIBLE and bounded
|
|
828
|
+
// rather than silently undefined while hits exist.
|
|
829
|
+
const pc = updated.predicted_count ?? 0;
|
|
830
|
+
const ph = updated.predict_hits ?? 0;
|
|
831
|
+
const predictDenom = Math.max(pc, ph);
|
|
832
|
+
updated.predict_precision = predictDenom > 0
|
|
833
|
+
? Math.min(1, Number((ph / predictDenom).toFixed(3)))
|
|
834
|
+
: undefined;
|
|
835
|
+
// P3: evidence-grounded proof_confidence. With NO outcome evidence yet, keep the
|
|
836
|
+
// authority prior (weight); once heeded/recurrence accrue, move to the Beta
|
|
837
|
+
// posterior so a rule that keeps being honored strengthens and one whose bug
|
|
838
|
+
// keeps recurring weakens. Kept SEPARATE from `precision` (heeded/retrieved) and
|
|
839
|
+
// from `weight` (static authority) — this is the evidence axis.
|
|
840
|
+
const heededC = updated.heeded_count ?? 0;
|
|
841
|
+
const recurC = updated.recurrence_count ?? 0;
|
|
842
|
+
updated.proof_confidence = (heededC + recurC) > 0
|
|
843
|
+
? Number(betaPosterior(heededC, recurC).toFixed(3))
|
|
844
|
+
: (updated.weight ?? defaultWeight(updated.severity));
|
|
845
|
+
// Re-write the JSON file atomically (tmp + rename — prevents truncation on
|
|
846
|
+
// SIGTERM). Reuse the EXISTING filename (see findExistingCorrectionFile
|
|
847
|
+
// doc) — never recompute via slugify, or a v1-named file would orphan.
|
|
848
|
+
const filename = findExistingCorrectionFile(dir, updated.id)
|
|
849
|
+
?? `${updated.date}--${slugify(updated.rule || updated.id)}.json`;
|
|
850
|
+
const filepath = path.join(dir, filename);
|
|
851
|
+
writeRecordAtomic(filepath, updated);
|
|
852
|
+
// W2-1: regenerate the materialized index on every corrections mutation.
|
|
853
|
+
// (The ledger-only early-returns above for triggered/not_triggered/unknown
|
|
854
|
+
// touch no per-record field the index renders — severity/status/date/rule/
|
|
855
|
+
// failure_class are all unchanged — so they deliberately skip this call,
|
|
856
|
+
// preserving the existing hot-path optimization documented above.)
|
|
857
|
+
regenerateCorrectionsIndex(outcome.project);
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
/** Outcome kinds that affect per-correction counters (mirrors recordOutcome's early-return for triggered/not_triggered/unknown — those are ledger-only). */
|
|
861
|
+
const COUNTER_KINDS = new Set([
|
|
862
|
+
"retrieved",
|
|
863
|
+
"heeded",
|
|
864
|
+
"recurred",
|
|
865
|
+
"predicted",
|
|
866
|
+
"predict_hit",
|
|
867
|
+
]);
|
|
868
|
+
/**
|
|
869
|
+
* Replay-order key for one ledger event.
|
|
870
|
+
*
|
|
871
|
+
* `recorded_at` (forensic, C3b) is the wall-clock instant recordOutcome() was
|
|
872
|
+
* actually CALLED — i.e. it IS the live call order. `at` (semantic) can be
|
|
873
|
+
* backdated by the dream-audit path, so two events can have `at` values out of
|
|
874
|
+
* order relative to when they were really recorded (a nightly dream-audit call
|
|
875
|
+
* backdates `at` to yesterday but is physically called well AFTER today's live
|
|
876
|
+
* calls). recordOutcome has NO "is this newer" guard — every relevant call
|
|
877
|
+
* unconditionally OVERWRITES last_retrieved/last_outcome/last_predicted with
|
|
878
|
+
* `outcome.at`. So the only way a replay reproduces bit-identical results to a
|
|
879
|
+
* lossless LIVE call sequence is to apply events in the order the CALLS
|
|
880
|
+
* happened (recorded_at), each one stamping its own `.at` value — NOT in
|
|
881
|
+
* semantic-day order. Falls back to `at` only for pre-C3b lines that lack
|
|
882
|
+
* `recorded_at` entirely: those lines predate the dream-audit backdating
|
|
883
|
+
* feature (C3b introduced both `recorded_at` AND backdating together), so for
|
|
884
|
+
* them `at` already WAS the wall-clock time — a safe, non-arbitrary fallback.
|
|
885
|
+
*/
|
|
886
|
+
function replayOrderKey(evt) {
|
|
887
|
+
const ts = evt.recorded_at ?? evt.at;
|
|
888
|
+
const t = new Date(ts).getTime();
|
|
889
|
+
return Number.isNaN(t) ? 0 : t;
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* Sort events into live-call order for replay. Array.prototype.sort is stable
|
|
893
|
+
* (guaranteed since ES2019 / all supported Node versions), so ties — identical
|
|
894
|
+
* recorded_at, or both timestamps missing/invalid — keep original ledger line
|
|
895
|
+
* order, the next-best tie-break absent finer-grained sequencing.
|
|
896
|
+
*/
|
|
897
|
+
function sortForReplay(events) {
|
|
898
|
+
return [...events]
|
|
899
|
+
.map((evt, idx) => ({ evt, idx }))
|
|
900
|
+
.sort((a, b) => replayOrderKey(a.evt) - replayOrderKey(b.evt) || a.idx - b.idx)
|
|
901
|
+
.map(({ evt }) => evt);
|
|
902
|
+
}
|
|
903
|
+
/**
|
|
904
|
+
* Pure replay of one correction's counter-affecting ledger events, already
|
|
905
|
+
* ordered by `sortForReplay`. Recomputes every field recordOutcome derives,
|
|
906
|
+
* applying the EXACT SAME per-event formulas (see recordOutcome, this file,
|
|
907
|
+
* ~L1032-1084) one event at a time — this is what makes the rebuild produce
|
|
908
|
+
* IDENTICAL results to a lossless live call sequence.
|
|
909
|
+
*
|
|
910
|
+
* `record` supplies ONLY the weight/severity fallback `proof_confidence` needs
|
|
911
|
+
* when no heeded/recurrence evidence exists yet (recordOutcome's own
|
|
912
|
+
* `updated.weight ?? defaultWeight(updated.severity)` fallback, L1084) — its
|
|
913
|
+
* counter fields are never read here; those are recomputed from scratch.
|
|
914
|
+
*
|
|
915
|
+
* Mirrors recordOutcome's asymmetry precisely: retrieved_count/heeded_count/
|
|
916
|
+
* recurrence_count are seeded to 0 as soon as ANY counter-kind event fires
|
|
917
|
+
* (recordOutcome's `{...target, retrieved_count: target.retrieved_count ?? 0, …}`
|
|
918
|
+
* runs unconditionally before the per-kind branch, L1032-1037), while
|
|
919
|
+
* predicted_count/predict_hits stay `undefined` until their OWN specific kind
|
|
920
|
+
* fires at least once (they are only ever seeded inline at their own
|
|
921
|
+
* increment site, L1049/L1052).
|
|
922
|
+
*/
|
|
923
|
+
export function recomputeCorrectionCounters(events, record) {
|
|
924
|
+
let retrieved;
|
|
925
|
+
let heeded;
|
|
926
|
+
let recurrence;
|
|
927
|
+
let predicted;
|
|
928
|
+
let predictHits;
|
|
929
|
+
let lastRetrieved;
|
|
930
|
+
let lastOutcome;
|
|
931
|
+
let lastPredicted;
|
|
932
|
+
for (const evt of events) {
|
|
933
|
+
if (!COUNTER_KINDS.has(evt.kind))
|
|
934
|
+
continue; // ledger-only — mirror recordOutcome's early return (L1007-1013)
|
|
935
|
+
// Unconditional baseline seed on EVERY counter-kind event (recordOutcome L1032-1037).
|
|
936
|
+
retrieved = retrieved ?? 0;
|
|
937
|
+
heeded = heeded ?? 0;
|
|
938
|
+
recurrence = recurrence ?? 0;
|
|
939
|
+
if (evt.kind === "retrieved") {
|
|
940
|
+
retrieved = retrieved + 1;
|
|
941
|
+
lastRetrieved = evt.at;
|
|
942
|
+
}
|
|
943
|
+
else if (evt.kind === "heeded") {
|
|
944
|
+
heeded = heeded + 1;
|
|
945
|
+
lastOutcome = evt.at;
|
|
946
|
+
}
|
|
947
|
+
else if (evt.kind === "recurred") {
|
|
948
|
+
recurrence = recurrence + 1;
|
|
949
|
+
lastOutcome = evt.at;
|
|
950
|
+
}
|
|
951
|
+
else if (evt.kind === "predicted") {
|
|
952
|
+
predicted = (predicted ?? 0) + 1;
|
|
953
|
+
lastPredicted = evt.at;
|
|
954
|
+
}
|
|
955
|
+
else if (evt.kind === "predict_hit") {
|
|
956
|
+
predictHits = (predictHits ?? 0) + 1;
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
// precision — recordOutcome L1054/L1060.
|
|
960
|
+
const r = retrieved ?? 0;
|
|
961
|
+
const precision = r > 0 ? Math.min(1, Number(((heeded ?? 0) / r).toFixed(3))) : undefined;
|
|
962
|
+
// predict_precision — recordOutcome L1068-1073 (denominator floored at
|
|
963
|
+
// predict_hits when predicted_count is inconsistent/missing).
|
|
964
|
+
const pc = predicted ?? 0;
|
|
965
|
+
const ph = predictHits ?? 0;
|
|
966
|
+
const predictDenom = Math.max(pc, ph);
|
|
967
|
+
const predictPrecision = predictDenom > 0 ? Math.min(1, Number((ph / predictDenom).toFixed(3))) : undefined;
|
|
968
|
+
// proof_confidence — recordOutcome L1080-1084.
|
|
969
|
+
const heededC = heeded ?? 0;
|
|
970
|
+
const recurC = recurrence ?? 0;
|
|
971
|
+
const proofConfidence = heededC + recurC > 0
|
|
972
|
+
? Number(betaPosterior(heededC, recurC).toFixed(3))
|
|
973
|
+
: (record.weight ?? defaultWeight(record.severity));
|
|
974
|
+
return {
|
|
975
|
+
retrieved_count: retrieved,
|
|
976
|
+
heeded_count: heeded,
|
|
977
|
+
recurrence_count: recurrence,
|
|
978
|
+
predicted_count: predicted,
|
|
979
|
+
predict_hits: predictHits,
|
|
980
|
+
last_retrieved: lastRetrieved,
|
|
981
|
+
last_outcome: lastOutcome,
|
|
982
|
+
last_predicted: lastPredicted,
|
|
983
|
+
precision,
|
|
984
|
+
predict_precision: predictPrecision,
|
|
985
|
+
proof_confidence: proofConfidence,
|
|
558
986
|
};
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
987
|
+
}
|
|
988
|
+
/**
|
|
989
|
+
* Bulk-read + split _outcomes.jsonl into per-correction event lists, quarantining
|
|
990
|
+
* malformed/shapeless lines instead of silently dropping them (unlike
|
|
991
|
+
* bucketOutcomesBy above, whose callers only need coarse kind-presence and
|
|
992
|
+
* already tolerate silent skips).
|
|
993
|
+
*
|
|
994
|
+
* BULK READ, not streamed: this mirrors bucketOutcomesBy's existing pattern in
|
|
995
|
+
* this exact file (fs.readFileSync + split("\n")) and keeps the doctor check
|
|
996
|
+
* that shares this function fully synchronous — store-doctor.ts's public API
|
|
997
|
+
* (runStoreDoctor) is synchronous today and is called from the session_start
|
|
998
|
+
* hot path; switching to a streaming reader would force it (and every caller)
|
|
999
|
+
* async, a much wider blast radius than this repair tool's scope. Acceptable
|
|
1000
|
+
* per-project: _outcomes.jsonl is one project's correction-outcome history,
|
|
1001
|
+
* not a global log.
|
|
1002
|
+
*/
|
|
1003
|
+
function parseOutcomesLedger(project) {
|
|
1004
|
+
const malformedRows = [];
|
|
1005
|
+
const eventsByCorrectionId = new Map();
|
|
1006
|
+
const p = outcomesPath(project);
|
|
1007
|
+
if (!fs.existsSync(p))
|
|
1008
|
+
return { malformedRows, eventsByCorrectionId };
|
|
1009
|
+
let raw;
|
|
1010
|
+
try {
|
|
1011
|
+
raw = fs.readFileSync(p, "utf-8");
|
|
562
1012
|
}
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
1013
|
+
catch (err) {
|
|
1014
|
+
malformedRows.push({
|
|
1015
|
+
line: 0,
|
|
1016
|
+
raw: "",
|
|
1017
|
+
error: `failed to read ledger: ${err instanceof Error ? err.message : String(err)}`,
|
|
1018
|
+
});
|
|
1019
|
+
return { malformedRows, eventsByCorrectionId };
|
|
566
1020
|
}
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
1021
|
+
const lines = raw.split("\n");
|
|
1022
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1023
|
+
const lineNo = i + 1;
|
|
1024
|
+
const trimmed = lines[i].trim();
|
|
1025
|
+
if (!trimmed)
|
|
1026
|
+
continue; // blank line — not malformed, just skip
|
|
1027
|
+
let evt;
|
|
1028
|
+
try {
|
|
1029
|
+
evt = JSON.parse(trimmed);
|
|
1030
|
+
}
|
|
1031
|
+
catch (err) {
|
|
1032
|
+
malformedRows.push({
|
|
1033
|
+
line: lineNo,
|
|
1034
|
+
raw: lines[i],
|
|
1035
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1036
|
+
});
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
1039
|
+
if (!evt || typeof evt !== "object" || !evt.correction_id || !evt.kind || !evt.at) {
|
|
1040
|
+
malformedRows.push({
|
|
1041
|
+
line: lineNo,
|
|
1042
|
+
raw: lines[i],
|
|
1043
|
+
error: "parsed but missing required field(s): correction_id, kind, at",
|
|
1044
|
+
});
|
|
1045
|
+
continue;
|
|
1046
|
+
}
|
|
1047
|
+
let arr = eventsByCorrectionId.get(evt.correction_id);
|
|
1048
|
+
if (!arr) {
|
|
1049
|
+
arr = [];
|
|
1050
|
+
eventsByCorrectionId.set(evt.correction_id, arr);
|
|
1051
|
+
}
|
|
1052
|
+
arr.push(evt);
|
|
570
1053
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
1054
|
+
return { malformedRows, eventsByCorrectionId };
|
|
1055
|
+
}
|
|
1056
|
+
/** Snapshot of a correction's CURRENT on-disk counter/derived fields (before any rebuild). */
|
|
1057
|
+
function currentCountersOf(r) {
|
|
1058
|
+
return {
|
|
1059
|
+
retrieved_count: r.retrieved_count,
|
|
1060
|
+
heeded_count: r.heeded_count,
|
|
1061
|
+
recurrence_count: r.recurrence_count,
|
|
1062
|
+
predicted_count: r.predicted_count,
|
|
1063
|
+
predict_hits: r.predict_hits,
|
|
1064
|
+
last_retrieved: r.last_retrieved,
|
|
1065
|
+
last_outcome: r.last_outcome,
|
|
1066
|
+
last_predicted: r.last_predicted,
|
|
1067
|
+
precision: r.precision,
|
|
1068
|
+
predict_precision: r.predict_precision,
|
|
1069
|
+
proof_confidence: r.proof_confidence,
|
|
1070
|
+
};
|
|
1071
|
+
}
|
|
1072
|
+
const COUNTER_FIELD_NAMES = [
|
|
1073
|
+
"retrieved_count",
|
|
1074
|
+
"heeded_count",
|
|
1075
|
+
"recurrence_count",
|
|
1076
|
+
"predicted_count",
|
|
1077
|
+
"predict_hits",
|
|
1078
|
+
"last_retrieved",
|
|
1079
|
+
"last_outcome",
|
|
1080
|
+
"last_predicted",
|
|
1081
|
+
"precision",
|
|
1082
|
+
"predict_precision",
|
|
1083
|
+
"proof_confidence",
|
|
1084
|
+
];
|
|
1085
|
+
function countersEqual(a, b) {
|
|
1086
|
+
return COUNTER_FIELD_NAMES.every((k) => a[k] === b[k]);
|
|
1087
|
+
}
|
|
1088
|
+
/**
|
|
1089
|
+
* Shared replay-and-compare core for a single project. Read-only — never
|
|
1090
|
+
* mutates, never locks. For every on-disk correction that has at least one
|
|
1091
|
+
* counter-affecting ledger event, replays that event history from scratch
|
|
1092
|
+
* (sortForReplay + recomputeCorrectionCounters) and compares the result
|
|
1093
|
+
* against what's CURRENTLY materialized on disk.
|
|
1094
|
+
*
|
|
1095
|
+
* Corrections with ZERO ledger events for their id are deliberately EXCLUDED
|
|
1096
|
+
* from the result — there is no ledger evidence to rebuild FROM, so "recompute
|
|
1097
|
+
* from scratch" would mean zeroing out counters that might legitimately have
|
|
1098
|
+
* come from a source other than this project's _outcomes.jsonl (e.g. a very
|
|
1099
|
+
* old pre-ledger record). Rebuild only ever touches ids the lossless ledger
|
|
1100
|
+
* can actually vouch for.
|
|
1101
|
+
*
|
|
1102
|
+
* This is the ONE implementation shared by `runOutcomesRebuild` (write-side,
|
|
1103
|
+
* corrections.ts) and store-doctor.ts's `outcomes_ledger_divergence` check
|
|
1104
|
+
* (read-only) — factored here so the two can never disagree.
|
|
1105
|
+
*/
|
|
1106
|
+
export function computeLedgerDivergence(project) {
|
|
1107
|
+
const { malformedRows, eventsByCorrectionId } = parseOutcomesLedger(project);
|
|
1108
|
+
const all = readCorrections(project);
|
|
1109
|
+
const entries = [];
|
|
1110
|
+
for (const record of all) {
|
|
1111
|
+
const raw = eventsByCorrectionId.get(record.id);
|
|
1112
|
+
if (!raw || raw.length === 0)
|
|
1113
|
+
continue;
|
|
1114
|
+
const relevant = raw.filter((e) => COUNTER_KINDS.has(e.kind));
|
|
1115
|
+
if (relevant.length === 0)
|
|
1116
|
+
continue;
|
|
1117
|
+
const ordered = sortForReplay(relevant);
|
|
1118
|
+
const after = recomputeCorrectionCounters(ordered, record);
|
|
1119
|
+
const before = currentCountersOf(record);
|
|
1120
|
+
entries.push({ id: record.id, record, before, after, changed: !countersEqual(before, after) });
|
|
575
1121
|
}
|
|
576
|
-
|
|
577
|
-
|
|
1122
|
+
return { malformedRows, entries };
|
|
1123
|
+
}
|
|
1124
|
+
function toPublicDiffs(entries) {
|
|
1125
|
+
return entries.map(({ id, before, after, changed }) => ({ id, before, after, changed }));
|
|
1126
|
+
}
|
|
1127
|
+
/**
|
|
1128
|
+
* Rebuild a project's materialized outcome counters from a full replay of the
|
|
1129
|
+
* (lossless) _outcomes.jsonl ledger — the WRITE-side repair for records
|
|
1130
|
+
* corrupted by the pre-05b3699 unlocked read-modify-write in recordOutcome.
|
|
1131
|
+
*
|
|
1132
|
+
* SAFETY INVARIANTS (mirrors store-repair.ts's StoreRepairOptions contract):
|
|
1133
|
+
* 1. DRY-RUN BY DEFAULT. `opts.apply` must be EXPLICITLY true to mutate.
|
|
1134
|
+
* 2. IDEMPOTENT. A second apply run recomputes from the SAME lossless ledger
|
|
1135
|
+
* and disk now matches it, so every entry's `changed` is false and
|
|
1136
|
+
* nothing is rewritten (not even the index).
|
|
1137
|
+
* 3. LOCK SAFETY. The apply pass runs inside `withLock(\`corrections-${project}\`,
|
|
1138
|
+
* …)` — the SAME lock name recordOutcome/retractCorrection/writeCorrection
|
|
1139
|
+
* use (05b3699) — so a rebuild-apply can never race a live recordOutcome
|
|
1140
|
+
* call for the same project.
|
|
1141
|
+
* 4. Only per-correction JSON files that actually differ are rewritten; the
|
|
1142
|
+
* shared _index.md is regenerated ONCE at the end, only if anything wrote.
|
|
1143
|
+
*/
|
|
1144
|
+
export function runOutcomesRebuild(project, opts = {}) {
|
|
1145
|
+
const apply = opts.apply === true;
|
|
1146
|
+
if (!apply) {
|
|
1147
|
+
const plan = computeLedgerDivergence(project);
|
|
1148
|
+
const corrections = toPublicDiffs(plan.entries);
|
|
1149
|
+
return {
|
|
1150
|
+
apply: false,
|
|
1151
|
+
malformedRows: plan.malformedRows,
|
|
1152
|
+
corrections,
|
|
1153
|
+
summary: {
|
|
1154
|
+
totalCorrections: corrections.length,
|
|
1155
|
+
changed: corrections.filter((c) => c.changed).length,
|
|
1156
|
+
malformed: plan.malformedRows.length,
|
|
1157
|
+
},
|
|
1158
|
+
};
|
|
578
1159
|
}
|
|
579
|
-
|
|
580
|
-
//
|
|
581
|
-
//
|
|
582
|
-
//
|
|
583
|
-
//
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
1160
|
+
// ── LOCKED apply pass ──────────────────────────────────────────────────────
|
|
1161
|
+
// Re-derive the plan INSIDE the lock: a concurrent live recordOutcome call
|
|
1162
|
+
// could append new ledger lines or rewrite a correction file between any
|
|
1163
|
+
// earlier unlocked plan computation and acquiring this lock. Same lock name
|
|
1164
|
+
// as writeCorrection/retractCorrection/recordOutcome above.
|
|
1165
|
+
return withLock(`corrections-${project}`, () => {
|
|
1166
|
+
const dir = correctionsDir(project);
|
|
1167
|
+
const plan = computeLedgerDivergence(project);
|
|
1168
|
+
let wrote = 0;
|
|
1169
|
+
for (const entry of plan.entries) {
|
|
1170
|
+
if (!entry.changed)
|
|
1171
|
+
continue;
|
|
1172
|
+
const updated = {
|
|
1173
|
+
...entry.record,
|
|
1174
|
+
retrieved_count: entry.after.retrieved_count,
|
|
1175
|
+
heeded_count: entry.after.heeded_count,
|
|
1176
|
+
recurrence_count: entry.after.recurrence_count,
|
|
1177
|
+
predicted_count: entry.after.predicted_count,
|
|
1178
|
+
predict_hits: entry.after.predict_hits,
|
|
1179
|
+
last_retrieved: entry.after.last_retrieved,
|
|
1180
|
+
last_outcome: entry.after.last_outcome,
|
|
1181
|
+
last_predicted: entry.after.last_predicted,
|
|
1182
|
+
precision: entry.after.precision,
|
|
1183
|
+
predict_precision: entry.after.predict_precision,
|
|
1184
|
+
proof_confidence: entry.after.proof_confidence,
|
|
1185
|
+
};
|
|
1186
|
+
// Reuse the EXISTING file's name (see findExistingCorrectionFile doc) —
|
|
1187
|
+
// never recompute via slugify, matching every other rewrite in this file.
|
|
1188
|
+
const filename = findExistingCorrectionFile(dir, updated.id) ??
|
|
1189
|
+
`${updated.date}--${slugify(updated.rule || updated.id)}.json`;
|
|
1190
|
+
writeRecordAtomic(path.join(dir, filename), updated);
|
|
1191
|
+
wrote++;
|
|
1192
|
+
}
|
|
1193
|
+
// Regenerate the shared index ONCE, only if something actually changed —
|
|
1194
|
+
// keeps a clean second apply-run a true no-op (idempotency invariant #2).
|
|
1195
|
+
if (wrote > 0) {
|
|
1196
|
+
regenerateCorrectionsIndex(project);
|
|
1197
|
+
}
|
|
1198
|
+
const corrections = toPublicDiffs(plan.entries);
|
|
1199
|
+
return {
|
|
1200
|
+
apply: true,
|
|
1201
|
+
malformedRows: plan.malformedRows,
|
|
1202
|
+
corrections,
|
|
1203
|
+
summary: {
|
|
1204
|
+
totalCorrections: corrections.length,
|
|
1205
|
+
changed: wrote,
|
|
1206
|
+
malformed: plan.malformedRows.length,
|
|
1207
|
+
},
|
|
1208
|
+
};
|
|
1209
|
+
});
|
|
612
1210
|
}
|
|
613
1211
|
/**
|
|
614
1212
|
* Best-effort: append one row to corrections/_rejected.jsonl recording a
|
|
@@ -883,13 +1481,10 @@ export function listUnknownVerdicts(project, date) {
|
|
|
883
1481
|
// Resolve correction records
|
|
884
1482
|
const allCorrections = readCorrections(project);
|
|
885
1483
|
const recordById = new Map(allCorrections.map((r) => [r.id, r]));
|
|
886
|
-
// Resolve journal file paths for targetDay
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
.replace(/^-+|-+$/g, "")
|
|
891
|
-
.slice(0, 100) || "unnamed";
|
|
892
|
-
const jDir = path.join(root, "projects", safe, "journal");
|
|
1484
|
+
// Resolve journal file paths for targetDay — reuse paths.ts's journalDir
|
|
1485
|
+
// (was a local re-derivation of the old sanitizer, same divergence risk
|
|
1486
|
+
// fixed in correctionsDir above).
|
|
1487
|
+
const jDir = journalDir(project);
|
|
893
1488
|
const journalPaths = [];
|
|
894
1489
|
if (fs.existsSync(jDir)) {
|
|
895
1490
|
try {
|