@ngockhoale/ukit 3.0.8 → 3.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +18 -1
  2. package/manifests/documentation.yaml +11 -0
  3. package/package.json +1 -1
  4. package/scripts/audit/decision-coverage.mjs +29 -2
  5. package/scripts/bench/data-foundation.mjs +52 -3
  6. package/scripts/bench/decision-runtime-baseline.mjs +427 -0
  7. package/scripts/bench/decision-runtime-metrics.mjs +67 -0
  8. package/scripts/bench/decision-runtime-variant.mjs +626 -0
  9. package/scripts/bench/memory-ablation.mjs +495 -0
  10. package/scripts/bench/memory-baseline.mjs +596 -0
  11. package/scripts/bench/memory-bench.mjs +661 -0
  12. package/scripts/bench/memory-canary.mjs +321 -0
  13. package/scripts/bench/memory-corpus.mjs +354 -0
  14. package/scripts/bench/memory-gate.mjs +389 -0
  15. package/scripts/bench/memory-metrics.mjs +179 -0
  16. package/scripts/bench/parallel-agents.mjs +33 -11
  17. package/scripts/bench/recorder-overhead.mjs +204 -0
  18. package/scripts/bench/sqlite-spike.mjs +451 -0
  19. package/scripts/measure-decision-gateway.mjs +306 -0
  20. package/scripts/perf/audit-perf.mjs +35 -17
  21. package/src/bug/triageBug.js +4 -3
  22. package/src/cli/commands/memory.js +357 -63
  23. package/src/context/detectProjectContext.js +11 -1
  24. package/src/core/agentRuntime/adapters.js +254 -0
  25. package/src/core/agentRuntime/artifacts.js +192 -0
  26. package/src/core/agentRuntime/completionGate.js +176 -0
  27. package/src/core/agentRuntime/context.js +149 -0
  28. package/src/core/agentRuntime/contract.js +247 -0
  29. package/src/core/agentRuntime/diagnostics.js +244 -0
  30. package/src/core/agentRuntime/evaluation.js +163 -0
  31. package/src/core/agentRuntime/eventStore.js +404 -0
  32. package/src/core/agentRuntime/liveness.js +60 -0
  33. package/src/core/agentRuntime/planCompiler.js +322 -0
  34. package/src/core/agentRuntime/promotion.js +53 -0
  35. package/src/core/agentRuntime/qualityComparison.js +112 -0
  36. package/src/core/agentRuntime/recovery.js +266 -0
  37. package/src/core/agentRuntime/resourcePolicy.js +78 -0
  38. package/src/core/agentRuntime/runtimeSupport.js +237 -0
  39. package/src/core/agentRuntime/supervisor.js +565 -0
  40. package/src/core/agentRuntime/vmEngine.js +621 -0
  41. package/src/core/codeintel/analogy.js +3 -2
  42. package/src/core/experiments/dynamicWorkflow.js +17 -2
  43. package/src/core/fileOps.js +21 -3
  44. package/src/core/memory/deltaOverlays.js +75 -30
  45. package/src/core/memory/learningCandidates.js +93 -48
  46. package/src/core/memory/memoryFlags.js +83 -0
  47. package/src/core/memory/memoryFreshness.js +190 -0
  48. package/src/core/memory/memoryHit.js +144 -0
  49. package/src/core/memory/migrate.js +69 -189
  50. package/src/core/memory/migrateMapping.js +232 -0
  51. package/src/core/memory/mutateMemory.js +323 -0
  52. package/src/core/memory/policy.js +96 -0
  53. package/src/core/memory/projectIdentity.js +266 -0
  54. package/src/core/memory/recordIndex.js +178 -0
  55. package/src/core/memory/recordStore.js +133 -20
  56. package/src/core/memory/records.js +144 -6
  57. package/src/core/memory/retrieval.js +259 -125
  58. package/src/core/memory/store.js +16 -5
  59. package/src/core/memory/storeBackup.js +226 -0
  60. package/src/core/memory/storeV2.js +63 -26
  61. package/src/core/memory/storeV2Loader.js +30 -12
  62. package/src/core/memory/userMemory.js +38 -20
  63. package/src/core/memory/writeClassification.js +161 -0
  64. package/src/core/memory/writeGuard.js +129 -0
  65. package/src/core/observability/adapters/hookTelemetryAdapter.js +90 -0
  66. package/src/core/observability/analytics/cohorts.js +148 -0
  67. package/src/core/observability/analytics/storeDigest.js +163 -0
  68. package/src/core/observability/evaluation/experimentPlan.js +95 -0
  69. package/src/core/observability/evaluation/findings.js +99 -0
  70. package/src/core/observability/evaluation/optimizationKnowledge.js +10 -1
  71. package/src/core/observability/evaluation/perturbation.js +273 -0
  72. package/src/core/observability/evaluation/replay.js +7 -1
  73. package/src/core/observability/evaluation/scorecard.js +23 -3
  74. package/src/core/observability/rollout.js +11 -7
  75. package/src/core/observability/schema/compatibility.js +135 -0
  76. package/src/core/observability/schema/registry.js +99 -0
  77. package/src/core/observability/schema/validate.js +7 -0
  78. package/src/core/observability/support/import.js +53 -9
  79. package/src/core/observability/support/paths.js +13 -3
  80. package/src/core/observability/support/projector.js +148 -12
  81. package/src/core/output/index.js +12 -2
  82. package/src/core/runtimeConfig.js +83 -0
  83. package/src/core/runtimePaths.js +3 -0
  84. package/src/core/sensitiveValueScanner.js +40 -0
  85. package/src/core/token/index.js +40 -3
  86. package/src/decision/client.js +37 -13
  87. package/src/decision/protocol.js +1 -1
  88. package/src/decision/registry.js +5 -3
  89. package/src/decision/runtimeDecide.js +242 -0
  90. package/src/decision/runtimeFilter.js +150 -0
  91. package/src/decision/runtimeScheduler.js +239 -0
  92. package/src/index/buildIndex.js +13 -12
  93. package/src/index/queryIndex.js +35 -14
  94. package/src/index/relatedTests.js +50 -8
  95. package/src/index/resolveContext.js +9 -4
  96. package/src/manifest/selectItems.js +7 -3
  97. package/src/render/instructionRenderer.js +17 -5
  98. package/template_project/.claude/ukit/index/lib/index-core.mjs +94 -39
  99. package/template_project/.claude/ukit/index/route-task.mjs +121 -19
  100. package/template_project/.claude/ukit/index/unic-decision.mjs +28 -13
  101. package/template_project/.claude/ukit/runtime/memory-flags.mjs +51 -0
  102. package/template_project/.claude/ukit/runtime/memory-freshness.mjs +155 -0
  103. package/template_project/.claude/ukit/runtime/memory-policy.mjs +286 -0
  104. package/template_project/.claude/ukit/runtime/output-compression.mjs +3 -0
  105. package/template_project/.claude/ukit/runtime/reinject-context.mjs +145 -14
@@ -0,0 +1,266 @@
1
+ import fs from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import crypto from 'node:crypto';
4
+ import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
5
+ import { buildUserPaths } from '../userPaths.js';
6
+
7
+ /**
8
+ * Host-owned stable project identity resolver (M01-01, SPEC §5 FR-001..003).
9
+ *
10
+ * Identity authority is a USER-LEVEL registry at
11
+ * `~/.ukit/storage/projects/registry.json` — user-level because uninstall
12
+ * removes the project `.ukit/` tree. `package.json.name` is never an identity
13
+ * authority; it only feeds the alias index (display-name collisions degrade
14
+ * to `ambiguous`, never a guess).
15
+ *
16
+ * Registry shape (SPEC §7):
17
+ * {schemaVersion:1,
18
+ * projects:{<projectId>:{roots:[realpath], gitCommonDirs:[abs],
19
+ * aliases:[name], createdAt, updatedAt}},
20
+ * aliasIndex:{<name>:[<projectId>|'ambiguous']}}
21
+ */
22
+
23
+ const REGISTRY_SCHEMA_VERSION = 1;
24
+
25
+ function registryPathFor(homeDir) {
26
+ const { storageRoot } = buildUserPaths(homeDir === undefined ? {} : { homeDir });
27
+ return path.join(storageRoot, 'projects', 'registry.json');
28
+ }
29
+
30
+ function emptyRegistry() {
31
+ return { schemaVersion: REGISTRY_SCHEMA_VERSION, projects: {}, aliasIndex: {} };
32
+ }
33
+
34
+ /**
35
+ * Tolerant registry read. Missing → empty registry. Corrupt → null sentinel
36
+ * (caller degrades to `unverified`; the file is NEVER overwritten on read).
37
+ */
38
+ async function readRegistry(regPath) {
39
+ let data;
40
+ try {
41
+ data = await readJsonIfExists(regPath);
42
+ } catch {
43
+ return null; // corrupt JSON — degrade, never throw, never rewrite
44
+ }
45
+ if (data === null) return emptyRegistry();
46
+ if (typeof data !== 'object' || typeof data.projects !== 'object' || data.projects === null) {
47
+ return null;
48
+ }
49
+ if (typeof data.aliasIndex !== 'object' || data.aliasIndex === null) {
50
+ data.aliasIndex = {};
51
+ }
52
+ return data;
53
+ }
54
+
55
+ async function realpathOrNull(target) {
56
+ try {
57
+ return await fs.realpath(target);
58
+ } catch {
59
+ return null;
60
+ }
61
+ }
62
+
63
+ /**
64
+ * Resolve the git common dir for a root: `.git` dir → itself; `.git` file
65
+ * (`gitdir: <p>`) → `<p>/commondir` (linked worktrees share the main common
66
+ * dir). Returns null when there is no git binding.
67
+ */
68
+ async function resolveGitCommonDir(projectRoot) {
69
+ const gitPath = path.join(projectRoot, '.git');
70
+ let stat;
71
+ try {
72
+ stat = await fs.stat(gitPath);
73
+ } catch {
74
+ return null;
75
+ }
76
+
77
+ if (stat.isDirectory()) {
78
+ return await realpathOrNull(gitPath);
79
+ }
80
+ if (!stat.isFile()) return null;
81
+
82
+ let content;
83
+ try {
84
+ content = await fs.readFile(gitPath, 'utf8');
85
+ } catch {
86
+ return null;
87
+ }
88
+ const match = content.match(/^gitdir:\s*(.+)$/m);
89
+ if (!match) return null;
90
+ const gitDir = path.resolve(projectRoot, match[1].trim());
91
+
92
+ let commonDirRel;
93
+ try {
94
+ commonDirRel = (await fs.readFile(path.join(gitDir, 'commondir'), 'utf8')).trim();
95
+ } catch {
96
+ commonDirRel = '';
97
+ }
98
+ const commonDir = commonDirRel ? path.resolve(gitDir, commonDirRel) : gitDir;
99
+ return await realpathOrNull(commonDir);
100
+ }
101
+
102
+ async function aliasesFor(projectRoot, realRoot) {
103
+ const names = new Set();
104
+ const base = path.basename(realRoot ?? projectRoot);
105
+ if (base) names.add(base);
106
+ const pkg = await readJsonIfExists(path.join(projectRoot, 'package.json')).catch(() => null);
107
+ if (typeof pkg?.name === 'string' && pkg.name.trim() !== '') {
108
+ names.add(pkg.name.trim());
109
+ }
110
+ return [...names];
111
+ }
112
+
113
+ function findByRoot(registry, realRoot) {
114
+ for (const [projectId, entry] of Object.entries(registry.projects)) {
115
+ if (Array.isArray(entry?.roots) && entry.roots.includes(realRoot)) {
116
+ return projectId;
117
+ }
118
+ }
119
+ return null;
120
+ }
121
+
122
+ function findByGitCommonDir(registry, commonDir) {
123
+ if (!commonDir) return null;
124
+ for (const [projectId, entry] of Object.entries(registry.projects)) {
125
+ if (Array.isArray(entry?.gitCommonDirs) && entry.gitCommonDirs.includes(commonDir)) {
126
+ return projectId;
127
+ }
128
+ }
129
+ return null;
130
+ }
131
+
132
+ function findByAlias(registry, aliases) {
133
+ const hits = new Set();
134
+ for (const name of aliases) {
135
+ const entry = registry.aliasIndex?.[name];
136
+ if (entry === 'ambiguous') {
137
+ hits.add('ambiguous');
138
+ continue;
139
+ }
140
+ if (Array.isArray(entry)) {
141
+ for (const id of entry) hits.add(id);
142
+ } else if (typeof entry === 'string' && entry) {
143
+ hits.add(entry);
144
+ }
145
+ }
146
+ return hits;
147
+ }
148
+
149
+ function mintProjectId(realRoot) {
150
+ const digest = crypto
151
+ .createHash('sha256')
152
+ .update(`${realRoot}|${crypto.randomBytes(16).toString('hex')}`)
153
+ .digest('hex');
154
+ return `prj_${digest.slice(0, 16)}`;
155
+ }
156
+
157
+ function indexAliases(registry, projectId, aliases) {
158
+ for (const name of aliases) {
159
+ const current = registry.aliasIndex[name];
160
+ if (current === 'ambiguous') continue;
161
+ const list = Array.isArray(current) ? current : (typeof current === 'string' && current ? [current] : []);
162
+ if (!list.includes(projectId)) list.push(projectId);
163
+ registry.aliasIndex[name] = list.length > 1 ? 'ambiguous' : list;
164
+ }
165
+ }
166
+
167
+ /**
168
+ * Resolve the stable project identity for `projectRoot`.
169
+ *
170
+ * Order: (1) canonical realpath in registry roots; (2) git common-dir binding
171
+ * (linked worktree → same project); (3) alias index — multiple distinct
172
+ * projectIds → ambiguous deny; (4) mint (default) under file lock, or
173
+ * `unverified` when `mint:false`.
174
+ *
175
+ * @returns {Promise<{projectId: string|null, source: 'registry'|'git-common-dir'|'alias'|'unverified', ambiguous: boolean}>}
176
+ */
177
+ export async function resolveProjectIdentity(projectRoot, { homeDir, mint = true } = {}) {
178
+ const unverified = { projectId: null, source: 'unverified', ambiguous: false };
179
+ const regPath = registryPathFor(homeDir);
180
+
181
+ const registry = await readRegistry(regPath);
182
+ if (registry === null) return unverified; // corrupt — never overwrite on read
183
+
184
+ const realRoot = (await realpathOrNull(projectRoot)) ?? path.resolve(projectRoot);
185
+
186
+ const byRoot = findByRoot(registry, realRoot);
187
+ if (byRoot) {
188
+ return {
189
+ projectId: byRoot,
190
+ source: 'registry',
191
+ ambiguous: false,
192
+ aliases: Array.isArray(registry.projects[byRoot]?.aliases)
193
+ ? registry.projects[byRoot].aliases.filter((a) => typeof a === 'string')
194
+ : [],
195
+ };
196
+ }
197
+
198
+ const commonDir = await resolveGitCommonDir(projectRoot);
199
+ const byGit = findByGitCommonDir(registry, commonDir);
200
+ if (byGit) {
201
+ return {
202
+ projectId: byGit,
203
+ source: 'git-common-dir',
204
+ ambiguous: false,
205
+ aliases: Array.isArray(registry.projects[byGit]?.aliases)
206
+ ? registry.projects[byGit].aliases.filter((a) => typeof a === 'string')
207
+ : [],
208
+ };
209
+ }
210
+
211
+ const aliases = await aliasesFor(projectRoot, realRoot);
212
+ // Alias is a weak display-name signal only: any hit for a different root
213
+ // denies (ambiguous) rather than letting a same-named repo inherit an id.
214
+ const aliasHits = findByAlias(registry, aliases);
215
+ if (aliasHits.size > 0) {
216
+ return { projectId: null, source: 'alias', ambiguous: true };
217
+ }
218
+
219
+ if (!mint) return unverified;
220
+
221
+ // Mint under the registry file lock; re-check inside the critical section so
222
+ // a concurrent minter cannot produce two ids for the same root.
223
+ const minted = await withFileLock(regPath, async () => {
224
+ const fresh = await readRegistry(regPath);
225
+ if (fresh === null) return null; // became corrupt between read and lock
226
+ const existing = findByRoot(fresh, realRoot)
227
+ ?? (commonDir ? findByGitCommonDir(fresh, commonDir) : null);
228
+ if (existing) return existing;
229
+
230
+ const now = new Date().toISOString();
231
+ const projectId = mintProjectId(realRoot);
232
+ fresh.projects[projectId] = {
233
+ roots: [realRoot],
234
+ gitCommonDirs: commonDir ? [commonDir] : [],
235
+ aliases,
236
+ createdAt: now,
237
+ updatedAt: now,
238
+ };
239
+ indexAliases(fresh, projectId, aliases);
240
+ await writeJson(regPath, fresh);
241
+ return projectId;
242
+ });
243
+
244
+ if (minted === undefined || minted === null) return unverified;
245
+ return { projectId: minted, source: 'registry', ambiguous: false, aliases };
246
+ }
247
+
248
+ /**
249
+ * lookupRegisteredProjectId(legacyId, { homeDir } = {}) → string | 'ambiguous' | null
250
+ * Read-only registry lookup for migration alias-collision checks (M04-01):
251
+ * a legacy project id that IS a registered projectId resolves to itself;
252
+ * an alias resolves to its bound projectId; 'ambiguous' when the alias is
253
+ * contested; null when unknown. Never mints, never writes.
254
+ */
255
+ export async function lookupRegisteredProjectId(legacyId, { homeDir } = {}) {
256
+ if (typeof legacyId !== 'string' || !legacyId) return null;
257
+ const registry = await readRegistry(registryPathFor(homeDir));
258
+ if (registry === null) return null; // corrupt registry → unknown, never guess
259
+ if (registry.projects[legacyId]) return legacyId;
260
+ const aliasEntry = registry.aliasIndex?.[legacyId];
261
+ if (aliasEntry === 'ambiguous') return 'ambiguous';
262
+ if (Array.isArray(aliasEntry) && aliasEntry.length === 1) return aliasEntry[0];
263
+ if (Array.isArray(aliasEntry) && aliasEntry.length > 1) return 'ambiguous';
264
+ if (typeof aliasEntry === 'string' && aliasEntry) return aliasEntry;
265
+ return null;
266
+ }
@@ -0,0 +1,178 @@
1
+ // TASK-003 (M04-J1b) — bounded in-process inverted index for v2 record
2
+ // retrieval (SPEC §5 FR-006/FR-007, §8). Token → postings map; the index
3
+ // narrows candidates, `eligible()` in policy.js still gates — the index
4
+ // accelerates, never decides. In-memory only, never persisted.
5
+ //
6
+ // createIndex({maxRecords?, maxPostings?}) → {build, query, invalidate, stats}
7
+ // build(records, {generation}) — rebuild postings; records beyond
8
+ // maxRecords/maxPostings are evicted,
9
+ // lowest base-score first.
10
+ // query(tokens, {limit}) — ranked [{record, id, score}] using the
11
+ // same token-overlap semantics as
12
+ // retrieval.js scoreRecord; [] when the
13
+ // index is empty, stale, or tokens empty.
14
+ // invalidate(generation) — record a newer store generation; a
15
+ // mismatch drops postings so no stale
16
+ // hits are served until the next build.
17
+ // stats() — {generation, records, postings, evicted}
18
+
19
+ const STOPWORDS = new Set([
20
+ 'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
21
+ 'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
22
+ 'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
23
+ ]);
24
+
25
+ const RECORD_TYPE_WEIGHTS = Object.freeze({
26
+ project_rule: 7, derived_fact: 5, procedure: 5, episode: 3,
27
+ });
28
+
29
+ export function normalize(text) {
30
+ return String(text ?? '')
31
+ .toLowerCase()
32
+ .replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
33
+ .replace(/\s+/g, ' ')
34
+ .trim();
35
+ }
36
+
37
+ export function tokenize(text) {
38
+ return normalize(text)
39
+ .split(/\s+/)
40
+ .filter((token) => token && !STOPWORDS.has(token) && token.length > 1);
41
+ }
42
+
43
+ function recordTokens(record) {
44
+ return new Set(tokenize(`${record.text ?? ''} ${record.provenance ?? ''}`));
45
+ }
46
+
47
+ // Identical scoring to retrieval.js scoreRecord — keep in sync (parity test).
48
+ export function scoreRecordTokens(record, queryTokens) {
49
+ const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
50
+ if (queryTokens.length === 0) {
51
+ return typeWeight * Math.max(0.1, record.confidence ?? 1);
52
+ }
53
+ const tokens = recordTokens(record);
54
+ let matched = 0;
55
+ for (const token of queryTokens) {
56
+ if (tokens.has(token)) matched += 1;
57
+ }
58
+ if (matched === 0) return 0;
59
+ const recencyBonus = record.created_at > 0
60
+ ? Math.min(1, record.created_at / Date.now())
61
+ : 0;
62
+ return typeWeight * matched * Math.max(0.1, record.confidence ?? 1) + recencyBonus;
63
+ }
64
+
65
+ // Base score used only for eviction order (lowest evicted first): type
66
+ // weight × confidence, then oldest first — mirrors the ranking inputs so
67
+ // eviction drops what would rank last anyway.
68
+ function evictionKey(record) {
69
+ const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
70
+ return typeWeight * Math.max(0.1, record.confidence ?? 1);
71
+ }
72
+
73
+ export function createIndex({ maxRecords = 10000, maxPostings = 50000 } = {}) {
74
+ let records = [];
75
+ let postings = new Map(); // token → number[] (indexes into records)
76
+ let postingsCount = 0;
77
+ let generation = 0;
78
+ let builtGeneration = -1; // generation the postings actually reflect
79
+ let evicted = 0;
80
+
81
+ function stats() {
82
+ return {
83
+ generation,
84
+ records: records.length,
85
+ postings: postingsCount,
86
+ evicted,
87
+ };
88
+ }
89
+
90
+ function dropRecord(idx) {
91
+ for (const token of recordTokens(records[idx])) {
92
+ const list = postings.get(token);
93
+ if (!list) continue;
94
+ const pos = list.indexOf(idx);
95
+ if (pos !== -1) {
96
+ list.splice(pos, 1);
97
+ postingsCount -= 1;
98
+ }
99
+ if (list.length === 0) postings.delete(token);
100
+ }
101
+ records[idx] = null;
102
+ evicted += 1;
103
+ }
104
+
105
+ function enforceBounds() {
106
+ if (records.length <= maxRecords && postingsCount <= maxPostings) return;
107
+ // Rank survivors by eviction priority; drop lowest until within bounds.
108
+ const order = records
109
+ .map((record, idx) => ({ record, idx }))
110
+ .filter((entry) => entry.record !== null)
111
+ .sort((a, b) => evictionKey(a.record) - evictionKey(b.record)
112
+ || (a.record.created_at ?? 0) - (b.record.created_at ?? 0));
113
+ for (const { idx } of order) {
114
+ if (records.filter(Boolean).length <= maxRecords && postingsCount <= maxPostings) break;
115
+ dropRecord(idx);
116
+ }
117
+ // Compact: rebuild postings against a dense records array.
118
+ records = records.filter(Boolean);
119
+ postings = new Map();
120
+ postingsCount = 0;
121
+ records.forEach((record, idx) => {
122
+ for (const token of recordTokens(record)) {
123
+ let list = postings.get(token);
124
+ if (!list) postings.set(token, (list = []));
125
+ list.push(idx);
126
+ postingsCount += 1;
127
+ }
128
+ });
129
+ }
130
+
131
+ function build(input, { generation: gen = 0 } = {}) {
132
+ records = Array.isArray(input) ? [...input] : [];
133
+ postings = new Map();
134
+ postingsCount = 0;
135
+ evicted = 0;
136
+ records.forEach((record, idx) => {
137
+ for (const token of recordTokens(record)) {
138
+ let list = postings.get(token);
139
+ if (!list) postings.set(token, (list = []));
140
+ list.push(idx);
141
+ postingsCount += 1;
142
+ }
143
+ });
144
+ enforceBounds();
145
+ generation = gen;
146
+ builtGeneration = gen;
147
+ }
148
+
149
+ function invalidate(newGeneration) {
150
+ generation = Number.isInteger(newGeneration) && newGeneration >= 0 ? newGeneration : generation;
151
+ if (generation !== builtGeneration) {
152
+ // Store moved on — never serve postings built against an older doc.
153
+ postings = new Map();
154
+ postingsCount = 0;
155
+ records = [];
156
+ }
157
+ }
158
+
159
+ function query(queryTokens, { limit = 10 } = {}) {
160
+ if (!Array.isArray(queryTokens) || queryTokens.length === 0) return [];
161
+ if (builtGeneration !== generation || records.length === 0) return [];
162
+ const seen = new Set();
163
+ for (const token of queryTokens) {
164
+ for (const idx of postings.get(token) ?? []) seen.add(idx);
165
+ }
166
+ if (seen.size === 0) return [];
167
+ return [...seen]
168
+ .map((idx) => ({ record: records[idx], score: scoreRecordTokens(records[idx], queryTokens) }))
169
+ .filter((entry) => entry.score > 0)
170
+ .sort((a, b) => b.score - a.score
171
+ || (b.record.created_at ?? 0) - (a.record.created_at ?? 0)
172
+ || String(b.record.text ?? '').localeCompare(String(a.record.text ?? '')))
173
+ .slice(0, limit)
174
+ .map((entry) => ({ record: entry.record, id: entry.record.id, score: entry.score }));
175
+ }
176
+
177
+ return { build, query, invalidate, stats };
178
+ }
@@ -1,16 +1,35 @@
1
1
  // Record-document store core — SPEC §3 / FR-007.
2
2
  // Path-agnostic read/write of the v2 records document
3
- // ({ schemaVersion: 2, records: [...] }) so both the project store
4
- // (storeV2.js) and the user store (userMemory.js) share one implementation.
5
- // Writes go through fileOps.writeJson (tmp+rename).
3
+ // ({ schemaVersion: 2, generation: int, tombstones: [...], records: [...] })
4
+ // so both the project store (storeV2.js) and the user store (userMemory.js)
5
+ // share one implementation. Writes go through fileOps.writeJson (tmp+rename,
6
+ // fsync before rename).
6
7
 
7
- import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
8
+ import fs from 'node:fs/promises';
9
+ import path from 'node:path';
10
+
11
+ import { copyFileRawExclusive, readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
8
12
  import { normalizeRecord } from './records.js';
9
13
 
10
14
  const SCHEMA_VERSION = 2;
15
+ const TOMBSTONE_MAX = 512;
16
+
17
+ /**
18
+ * Thrown by mutateRecordStore when the on-disk doc is corrupt or carries an
19
+ * unsupported schema. `code` is 'store-corrupt' or 'unsupported-schema';
20
+ * the message carries the quarantine path, never file content.
21
+ */
22
+ export class StoreCorruptError extends Error {
23
+ constructor(code, message) {
24
+ super(message);
25
+ this.name = 'StoreCorruptError';
26
+ this.code = code;
27
+ }
28
+ }
11
29
 
12
30
  export const PATCHABLE_FIELDS = ['type', 'scope', 'text', 'provenance', 'confidence',
13
- 'valid_until', 'status', 'project_id', 'meta', 'source_fingerprint'];
31
+ 'valid_until', 'status', 'project_id', 'meta', 'source_fingerprint', 'revision',
32
+ 'supersedes', 'superseded_by', 'reason', 'evidence', 'trust_tier'];
14
33
 
15
34
  export function isRawRecord(input) {
16
35
  return input && typeof input === 'object' && typeof input.id === 'string'
@@ -18,18 +37,26 @@ export function isRawRecord(input) {
18
37
  }
19
38
 
20
39
  /**
21
- * readRecordStore(recordsPath) → { records, invalidSkipped }
22
- * Tolerant: missing file → {records:[]}; unparseable → {records:[], invalidSkipped:1};
23
- * per-record normalizeRecord skip.
40
+ * readRecordStore(recordsPath) → { records, invalidSkipped, state }
41
+ * Typed states (M01-03): 'missing' (ENOENT/null doc), 'corrupt' (parse
42
+ * failure), 'empty' (parsed, zero usable records), 'ok' (≥1 record).
43
+ * Tolerant: per-record normalizeRecord skip. Read path never writes —
44
+ * corrupt bytes stay byte-identical.
24
45
  */
25
46
  export async function readRecordStore(recordsPath) {
26
47
  let doc;
27
48
  try {
28
49
  doc = await readJsonIfExists(recordsPath);
29
50
  } catch {
30
- return { records: [], invalidSkipped: 1 };
51
+ return { records: [], invalidSkipped: 1, state: 'corrupt', generation: 0, tombstones: [] };
52
+ }
53
+ if (!doc) return { records: [], invalidSkipped: 0, state: 'missing', generation: 0, tombstones: [] };
54
+
55
+ // Newer/foreign schema: never yield records — a newer build may have written
56
+ // fields this build cannot roundtrip (SPEC §14).
57
+ if (typeof doc.schemaVersion !== 'number' || doc.schemaVersion > SCHEMA_VERSION) {
58
+ return { records: [], invalidSkipped: 0, state: 'unsupported-schema', generation: 0, tombstones: [] };
31
59
  }
32
- if (!doc) return { records: [], invalidSkipped: 0 };
33
60
 
34
61
  const rawRecords = Array.isArray(doc.records) ? doc.records : [];
35
62
  const records = [];
@@ -39,21 +66,47 @@ export async function readRecordStore(recordsPath) {
39
66
  if (normalized) records.push(normalized);
40
67
  else invalidSkipped += 1;
41
68
  }
42
- return { records, invalidSkipped };
69
+ const generation = Number.isInteger(doc.generation) && doc.generation >= 0 ? doc.generation : 0;
70
+ const tombstones = Array.isArray(doc.tombstones) ? doc.tombstones : [];
71
+ return { records, invalidSkipped, state: records.length > 0 ? 'ok' : 'empty', generation, tombstones };
43
72
  }
44
73
 
45
74
  /**
46
75
  * writeRecordStore(recordsPath, records) → void
47
76
  * Atomic write; drops unrecoverable entries.
77
+ *
78
+ * Generation is strictly monotonic (SPEC §5 FR-005): the persisted value is
79
+ * `max(supplied, on-disk prior) + 1`, so a caller that passes a stale or
80
+ * backwards generation can never regress the doc. The prior re-read is cheap
81
+ * (one stat+parse) and keeps this function safe even when invoked outside
82
+ * mutateRecordStore's lock.
48
83
  */
49
- export async function writeRecordStore(recordsPath, records) {
84
+ export async function writeRecordStore(recordsPath, records, { generation, tombstones } = {}) {
50
85
  const normalized = (Array.isArray(records) ? records : [])
51
86
  .map((raw) => normalizeRecord(raw))
52
87
  .filter(Boolean);
88
+ const prior = await readRecordStore(recordsPath);
89
+ const supplied = Number.isInteger(generation) && generation >= 0 ? generation : 0;
90
+ const baseGeneration = Math.max(supplied, prior.generation);
91
+ const baseTombstones = tombstones === undefined ? prior.tombstones : tombstones;
92
+ const nextTombstones = (Array.isArray(baseTombstones) ? baseTombstones : []).slice(-TOMBSTONE_MAX);
53
93
  await writeJson(recordsPath, {
54
94
  schemaVersion: SCHEMA_VERSION,
95
+ generation: baseGeneration + 1,
96
+ tombstones: nextTombstones,
55
97
  records: normalized,
56
- });
98
+ }, { fsync: true });
99
+ }
100
+
101
+ /**
102
+ * isTombstoned(doc, {id, fingerprint}) → boolean
103
+ * `doc` is a readRecordStore result or any `{ tombstones }` shape. A record
104
+ * matches when its id OR fingerprint equals a tombstone entry.
105
+ */
106
+ export function isTombstoned(doc, { id, fingerprint } = {}) {
107
+ const tombstones = Array.isArray(doc?.tombstones) ? doc.tombstones : [];
108
+ return tombstones.some((t) => (id !== undefined && t.id === id)
109
+ || (fingerprint !== undefined && t.fingerprint === fingerprint));
57
110
  }
58
111
 
59
112
  /**
@@ -85,16 +138,54 @@ export function applyRecordPatch(records, id, patch = {}) {
85
138
  * Serializes a read-modify-write cycle on the records document under the
86
139
  * shared file lock (fileOps.withFileLock) so concurrent add/update flows —
87
140
  * in-process or across hook processes — never lose each other's writes.
88
- * `mutate(records)` returns { result, records } to persist, or null to skip
89
- * the write (e.g. id not found). Throws when the lock wait expires: a
90
- * silently dropped mutation would corrupt caller state worse than an error.
141
+ * `mutate(records, prior)` returns:
142
+ * - null → skip the write entirely (result null);
143
+ * - { result } → skip the write but return `result` (typed no-op);
144
+ * - { result, records, tombstones?, postWrite? } → persist records
145
+ * (tombstones defaults to prior's), then run `postWrite()` inside the
146
+ * same lock after the doc write (journal appends).
147
+ * Throws when the lock wait expires: a silently dropped mutation would
148
+ * corrupt caller state worse than an error.
91
149
  */
92
- export async function mutateRecordStore(recordsPath, mutate) {
150
+ export async function mutateRecordStore(recordsPath, mutate, opts = {}) {
93
151
  const outcome = await withFileLock(recordsPath, async () => {
94
- const { records } = await readRecordStore(recordsPath);
95
- const mutation = await mutate(records);
152
+ const prior = await readRecordStore(recordsPath);
153
+ let { records } = prior;
154
+ if (prior.state === 'corrupt' || prior.state === 'unsupported-schema') {
155
+ const quarantinePath = await quarantineStoreFile(recordsPath);
156
+ if (prior.state === 'unsupported-schema') {
157
+ // Never proceed empty on a newer-schema doc — that would destroy data
158
+ // written by a newer build. No onCorrupt escape hatch.
159
+ throw new StoreCorruptError(
160
+ 'unsupported-schema',
161
+ `mutateRecordStore: unsupported schema in ${recordsPath} — quarantined to ${quarantinePath}`,
162
+ );
163
+ }
164
+ if (opts.onCorrupt !== 'quarantine-empty') {
165
+ throw new StoreCorruptError(
166
+ 'store-corrupt',
167
+ `mutateRecordStore: corrupt records doc ${recordsPath} — quarantined to ${quarantinePath}`,
168
+ );
169
+ }
170
+ // Explicit restore path: quarantine preserved the bytes; proceed empty.
171
+ records = [];
172
+ }
173
+ const mutation = await mutate(records, prior);
96
174
  if (!mutation) return null;
97
- await writeRecordStore(recordsPath, mutation.records);
175
+ if (mutation.records !== undefined) {
176
+ // Generation contract: pass the generation read under THIS lock —
177
+ // writeRecordStore persists prior.generation + 1 (and clamps against
178
+ // the on-disk doc, so a stale caller can never regress it).
179
+ await writeRecordStore(recordsPath, mutation.records, {
180
+ generation: prior.generation,
181
+ tombstones: mutation.tombstones ?? prior.tombstones,
182
+ });
183
+ // Test-only crash-injection hook (TASK-002): UKIT_TEST_CRASH_AT=
184
+ // 'post-doc-write' kills the process after the doc rename but before
185
+ // postWrite (journal append), proving crash recovery never duplicates.
186
+ if (process.env.UKIT_TEST_CRASH_AT === 'post-doc-write') process.exit(1);
187
+ if (typeof mutation.postWrite === 'function') await mutation.postWrite();
188
+ }
98
189
  return mutation.result ?? null;
99
190
  });
100
191
  if (outcome === undefined) {
@@ -102,3 +193,25 @@ export async function mutateRecordStore(recordsPath, mutate) {
102
193
  }
103
194
  return outcome;
104
195
  }
196
+
197
+ /**
198
+ * Copy the corrupt doc aside for manual review — never overwrite, never
199
+ * delete the original. Exclusive-create destination + 0600 mode (the copy
200
+ * may contain secrets — same boundary as the canonical store, SPEC §14).
201
+ * Best-effort: a failed copy still yields the error path, with the attempted
202
+ * path in the message.
203
+ */
204
+ async function quarantineStoreFile(recordsPath) {
205
+ const quarantinePath = path.join(
206
+ path.dirname(recordsPath),
207
+ 'quarantine',
208
+ `${path.basename(recordsPath)}-${Date.now()}.json`,
209
+ );
210
+ try {
211
+ await copyFileRawExclusive(recordsPath, quarantinePath);
212
+ await fs.chmod(quarantinePath, 0o600).catch(() => {});
213
+ } catch {
214
+ // Quarantine copy failed — still fail closed; the original stays untouched.
215
+ }
216
+ return quarantinePath;
217
+ }