@ngockhoale/ukit 3.0.8 → 3.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/manifests/documentation.yaml +11 -0
- package/package.json +1 -1
- package/scripts/audit/decision-coverage.mjs +29 -2
- package/scripts/bench/data-foundation.mjs +52 -3
- package/scripts/bench/decision-runtime-baseline.mjs +427 -0
- package/scripts/bench/decision-runtime-metrics.mjs +67 -0
- package/scripts/bench/decision-runtime-variant.mjs +626 -0
- package/scripts/bench/memory-ablation.mjs +495 -0
- package/scripts/bench/memory-baseline.mjs +596 -0
- package/scripts/bench/memory-bench.mjs +661 -0
- package/scripts/bench/memory-canary.mjs +321 -0
- package/scripts/bench/memory-corpus.mjs +354 -0
- package/scripts/bench/memory-gate.mjs +389 -0
- package/scripts/bench/memory-metrics.mjs +179 -0
- package/scripts/bench/parallel-agents.mjs +33 -11
- package/scripts/bench/recorder-overhead.mjs +204 -0
- package/scripts/bench/sqlite-spike.mjs +451 -0
- package/scripts/measure-decision-gateway.mjs +306 -0
- package/scripts/perf/audit-perf.mjs +35 -17
- package/src/bug/triageBug.js +4 -3
- package/src/cli/commands/memory.js +357 -63
- package/src/context/detectProjectContext.js +11 -1
- package/src/core/agentRuntime/adapters.js +254 -0
- package/src/core/agentRuntime/artifacts.js +192 -0
- package/src/core/agentRuntime/completionGate.js +176 -0
- package/src/core/agentRuntime/context.js +149 -0
- package/src/core/agentRuntime/contract.js +247 -0
- package/src/core/agentRuntime/diagnostics.js +244 -0
- package/src/core/agentRuntime/evaluation.js +163 -0
- package/src/core/agentRuntime/eventStore.js +404 -0
- package/src/core/agentRuntime/liveness.js +60 -0
- package/src/core/agentRuntime/planCompiler.js +322 -0
- package/src/core/agentRuntime/promotion.js +53 -0
- package/src/core/agentRuntime/qualityComparison.js +112 -0
- package/src/core/agentRuntime/recovery.js +266 -0
- package/src/core/agentRuntime/resourcePolicy.js +78 -0
- package/src/core/agentRuntime/runtimeSupport.js +237 -0
- package/src/core/agentRuntime/supervisor.js +565 -0
- package/src/core/agentRuntime/vmEngine.js +621 -0
- package/src/core/codeintel/analogy.js +3 -2
- package/src/core/experiments/dynamicWorkflow.js +17 -2
- package/src/core/fileOps.js +21 -3
- package/src/core/memory/deltaOverlays.js +75 -30
- package/src/core/memory/learningCandidates.js +93 -48
- package/src/core/memory/memoryFlags.js +83 -0
- package/src/core/memory/memoryFreshness.js +190 -0
- package/src/core/memory/memoryHit.js +144 -0
- package/src/core/memory/migrate.js +69 -189
- package/src/core/memory/migrateMapping.js +232 -0
- package/src/core/memory/mutateMemory.js +323 -0
- package/src/core/memory/policy.js +96 -0
- package/src/core/memory/projectIdentity.js +266 -0
- package/src/core/memory/recordIndex.js +178 -0
- package/src/core/memory/recordStore.js +133 -20
- package/src/core/memory/records.js +144 -6
- package/src/core/memory/retrieval.js +259 -125
- package/src/core/memory/store.js +16 -5
- package/src/core/memory/storeBackup.js +226 -0
- package/src/core/memory/storeV2.js +63 -26
- package/src/core/memory/storeV2Loader.js +30 -12
- package/src/core/memory/userMemory.js +38 -20
- package/src/core/memory/writeClassification.js +161 -0
- package/src/core/memory/writeGuard.js +129 -0
- package/src/core/observability/adapters/hookTelemetryAdapter.js +90 -0
- package/src/core/observability/analytics/cohorts.js +148 -0
- package/src/core/observability/analytics/storeDigest.js +163 -0
- package/src/core/observability/evaluation/experimentPlan.js +95 -0
- package/src/core/observability/evaluation/findings.js +99 -0
- package/src/core/observability/evaluation/optimizationKnowledge.js +10 -1
- package/src/core/observability/evaluation/perturbation.js +273 -0
- package/src/core/observability/evaluation/replay.js +7 -1
- package/src/core/observability/evaluation/scorecard.js +23 -3
- package/src/core/observability/rollout.js +11 -7
- package/src/core/observability/schema/compatibility.js +135 -0
- package/src/core/observability/schema/registry.js +99 -0
- package/src/core/observability/schema/validate.js +7 -0
- package/src/core/observability/support/import.js +53 -9
- package/src/core/observability/support/paths.js +13 -3
- package/src/core/observability/support/projector.js +148 -12
- package/src/core/output/index.js +12 -2
- package/src/core/runtimeConfig.js +83 -0
- package/src/core/runtimePaths.js +3 -0
- package/src/core/sensitiveValueScanner.js +40 -0
- package/src/core/token/index.js +40 -3
- package/src/decision/client.js +37 -13
- package/src/decision/protocol.js +1 -1
- package/src/decision/registry.js +5 -3
- package/src/decision/runtimeDecide.js +242 -0
- package/src/decision/runtimeFilter.js +150 -0
- package/src/decision/runtimeScheduler.js +239 -0
- package/src/index/buildIndex.js +13 -12
- package/src/index/queryIndex.js +35 -14
- package/src/index/relatedTests.js +50 -8
- package/src/index/resolveContext.js +9 -4
- package/src/manifest/selectItems.js +7 -3
- package/src/render/instructionRenderer.js +17 -5
- package/template_project/.claude/ukit/index/lib/index-core.mjs +94 -39
- package/template_project/.claude/ukit/index/route-task.mjs +121 -19
- package/template_project/.claude/ukit/index/unic-decision.mjs +28 -13
- package/template_project/.claude/ukit/runtime/memory-flags.mjs +51 -0
- package/template_project/.claude/ukit/runtime/memory-freshness.mjs +155 -0
- package/template_project/.claude/ukit/runtime/memory-policy.mjs +286 -0
- package/template_project/.claude/ukit/runtime/output-compression.mjs +3 -0
- package/template_project/.claude/ukit/runtime/reinject-context.mjs +145 -14
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import crypto from 'node:crypto';
|
|
4
|
+
import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
|
|
5
|
+
import { buildUserPaths } from '../userPaths.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Host-owned stable project identity resolver (M01-01, SPEC §5 FR-001..003).
|
|
9
|
+
*
|
|
10
|
+
* Identity authority is a USER-LEVEL registry at
|
|
11
|
+
* `~/.ukit/storage/projects/registry.json` — user-level because uninstall
|
|
12
|
+
* removes the project `.ukit/` tree. `package.json.name` is never an identity
|
|
13
|
+
* authority; it only feeds the alias index (display-name collisions degrade
|
|
14
|
+
* to `ambiguous`, never a guess).
|
|
15
|
+
*
|
|
16
|
+
* Registry shape (SPEC §7):
|
|
17
|
+
* {schemaVersion:1,
|
|
18
|
+
* projects:{<projectId>:{roots:[realpath], gitCommonDirs:[abs],
|
|
19
|
+
* aliases:[name], createdAt, updatedAt}},
|
|
20
|
+
* aliasIndex:{<name>:[<projectId>|'ambiguous']}}
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const REGISTRY_SCHEMA_VERSION = 1;
|
|
24
|
+
|
|
25
|
+
function registryPathFor(homeDir) {
|
|
26
|
+
const { storageRoot } = buildUserPaths(homeDir === undefined ? {} : { homeDir });
|
|
27
|
+
return path.join(storageRoot, 'projects', 'registry.json');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function emptyRegistry() {
|
|
31
|
+
return { schemaVersion: REGISTRY_SCHEMA_VERSION, projects: {}, aliasIndex: {} };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Tolerant registry read. Missing → empty registry. Corrupt → null sentinel
|
|
36
|
+
* (caller degrades to `unverified`; the file is NEVER overwritten on read).
|
|
37
|
+
*/
|
|
38
|
+
async function readRegistry(regPath) {
|
|
39
|
+
let data;
|
|
40
|
+
try {
|
|
41
|
+
data = await readJsonIfExists(regPath);
|
|
42
|
+
} catch {
|
|
43
|
+
return null; // corrupt JSON — degrade, never throw, never rewrite
|
|
44
|
+
}
|
|
45
|
+
if (data === null) return emptyRegistry();
|
|
46
|
+
if (typeof data !== 'object' || typeof data.projects !== 'object' || data.projects === null) {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
if (typeof data.aliasIndex !== 'object' || data.aliasIndex === null) {
|
|
50
|
+
data.aliasIndex = {};
|
|
51
|
+
}
|
|
52
|
+
return data;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function realpathOrNull(target) {
|
|
56
|
+
try {
|
|
57
|
+
return await fs.realpath(target);
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Resolve the git common dir for a root: `.git` dir → itself; `.git` file
|
|
65
|
+
* (`gitdir: <p>`) → `<p>/commondir` (linked worktrees share the main common
|
|
66
|
+
* dir). Returns null when there is no git binding.
|
|
67
|
+
*/
|
|
68
|
+
async function resolveGitCommonDir(projectRoot) {
|
|
69
|
+
const gitPath = path.join(projectRoot, '.git');
|
|
70
|
+
let stat;
|
|
71
|
+
try {
|
|
72
|
+
stat = await fs.stat(gitPath);
|
|
73
|
+
} catch {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (stat.isDirectory()) {
|
|
78
|
+
return await realpathOrNull(gitPath);
|
|
79
|
+
}
|
|
80
|
+
if (!stat.isFile()) return null;
|
|
81
|
+
|
|
82
|
+
let content;
|
|
83
|
+
try {
|
|
84
|
+
content = await fs.readFile(gitPath, 'utf8');
|
|
85
|
+
} catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
const match = content.match(/^gitdir:\s*(.+)$/m);
|
|
89
|
+
if (!match) return null;
|
|
90
|
+
const gitDir = path.resolve(projectRoot, match[1].trim());
|
|
91
|
+
|
|
92
|
+
let commonDirRel;
|
|
93
|
+
try {
|
|
94
|
+
commonDirRel = (await fs.readFile(path.join(gitDir, 'commondir'), 'utf8')).trim();
|
|
95
|
+
} catch {
|
|
96
|
+
commonDirRel = '';
|
|
97
|
+
}
|
|
98
|
+
const commonDir = commonDirRel ? path.resolve(gitDir, commonDirRel) : gitDir;
|
|
99
|
+
return await realpathOrNull(commonDir);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function aliasesFor(projectRoot, realRoot) {
|
|
103
|
+
const names = new Set();
|
|
104
|
+
const base = path.basename(realRoot ?? projectRoot);
|
|
105
|
+
if (base) names.add(base);
|
|
106
|
+
const pkg = await readJsonIfExists(path.join(projectRoot, 'package.json')).catch(() => null);
|
|
107
|
+
if (typeof pkg?.name === 'string' && pkg.name.trim() !== '') {
|
|
108
|
+
names.add(pkg.name.trim());
|
|
109
|
+
}
|
|
110
|
+
return [...names];
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function findByRoot(registry, realRoot) {
|
|
114
|
+
for (const [projectId, entry] of Object.entries(registry.projects)) {
|
|
115
|
+
if (Array.isArray(entry?.roots) && entry.roots.includes(realRoot)) {
|
|
116
|
+
return projectId;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function findByGitCommonDir(registry, commonDir) {
|
|
123
|
+
if (!commonDir) return null;
|
|
124
|
+
for (const [projectId, entry] of Object.entries(registry.projects)) {
|
|
125
|
+
if (Array.isArray(entry?.gitCommonDirs) && entry.gitCommonDirs.includes(commonDir)) {
|
|
126
|
+
return projectId;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function findByAlias(registry, aliases) {
|
|
133
|
+
const hits = new Set();
|
|
134
|
+
for (const name of aliases) {
|
|
135
|
+
const entry = registry.aliasIndex?.[name];
|
|
136
|
+
if (entry === 'ambiguous') {
|
|
137
|
+
hits.add('ambiguous');
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (Array.isArray(entry)) {
|
|
141
|
+
for (const id of entry) hits.add(id);
|
|
142
|
+
} else if (typeof entry === 'string' && entry) {
|
|
143
|
+
hits.add(entry);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return hits;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function mintProjectId(realRoot) {
|
|
150
|
+
const digest = crypto
|
|
151
|
+
.createHash('sha256')
|
|
152
|
+
.update(`${realRoot}|${crypto.randomBytes(16).toString('hex')}`)
|
|
153
|
+
.digest('hex');
|
|
154
|
+
return `prj_${digest.slice(0, 16)}`;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function indexAliases(registry, projectId, aliases) {
|
|
158
|
+
for (const name of aliases) {
|
|
159
|
+
const current = registry.aliasIndex[name];
|
|
160
|
+
if (current === 'ambiguous') continue;
|
|
161
|
+
const list = Array.isArray(current) ? current : (typeof current === 'string' && current ? [current] : []);
|
|
162
|
+
if (!list.includes(projectId)) list.push(projectId);
|
|
163
|
+
registry.aliasIndex[name] = list.length > 1 ? 'ambiguous' : list;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Resolve the stable project identity for `projectRoot`.
|
|
169
|
+
*
|
|
170
|
+
* Order: (1) canonical realpath in registry roots; (2) git common-dir binding
|
|
171
|
+
* (linked worktree → same project); (3) alias index — multiple distinct
|
|
172
|
+
* projectIds → ambiguous deny; (4) mint (default) under file lock, or
|
|
173
|
+
* `unverified` when `mint:false`.
|
|
174
|
+
*
|
|
175
|
+
* @returns {Promise<{projectId: string|null, source: 'registry'|'git-common-dir'|'alias'|'unverified', ambiguous: boolean}>}
|
|
176
|
+
*/
|
|
177
|
+
export async function resolveProjectIdentity(projectRoot, { homeDir, mint = true } = {}) {
|
|
178
|
+
const unverified = { projectId: null, source: 'unverified', ambiguous: false };
|
|
179
|
+
const regPath = registryPathFor(homeDir);
|
|
180
|
+
|
|
181
|
+
const registry = await readRegistry(regPath);
|
|
182
|
+
if (registry === null) return unverified; // corrupt — never overwrite on read
|
|
183
|
+
|
|
184
|
+
const realRoot = (await realpathOrNull(projectRoot)) ?? path.resolve(projectRoot);
|
|
185
|
+
|
|
186
|
+
const byRoot = findByRoot(registry, realRoot);
|
|
187
|
+
if (byRoot) {
|
|
188
|
+
return {
|
|
189
|
+
projectId: byRoot,
|
|
190
|
+
source: 'registry',
|
|
191
|
+
ambiguous: false,
|
|
192
|
+
aliases: Array.isArray(registry.projects[byRoot]?.aliases)
|
|
193
|
+
? registry.projects[byRoot].aliases.filter((a) => typeof a === 'string')
|
|
194
|
+
: [],
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const commonDir = await resolveGitCommonDir(projectRoot);
|
|
199
|
+
const byGit = findByGitCommonDir(registry, commonDir);
|
|
200
|
+
if (byGit) {
|
|
201
|
+
return {
|
|
202
|
+
projectId: byGit,
|
|
203
|
+
source: 'git-common-dir',
|
|
204
|
+
ambiguous: false,
|
|
205
|
+
aliases: Array.isArray(registry.projects[byGit]?.aliases)
|
|
206
|
+
? registry.projects[byGit].aliases.filter((a) => typeof a === 'string')
|
|
207
|
+
: [],
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const aliases = await aliasesFor(projectRoot, realRoot);
|
|
212
|
+
// Alias is a weak display-name signal only: any hit for a different root
|
|
213
|
+
// denies (ambiguous) rather than letting a same-named repo inherit an id.
|
|
214
|
+
const aliasHits = findByAlias(registry, aliases);
|
|
215
|
+
if (aliasHits.size > 0) {
|
|
216
|
+
return { projectId: null, source: 'alias', ambiguous: true };
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (!mint) return unverified;
|
|
220
|
+
|
|
221
|
+
// Mint under the registry file lock; re-check inside the critical section so
|
|
222
|
+
// a concurrent minter cannot produce two ids for the same root.
|
|
223
|
+
const minted = await withFileLock(regPath, async () => {
|
|
224
|
+
const fresh = await readRegistry(regPath);
|
|
225
|
+
if (fresh === null) return null; // became corrupt between read and lock
|
|
226
|
+
const existing = findByRoot(fresh, realRoot)
|
|
227
|
+
?? (commonDir ? findByGitCommonDir(fresh, commonDir) : null);
|
|
228
|
+
if (existing) return existing;
|
|
229
|
+
|
|
230
|
+
const now = new Date().toISOString();
|
|
231
|
+
const projectId = mintProjectId(realRoot);
|
|
232
|
+
fresh.projects[projectId] = {
|
|
233
|
+
roots: [realRoot],
|
|
234
|
+
gitCommonDirs: commonDir ? [commonDir] : [],
|
|
235
|
+
aliases,
|
|
236
|
+
createdAt: now,
|
|
237
|
+
updatedAt: now,
|
|
238
|
+
};
|
|
239
|
+
indexAliases(fresh, projectId, aliases);
|
|
240
|
+
await writeJson(regPath, fresh);
|
|
241
|
+
return projectId;
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
if (minted === undefined || minted === null) return unverified;
|
|
245
|
+
return { projectId: minted, source: 'registry', ambiguous: false, aliases };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* lookupRegisteredProjectId(legacyId, { homeDir } = {}) → string | 'ambiguous' | null
|
|
250
|
+
* Read-only registry lookup for migration alias-collision checks (M04-01):
|
|
251
|
+
* a legacy project id that IS a registered projectId resolves to itself;
|
|
252
|
+
* an alias resolves to its bound projectId; 'ambiguous' when the alias is
|
|
253
|
+
* contested; null when unknown. Never mints, never writes.
|
|
254
|
+
*/
|
|
255
|
+
export async function lookupRegisteredProjectId(legacyId, { homeDir } = {}) {
|
|
256
|
+
if (typeof legacyId !== 'string' || !legacyId) return null;
|
|
257
|
+
const registry = await readRegistry(registryPathFor(homeDir));
|
|
258
|
+
if (registry === null) return null; // corrupt registry → unknown, never guess
|
|
259
|
+
if (registry.projects[legacyId]) return legacyId;
|
|
260
|
+
const aliasEntry = registry.aliasIndex?.[legacyId];
|
|
261
|
+
if (aliasEntry === 'ambiguous') return 'ambiguous';
|
|
262
|
+
if (Array.isArray(aliasEntry) && aliasEntry.length === 1) return aliasEntry[0];
|
|
263
|
+
if (Array.isArray(aliasEntry) && aliasEntry.length > 1) return 'ambiguous';
|
|
264
|
+
if (typeof aliasEntry === 'string' && aliasEntry) return aliasEntry;
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// TASK-003 (M04-J1b) — bounded in-process inverted index for v2 record
|
|
2
|
+
// retrieval (SPEC §5 FR-006/FR-007, §8). Token → postings map; the index
|
|
3
|
+
// narrows candidates, `eligible()` in policy.js still gates — the index
|
|
4
|
+
// accelerates, never decides. In-memory only, never persisted.
|
|
5
|
+
//
|
|
6
|
+
// createIndex({maxRecords?, maxPostings?}) → {build, query, invalidate, stats}
|
|
7
|
+
// build(records, {generation}) — rebuild postings; records beyond
|
|
8
|
+
// maxRecords/maxPostings are evicted,
|
|
9
|
+
// lowest base-score first.
|
|
10
|
+
// query(tokens, {limit}) — ranked [{record, id, score}] using the
|
|
11
|
+
// same token-overlap semantics as
|
|
12
|
+
// retrieval.js scoreRecord; [] when the
|
|
13
|
+
// index is empty, stale, or tokens empty.
|
|
14
|
+
// invalidate(generation) — record a newer store generation; a
|
|
15
|
+
// mismatch drops postings so no stale
|
|
16
|
+
// hits are served until the next build.
|
|
17
|
+
// stats() — {generation, records, postings, evicted}
|
|
18
|
+
|
|
19
|
+
const STOPWORDS = new Set([
|
|
20
|
+
'the', 'a', 'an', 'and', 'or', 'to', 'for', 'of', 'with', 'in', 'on', 'is', 'are',
|
|
21
|
+
'this', 'that', 'it', 'as', 'by', 'be', 'use', 'using', 'implement', 'fix', 'task',
|
|
22
|
+
'cần', 'và', 'là', 'cho', 'một', 'những', 'dùng',
|
|
23
|
+
]);
|
|
24
|
+
|
|
25
|
+
const RECORD_TYPE_WEIGHTS = Object.freeze({
|
|
26
|
+
project_rule: 7, derived_fact: 5, procedure: 5, episode: 3,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
export function normalize(text) {
|
|
30
|
+
return String(text ?? '')
|
|
31
|
+
.toLowerCase()
|
|
32
|
+
.replace(/[^\p{L}\p{N}\s./:_-]/gu, ' ')
|
|
33
|
+
.replace(/\s+/g, ' ')
|
|
34
|
+
.trim();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function tokenize(text) {
|
|
38
|
+
return normalize(text)
|
|
39
|
+
.split(/\s+/)
|
|
40
|
+
.filter((token) => token && !STOPWORDS.has(token) && token.length > 1);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function recordTokens(record) {
|
|
44
|
+
return new Set(tokenize(`${record.text ?? ''} ${record.provenance ?? ''}`));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Identical scoring to retrieval.js scoreRecord — keep in sync (parity test).
|
|
48
|
+
export function scoreRecordTokens(record, queryTokens) {
|
|
49
|
+
const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
|
|
50
|
+
if (queryTokens.length === 0) {
|
|
51
|
+
return typeWeight * Math.max(0.1, record.confidence ?? 1);
|
|
52
|
+
}
|
|
53
|
+
const tokens = recordTokens(record);
|
|
54
|
+
let matched = 0;
|
|
55
|
+
for (const token of queryTokens) {
|
|
56
|
+
if (tokens.has(token)) matched += 1;
|
|
57
|
+
}
|
|
58
|
+
if (matched === 0) return 0;
|
|
59
|
+
const recencyBonus = record.created_at > 0
|
|
60
|
+
? Math.min(1, record.created_at / Date.now())
|
|
61
|
+
: 0;
|
|
62
|
+
return typeWeight * matched * Math.max(0.1, record.confidence ?? 1) + recencyBonus;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Base score used only for eviction order (lowest evicted first): type
|
|
66
|
+
// weight × confidence, then oldest first — mirrors the ranking inputs so
|
|
67
|
+
// eviction drops what would rank last anyway.
|
|
68
|
+
function evictionKey(record) {
|
|
69
|
+
const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
|
|
70
|
+
return typeWeight * Math.max(0.1, record.confidence ?? 1);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function createIndex({ maxRecords = 10000, maxPostings = 50000 } = {}) {
|
|
74
|
+
let records = [];
|
|
75
|
+
let postings = new Map(); // token → number[] (indexes into records)
|
|
76
|
+
let postingsCount = 0;
|
|
77
|
+
let generation = 0;
|
|
78
|
+
let builtGeneration = -1; // generation the postings actually reflect
|
|
79
|
+
let evicted = 0;
|
|
80
|
+
|
|
81
|
+
function stats() {
|
|
82
|
+
return {
|
|
83
|
+
generation,
|
|
84
|
+
records: records.length,
|
|
85
|
+
postings: postingsCount,
|
|
86
|
+
evicted,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function dropRecord(idx) {
|
|
91
|
+
for (const token of recordTokens(records[idx])) {
|
|
92
|
+
const list = postings.get(token);
|
|
93
|
+
if (!list) continue;
|
|
94
|
+
const pos = list.indexOf(idx);
|
|
95
|
+
if (pos !== -1) {
|
|
96
|
+
list.splice(pos, 1);
|
|
97
|
+
postingsCount -= 1;
|
|
98
|
+
}
|
|
99
|
+
if (list.length === 0) postings.delete(token);
|
|
100
|
+
}
|
|
101
|
+
records[idx] = null;
|
|
102
|
+
evicted += 1;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function enforceBounds() {
|
|
106
|
+
if (records.length <= maxRecords && postingsCount <= maxPostings) return;
|
|
107
|
+
// Rank survivors by eviction priority; drop lowest until within bounds.
|
|
108
|
+
const order = records
|
|
109
|
+
.map((record, idx) => ({ record, idx }))
|
|
110
|
+
.filter((entry) => entry.record !== null)
|
|
111
|
+
.sort((a, b) => evictionKey(a.record) - evictionKey(b.record)
|
|
112
|
+
|| (a.record.created_at ?? 0) - (b.record.created_at ?? 0));
|
|
113
|
+
for (const { idx } of order) {
|
|
114
|
+
if (records.filter(Boolean).length <= maxRecords && postingsCount <= maxPostings) break;
|
|
115
|
+
dropRecord(idx);
|
|
116
|
+
}
|
|
117
|
+
// Compact: rebuild postings against a dense records array.
|
|
118
|
+
records = records.filter(Boolean);
|
|
119
|
+
postings = new Map();
|
|
120
|
+
postingsCount = 0;
|
|
121
|
+
records.forEach((record, idx) => {
|
|
122
|
+
for (const token of recordTokens(record)) {
|
|
123
|
+
let list = postings.get(token);
|
|
124
|
+
if (!list) postings.set(token, (list = []));
|
|
125
|
+
list.push(idx);
|
|
126
|
+
postingsCount += 1;
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function build(input, { generation: gen = 0 } = {}) {
|
|
132
|
+
records = Array.isArray(input) ? [...input] : [];
|
|
133
|
+
postings = new Map();
|
|
134
|
+
postingsCount = 0;
|
|
135
|
+
evicted = 0;
|
|
136
|
+
records.forEach((record, idx) => {
|
|
137
|
+
for (const token of recordTokens(record)) {
|
|
138
|
+
let list = postings.get(token);
|
|
139
|
+
if (!list) postings.set(token, (list = []));
|
|
140
|
+
list.push(idx);
|
|
141
|
+
postingsCount += 1;
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
enforceBounds();
|
|
145
|
+
generation = gen;
|
|
146
|
+
builtGeneration = gen;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function invalidate(newGeneration) {
|
|
150
|
+
generation = Number.isInteger(newGeneration) && newGeneration >= 0 ? newGeneration : generation;
|
|
151
|
+
if (generation !== builtGeneration) {
|
|
152
|
+
// Store moved on — never serve postings built against an older doc.
|
|
153
|
+
postings = new Map();
|
|
154
|
+
postingsCount = 0;
|
|
155
|
+
records = [];
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function query(queryTokens, { limit = 10 } = {}) {
|
|
160
|
+
if (!Array.isArray(queryTokens) || queryTokens.length === 0) return [];
|
|
161
|
+
if (builtGeneration !== generation || records.length === 0) return [];
|
|
162
|
+
const seen = new Set();
|
|
163
|
+
for (const token of queryTokens) {
|
|
164
|
+
for (const idx of postings.get(token) ?? []) seen.add(idx);
|
|
165
|
+
}
|
|
166
|
+
if (seen.size === 0) return [];
|
|
167
|
+
return [...seen]
|
|
168
|
+
.map((idx) => ({ record: records[idx], score: scoreRecordTokens(records[idx], queryTokens) }))
|
|
169
|
+
.filter((entry) => entry.score > 0)
|
|
170
|
+
.sort((a, b) => b.score - a.score
|
|
171
|
+
|| (b.record.created_at ?? 0) - (a.record.created_at ?? 0)
|
|
172
|
+
|| String(b.record.text ?? '').localeCompare(String(a.record.text ?? '')))
|
|
173
|
+
.slice(0, limit)
|
|
174
|
+
.map((entry) => ({ record: entry.record, id: entry.record.id, score: entry.score }));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
return { build, query, invalidate, stats };
|
|
178
|
+
}
|
|
@@ -1,16 +1,35 @@
|
|
|
1
1
|
// Record-document store core — SPEC §3 / FR-007.
|
|
2
2
|
// Path-agnostic read/write of the v2 records document
|
|
3
|
-
// ({ schemaVersion: 2, records: [...] })
|
|
4
|
-
// (storeV2.js) and the user store (userMemory.js)
|
|
5
|
-
// Writes go through fileOps.writeJson (tmp+rename
|
|
3
|
+
// ({ schemaVersion: 2, generation: int, tombstones: [...], records: [...] })
|
|
4
|
+
// so both the project store (storeV2.js) and the user store (userMemory.js)
|
|
5
|
+
// share one implementation. Writes go through fileOps.writeJson (tmp+rename,
|
|
6
|
+
// fsync before rename).
|
|
6
7
|
|
|
7
|
-
import
|
|
8
|
+
import fs from 'node:fs/promises';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
|
|
11
|
+
import { copyFileRawExclusive, readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
|
|
8
12
|
import { normalizeRecord } from './records.js';
|
|
9
13
|
|
|
10
14
|
const SCHEMA_VERSION = 2;
|
|
15
|
+
const TOMBSTONE_MAX = 512;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Thrown by mutateRecordStore when the on-disk doc is corrupt or carries an
|
|
19
|
+
* unsupported schema. `code` is 'store-corrupt' or 'unsupported-schema';
|
|
20
|
+
* the message carries the quarantine path, never file content.
|
|
21
|
+
*/
|
|
22
|
+
export class StoreCorruptError extends Error {
|
|
23
|
+
constructor(code, message) {
|
|
24
|
+
super(message);
|
|
25
|
+
this.name = 'StoreCorruptError';
|
|
26
|
+
this.code = code;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
11
29
|
|
|
12
30
|
export const PATCHABLE_FIELDS = ['type', 'scope', 'text', 'provenance', 'confidence',
|
|
13
|
-
'valid_until', 'status', 'project_id', 'meta', 'source_fingerprint'
|
|
31
|
+
'valid_until', 'status', 'project_id', 'meta', 'source_fingerprint', 'revision',
|
|
32
|
+
'supersedes', 'superseded_by', 'reason', 'evidence', 'trust_tier'];
|
|
14
33
|
|
|
15
34
|
export function isRawRecord(input) {
|
|
16
35
|
return input && typeof input === 'object' && typeof input.id === 'string'
|
|
@@ -18,18 +37,26 @@ export function isRawRecord(input) {
|
|
|
18
37
|
}
|
|
19
38
|
|
|
20
39
|
/**
|
|
21
|
-
* readRecordStore(recordsPath) → { records, invalidSkipped }
|
|
22
|
-
*
|
|
23
|
-
*
|
|
40
|
+
* readRecordStore(recordsPath) → { records, invalidSkipped, state }
|
|
41
|
+
* Typed states (M01-03): 'missing' (ENOENT/null doc), 'corrupt' (parse
|
|
42
|
+
* failure), 'empty' (parsed, zero usable records), 'ok' (≥1 record).
|
|
43
|
+
* Tolerant: per-record normalizeRecord skip. Read path never writes —
|
|
44
|
+
* corrupt bytes stay byte-identical.
|
|
24
45
|
*/
|
|
25
46
|
export async function readRecordStore(recordsPath) {
|
|
26
47
|
let doc;
|
|
27
48
|
try {
|
|
28
49
|
doc = await readJsonIfExists(recordsPath);
|
|
29
50
|
} catch {
|
|
30
|
-
return { records: [], invalidSkipped: 1 };
|
|
51
|
+
return { records: [], invalidSkipped: 1, state: 'corrupt', generation: 0, tombstones: [] };
|
|
52
|
+
}
|
|
53
|
+
if (!doc) return { records: [], invalidSkipped: 0, state: 'missing', generation: 0, tombstones: [] };
|
|
54
|
+
|
|
55
|
+
// Newer/foreign schema: never yield records — a newer build may have written
|
|
56
|
+
// fields this build cannot roundtrip (SPEC §14).
|
|
57
|
+
if (typeof doc.schemaVersion !== 'number' || doc.schemaVersion > SCHEMA_VERSION) {
|
|
58
|
+
return { records: [], invalidSkipped: 0, state: 'unsupported-schema', generation: 0, tombstones: [] };
|
|
31
59
|
}
|
|
32
|
-
if (!doc) return { records: [], invalidSkipped: 0 };
|
|
33
60
|
|
|
34
61
|
const rawRecords = Array.isArray(doc.records) ? doc.records : [];
|
|
35
62
|
const records = [];
|
|
@@ -39,21 +66,47 @@ export async function readRecordStore(recordsPath) {
|
|
|
39
66
|
if (normalized) records.push(normalized);
|
|
40
67
|
else invalidSkipped += 1;
|
|
41
68
|
}
|
|
42
|
-
|
|
69
|
+
const generation = Number.isInteger(doc.generation) && doc.generation >= 0 ? doc.generation : 0;
|
|
70
|
+
const tombstones = Array.isArray(doc.tombstones) ? doc.tombstones : [];
|
|
71
|
+
return { records, invalidSkipped, state: records.length > 0 ? 'ok' : 'empty', generation, tombstones };
|
|
43
72
|
}
|
|
44
73
|
|
|
45
74
|
/**
|
|
46
75
|
* writeRecordStore(recordsPath, records) → void
|
|
47
76
|
* Atomic write; drops unrecoverable entries.
|
|
77
|
+
*
|
|
78
|
+
* Generation is strictly monotonic (SPEC §5 FR-005): the persisted value is
|
|
79
|
+
* `max(supplied, on-disk prior) + 1`, so a caller that passes a stale or
|
|
80
|
+
* backwards generation can never regress the doc. The prior re-read is cheap
|
|
81
|
+
* (one stat+parse) and keeps this function safe even when invoked outside
|
|
82
|
+
* mutateRecordStore's lock.
|
|
48
83
|
*/
|
|
49
|
-
export async function writeRecordStore(recordsPath, records) {
|
|
84
|
+
export async function writeRecordStore(recordsPath, records, { generation, tombstones } = {}) {
|
|
50
85
|
const normalized = (Array.isArray(records) ? records : [])
|
|
51
86
|
.map((raw) => normalizeRecord(raw))
|
|
52
87
|
.filter(Boolean);
|
|
88
|
+
const prior = await readRecordStore(recordsPath);
|
|
89
|
+
const supplied = Number.isInteger(generation) && generation >= 0 ? generation : 0;
|
|
90
|
+
const baseGeneration = Math.max(supplied, prior.generation);
|
|
91
|
+
const baseTombstones = tombstones === undefined ? prior.tombstones : tombstones;
|
|
92
|
+
const nextTombstones = (Array.isArray(baseTombstones) ? baseTombstones : []).slice(-TOMBSTONE_MAX);
|
|
53
93
|
await writeJson(recordsPath, {
|
|
54
94
|
schemaVersion: SCHEMA_VERSION,
|
|
95
|
+
generation: baseGeneration + 1,
|
|
96
|
+
tombstones: nextTombstones,
|
|
55
97
|
records: normalized,
|
|
56
|
-
});
|
|
98
|
+
}, { fsync: true });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* isTombstoned(doc, {id, fingerprint}) → boolean
|
|
103
|
+
* `doc` is a readRecordStore result or any `{ tombstones }` shape. A record
|
|
104
|
+
* matches when its id OR fingerprint equals a tombstone entry.
|
|
105
|
+
*/
|
|
106
|
+
export function isTombstoned(doc, { id, fingerprint } = {}) {
|
|
107
|
+
const tombstones = Array.isArray(doc?.tombstones) ? doc.tombstones : [];
|
|
108
|
+
return tombstones.some((t) => (id !== undefined && t.id === id)
|
|
109
|
+
|| (fingerprint !== undefined && t.fingerprint === fingerprint));
|
|
57
110
|
}
|
|
58
111
|
|
|
59
112
|
/**
|
|
@@ -85,16 +138,54 @@ export function applyRecordPatch(records, id, patch = {}) {
|
|
|
85
138
|
* Serializes a read-modify-write cycle on the records document under the
|
|
86
139
|
* shared file lock (fileOps.withFileLock) so concurrent add/update flows —
|
|
87
140
|
* in-process or across hook processes — never lose each other's writes.
|
|
88
|
-
* `mutate(records)` returns
|
|
89
|
-
*
|
|
90
|
-
*
|
|
141
|
+
* `mutate(records, prior)` returns:
|
|
142
|
+
* - null → skip the write entirely (result null);
|
|
143
|
+
* - { result } → skip the write but return `result` (typed no-op);
|
|
144
|
+
* - { result, records, tombstones?, postWrite? } → persist records
|
|
145
|
+
* (tombstones defaults to prior's), then run `postWrite()` inside the
|
|
146
|
+
* same lock after the doc write (journal appends).
|
|
147
|
+
* Throws when the lock wait expires: a silently dropped mutation would
|
|
148
|
+
* corrupt caller state worse than an error.
|
|
91
149
|
*/
|
|
92
|
-
export async function mutateRecordStore(recordsPath, mutate) {
|
|
150
|
+
export async function mutateRecordStore(recordsPath, mutate, opts = {}) {
|
|
93
151
|
const outcome = await withFileLock(recordsPath, async () => {
|
|
94
|
-
const
|
|
95
|
-
|
|
152
|
+
const prior = await readRecordStore(recordsPath);
|
|
153
|
+
let { records } = prior;
|
|
154
|
+
if (prior.state === 'corrupt' || prior.state === 'unsupported-schema') {
|
|
155
|
+
const quarantinePath = await quarantineStoreFile(recordsPath);
|
|
156
|
+
if (prior.state === 'unsupported-schema') {
|
|
157
|
+
// Never proceed empty on a newer-schema doc — that would destroy data
|
|
158
|
+
// written by a newer build. No onCorrupt escape hatch.
|
|
159
|
+
throw new StoreCorruptError(
|
|
160
|
+
'unsupported-schema',
|
|
161
|
+
`mutateRecordStore: unsupported schema in ${recordsPath} — quarantined to ${quarantinePath}`,
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
if (opts.onCorrupt !== 'quarantine-empty') {
|
|
165
|
+
throw new StoreCorruptError(
|
|
166
|
+
'store-corrupt',
|
|
167
|
+
`mutateRecordStore: corrupt records doc ${recordsPath} — quarantined to ${quarantinePath}`,
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
// Explicit restore path: quarantine preserved the bytes; proceed empty.
|
|
171
|
+
records = [];
|
|
172
|
+
}
|
|
173
|
+
const mutation = await mutate(records, prior);
|
|
96
174
|
if (!mutation) return null;
|
|
97
|
-
|
|
175
|
+
if (mutation.records !== undefined) {
|
|
176
|
+
// Generation contract: pass the generation read under THIS lock —
|
|
177
|
+
// writeRecordStore persists prior.generation + 1 (and clamps against
|
|
178
|
+
// the on-disk doc, so a stale caller can never regress it).
|
|
179
|
+
await writeRecordStore(recordsPath, mutation.records, {
|
|
180
|
+
generation: prior.generation,
|
|
181
|
+
tombstones: mutation.tombstones ?? prior.tombstones,
|
|
182
|
+
});
|
|
183
|
+
// Test-only crash-injection hook (TASK-002): UKIT_TEST_CRASH_AT=
|
|
184
|
+
// 'post-doc-write' kills the process after the doc rename but before
|
|
185
|
+
// postWrite (journal append), proving crash recovery never duplicates.
|
|
186
|
+
if (process.env.UKIT_TEST_CRASH_AT === 'post-doc-write') process.exit(1);
|
|
187
|
+
if (typeof mutation.postWrite === 'function') await mutation.postWrite();
|
|
188
|
+
}
|
|
98
189
|
return mutation.result ?? null;
|
|
99
190
|
});
|
|
100
191
|
if (outcome === undefined) {
|
|
@@ -102,3 +193,25 @@ export async function mutateRecordStore(recordsPath, mutate) {
|
|
|
102
193
|
}
|
|
103
194
|
return outcome;
|
|
104
195
|
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Copy the corrupt doc aside for manual review — never overwrite, never
|
|
199
|
+
* delete the original. Exclusive-create destination + 0600 mode (the copy
|
|
200
|
+
* may contain secrets — same boundary as the canonical store, SPEC §14).
|
|
201
|
+
* Best-effort: a failed copy still yields the error path, with the attempted
|
|
202
|
+
* path in the message.
|
|
203
|
+
*/
|
|
204
|
+
async function quarantineStoreFile(recordsPath) {
|
|
205
|
+
const quarantinePath = path.join(
|
|
206
|
+
path.dirname(recordsPath),
|
|
207
|
+
'quarantine',
|
|
208
|
+
`${path.basename(recordsPath)}-${Date.now()}.json`,
|
|
209
|
+
);
|
|
210
|
+
try {
|
|
211
|
+
await copyFileRawExclusive(recordsPath, quarantinePath);
|
|
212
|
+
await fs.chmod(quarantinePath, 0o600).catch(() => {});
|
|
213
|
+
} catch {
|
|
214
|
+
// Quarantine copy failed — still fail closed; the original stays untouched.
|
|
215
|
+
}
|
|
216
|
+
return quarantinePath;
|
|
217
|
+
}
|