@ngockhoale/ukit 3.5.1 → 3.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -297
- package/README.md +1 -0
- package/manifests/platform.full.yaml +23 -0
- package/package.json +8 -1
- package/src/cli/commands/doctor.js +14 -2
- package/src/cli/commands/memory.js +30 -2
- package/src/core/memory/memoryHit.js +2 -0
- package/src/core/memory/mutateMemory.js +9 -0
- package/src/core/memory/projectIdentity.js +9 -2
- package/src/core/memory/recallEngine.js +86 -4
- package/src/core/memory/registryPrune.js +230 -0
- package/src/core/memory/retrieval.js +12 -3
- package/src/core/runtimeConfig.js +1 -0
- package/src/index/buildIndex.js +22 -0
- package/src/index/impactContext.js +11 -1
- package/src/index/queryIndex.js +151 -20
- package/template_project/.claude/hooks/edit-hint.mjs +119 -0
- package/template_project/.claude/settings.json +2 -2
- package/template_project/.claude/ukit/index/edit-graph.mjs +404 -0
- package/template_project/.claude/ukit/index/lib/index-core.mjs +185 -20
- package/template_project/.claude/ukit/index/post-edit-verify.mjs +8 -5
- package/template_project/.claude/ukit/index/resolve-context.mjs +17 -0
- package/template_project/.claude/ukit/index/route-task.mjs +4 -1
- package/template_project/.claude/ukit/runtime/hook-chain-runner.mjs +62 -5
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +259 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +86 -4
- package/template_project/.claude/ukit/runtime/memory-validity.mjs +232 -0
- package/template_project/.claude/ukit/runtime/reinject-context.mjs +4 -2
- package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
- package/template_project/AGENTS.md +1 -1
- package/template_project/CLAUDE.md +1 -1
- package/template_project/instructions/core.md +1 -1
- package/src/core/observability/evaluation/experimentPlan.js +0 -95
- package/src/core/observability/evaluation/perturbation.js +0 -273
- package/src/diagnostics/classifyHang.js +0 -246
- package/src/diagnostics/ompStallSnapshot.js +0 -199
- package/src/render/instructionRenderer.js +0 -290
- package/src/skill/auditSkill.js +0 -140
- package/template_project/instructions/overlays/repo.md +0 -9
|
@@ -31,6 +31,8 @@ import { pathExists } from '../../core/fileOps.js';
|
|
|
31
31
|
import { detectProjectContext } from '../../context/detectProjectContext.js';
|
|
32
32
|
import { episodeText, readRouteState, resolveLedger, writeEpisode } from '../../core/memory/episodes.js';
|
|
33
33
|
import { archiveLegacyEpisodes, purgeArchivedEpisode, restoreArchivedEpisodes } from '../../core/memory/episodeArchive.js';
|
|
34
|
+
import { loadMarkdownDocs, normalizeMemorySources } from '../../core/memory/recallEngine.js';
|
|
35
|
+
import { captureAnchors } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
|
|
34
36
|
import fs from 'node:fs/promises';
|
|
35
37
|
import os from 'node:os';
|
|
36
38
|
import path from 'node:path';
|
|
@@ -167,6 +169,32 @@ async function runMemoryV2(projectRoot, homeDir, args) {
|
|
|
167
169
|
return;
|
|
168
170
|
}
|
|
169
171
|
|
|
172
|
+
if (op === 'confirm') {
|
|
173
|
+
// Re-anchor a note to its named files' current state (clears the
|
|
174
|
+
// "[source changed since noted — verify]" marker). Record id or markdown path:line.
|
|
175
|
+
const id = rest.join(' ').trim();
|
|
176
|
+
if (!id) {
|
|
177
|
+
throw new Error('Missing id. Usage: ukit memory v2 confirm <record-id | path:line>');
|
|
178
|
+
}
|
|
179
|
+
let doc = null;
|
|
180
|
+
const record = await getRecord(projectRoot, id, { homeDir });
|
|
181
|
+
if (record) {
|
|
182
|
+
doc = { id: `record:${record.id}`, type: record.type, provenance: record.provenance, text: record.text, evidence: record.evidence };
|
|
183
|
+
} else {
|
|
184
|
+
const { config } = await inspectRuntimeConfig(projectRoot, { homeDir });
|
|
185
|
+
const mdDocs = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config)).catch(() => []);
|
|
186
|
+
doc = mdDocs.find((d) => d.provenance === id || d.id === id) ?? null;
|
|
187
|
+
}
|
|
188
|
+
if (!doc) {
|
|
189
|
+
throw new Error(`Memory item not found: ${id}`);
|
|
190
|
+
}
|
|
191
|
+
const n = await captureAnchors(projectRoot, [doc]);
|
|
192
|
+
console.log(n > 0
|
|
193
|
+
? `[UKit] confirmed ${id}: anchored to current source state.`
|
|
194
|
+
: `[UKit] ${id} names no existing file; nothing to anchor.`);
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
|
|
170
198
|
if (op === 'promote') {
|
|
171
199
|
const id = rest.join(' ').trim();
|
|
172
200
|
if (!id) {
|
|
@@ -276,7 +304,7 @@ async function runMemoryV2(projectRoot, homeDir, args) {
|
|
|
276
304
|
console.log(`[UKit] v2 store: ${stats.total} record(s) — ${JSON.stringify(stats.byType)}`);
|
|
277
305
|
return;
|
|
278
306
|
}
|
|
279
|
-
throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|promote|stale|migrate.`);
|
|
307
|
+
throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|confirm|promote|stale|migrate.`);
|
|
280
308
|
}
|
|
281
309
|
|
|
282
310
|
// ---- `ukit memory --user` — user-level store at ~/.ukit (FR-011) ----
|
|
@@ -1029,6 +1057,6 @@ export function printMemoryHelp() {
|
|
|
1029
1057
|
console.log(' backup [--out <dir>] [--user] Snapshot v2 store(s) + sha256 manifest into a backup dir');
|
|
1030
1058
|
console.log(' restore <dir> [--dry-run] [--user] Restore store(s) from a backup dir (plan only with --dry-run)');
|
|
1031
1059
|
console.log(' doctor [--user] Diagnose v2 store state + counts + quarantine (read-only)');
|
|
1032
|
-
console.log(' v2 <list|show|promote|stale|migrate> Operate on the project v2 record store (migrate supports --dry-run plan output)');
|
|
1060
|
+
console.log(' v2 <list|show|confirm|promote|stale|migrate> Operate on the project v2 record store (migrate supports --dry-run plan output)');
|
|
1033
1061
|
console.log(' --user [v2] <list|add|get|update|forget|purge|stats> Operate on the user-level store at ~/.ukit/storage/memory/v2/records.json (purge requires --confirm)');
|
|
1034
1062
|
}
|
|
@@ -67,6 +67,7 @@ export function buildMemoryHit(record, { freshness = null, score = 0, reasons =
|
|
|
67
67
|
summary: snippet,
|
|
68
68
|
source: record?.type ?? null,
|
|
69
69
|
timestamp: createdAt,
|
|
70
|
+
...(record?.validity ? { validity: record.validity } : {}),
|
|
70
71
|
};
|
|
71
72
|
}
|
|
72
73
|
|
|
@@ -115,6 +116,7 @@ export function buildMarkdownHit(doc, { score = 0 } = {}) {
|
|
|
115
116
|
source: 'markdown',
|
|
116
117
|
timestamp: 0,
|
|
117
118
|
provenance: doc.provenance,
|
|
119
|
+
...(doc.validity ? { validity: doc.validity } : {}),
|
|
118
120
|
};
|
|
119
121
|
}
|
|
120
122
|
|
|
@@ -28,6 +28,7 @@ import { guardWritePayload, redactWritePayload } from './writeGuard.js';
|
|
|
28
28
|
import { classifyWrite, mergeEvidence, REVIEW_QUEUE_MAX } from './writeClassification.js';
|
|
29
29
|
import { resolveMemoryStage, appendRolloutReceipt, latencyBandForMs } from './memoryFlags.js';
|
|
30
30
|
import { resolveProjectIdentity } from './projectIdentity.js';
|
|
31
|
+
import { captureAnchors } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
|
|
31
32
|
|
|
32
33
|
const JOURNAL_MAX = 256;
|
|
33
34
|
const OPS = new Set(['add', 'update', 'archive', 'forget', 'purge', 'restore', 'relocate']);
|
|
@@ -351,6 +352,14 @@ export async function mutateMemory(target, envelope = {}, { classifyAdd, guard,
|
|
|
351
352
|
latencyBand: latencyBandForMs(Date.now() - startedAt),
|
|
352
353
|
});
|
|
353
354
|
}
|
|
355
|
+
// 3.5.3: anchor a freshly written note to the files it names (sidecar only,
|
|
356
|
+
// outside the store lock, never fails the write).
|
|
357
|
+
if (result?.status === 'ok' && op === 'add' && target.kind === 'project' && result.record && result.reason !== 'review') {
|
|
358
|
+
await captureAnchors(target.projectRoot, [{
|
|
359
|
+
id: `record:${result.record.id}`, type: result.record.type, provenance: result.record.provenance,
|
|
360
|
+
text: result.record.text, evidence: result.record.evidence,
|
|
361
|
+
}]).catch(() => 0);
|
|
362
|
+
}
|
|
354
363
|
return result;
|
|
355
364
|
} catch (error) {
|
|
356
365
|
if (error instanceof StoreCorruptError || error?.name === 'StoreCorruptError') {
|
|
@@ -3,6 +3,7 @@ import path from 'node:path';
|
|
|
3
3
|
import crypto from 'node:crypto';
|
|
4
4
|
import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
|
|
5
5
|
import { buildUserPaths } from '../userPaths.js';
|
|
6
|
+
import { maybeAutoPrune } from './registryPrune.js';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Host-owned stable project identity resolver (M01-01, SPEC §5 FR-001..003).
|
|
@@ -225,7 +226,7 @@ export async function resolveProjectIdentity(projectRoot, { homeDir, mint = true
|
|
|
225
226
|
if (fresh === null) return null; // became corrupt between read and lock
|
|
226
227
|
const existing = findByRoot(fresh, realRoot)
|
|
227
228
|
?? (commonDir ? findByGitCommonDir(fresh, commonDir) : null);
|
|
228
|
-
if (existing) return existing;
|
|
229
|
+
if (existing) return { projectId: existing, entryCount: 0 };
|
|
229
230
|
|
|
230
231
|
const now = new Date().toISOString();
|
|
231
232
|
const projectId = mintProjectId(realRoot);
|
|
@@ -238,10 +239,16 @@ export async function resolveProjectIdentity(projectRoot, { homeDir, mint = true
|
|
|
238
239
|
};
|
|
239
240
|
indexAliases(fresh, projectId, aliases);
|
|
240
241
|
await writeJson(regPath, fresh);
|
|
241
|
-
return projectId;
|
|
242
|
+
return { projectId, entryCount: Object.keys(fresh.projects).length };
|
|
242
243
|
});
|
|
243
244
|
|
|
244
245
|
if (minted === undefined || minted === null) return unverified;
|
|
246
|
+
if (typeof minted === 'object') {
|
|
247
|
+
// Amortised stale-entry cleanup (never awaited by the caller's deadline path beyond its
|
|
248
|
+
// own bounded budget; failure-proof). Runs after the lock is released.
|
|
249
|
+
await maybeAutoPrune({ homeDir, entryCount: minted.entryCount, currentRoot: realRoot });
|
|
250
|
+
return { projectId: minted.projectId, source: 'registry', ambiguous: false, aliases };
|
|
251
|
+
}
|
|
245
252
|
return { projectId: minted, source: 'registry', ambiguous: false, aliases };
|
|
246
253
|
}
|
|
247
254
|
|
|
@@ -161,6 +161,69 @@ export function docTokens(doc) {
|
|
|
161
161
|
return tokenize(text.replace(EPISODE_LABEL_RE, ' '));
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
// Source-validity marker shown next to an item whose anchored file changed or
|
|
165
|
+
// vanished (set on docs as `validity` by memory-validity.mjs).
|
|
166
|
+
export const STALE_MARKER = '[source changed since noted — verify]';
|
|
167
|
+
export const RRF_K = 60;
|
|
168
|
+
export const validityRank = (doc) => (doc?.validity === 'missing' ? 2 : doc?.validity === 'changed' ? 1 : 0);
|
|
169
|
+
const bareId = (id) => String(id ?? '').replace(/^record:/, '');
|
|
170
|
+
|
|
171
|
+
// A doc is superseded when it says so (`superseded_by`) or another candidate
|
|
172
|
+
// lists it in `supersedes`: the superseding doc wins even if the old text
|
|
173
|
+
// scores higher, so the old one never reaches the ranking.
|
|
174
|
+
export function dropSuperseded(docs) {
|
|
175
|
+
const replaced = new Set();
|
|
176
|
+
for (const doc of docs) {
|
|
177
|
+
if (!doc || !Array.isArray(doc.supersedes)) continue;
|
|
178
|
+
for (const id of doc.supersedes) replaced.add(bareId(id));
|
|
179
|
+
}
|
|
180
|
+
return docs.filter((doc) => doc != null && !doc.superseded_by && !replaced.has(bareId(doc.id)));
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Reciprocal rank fusion over ordered id lists (best first); lists are lanes.
|
|
184
|
+
// Score = sum weight/(k + rank). Ties break on id, never on a timestamp.
|
|
185
|
+
export function fuseRanked(lanes, { k = RRF_K, weights = [] } = {}) {
|
|
186
|
+
const scores = new Map();
|
|
187
|
+
(Array.isArray(lanes) ? lanes : []).forEach((lane, idx) => {
|
|
188
|
+
const weight = Number.isFinite(weights[idx]) ? weights[idx] : 1;
|
|
189
|
+
const seen = new Set();
|
|
190
|
+
let rank = 0;
|
|
191
|
+
for (const id of lane ?? []) {
|
|
192
|
+
if (seen.has(id)) continue;
|
|
193
|
+
seen.add(id);
|
|
194
|
+
rank += 1;
|
|
195
|
+
scores.set(id, (scores.get(id) ?? 0) + weight / (k + rank));
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
return [...scores].map(([id, score]) => ({ id, score }))
|
|
199
|
+
.sort((a, b) => b.score - a.score || String(a.id).localeCompare(String(b.id)));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// Validity as one more fusion lane (3.5.3). `changed` (an anchored source was
|
|
203
|
+
// edited) is always labelled but only mildly demoted: lane 1 is the incoming
|
|
204
|
+
// relevance order, lane 2 the same items valid-first, fused with a small weight,
|
|
205
|
+
// so a clearly more relevant changed note still beats a weakly relevant valid
|
|
206
|
+
// one. `missing` (file gone) is labelled and ranked last. Nothing is dropped.
|
|
207
|
+
// Measurement knob UKIT_MEMORY_VALIDITY_RANK: `hard` (every changed below every
|
|
208
|
+
// valid), `label` (marker only, no rank change), `mild[:<weight>]` (default).
|
|
209
|
+
export const VALIDITY_WEIGHT = 0.35;
|
|
210
|
+
const pad = (i) => String(i).padStart(6, '0');
|
|
211
|
+
export function demoteByValidity(items, docOf = (item) => item?.doc ?? item) {
|
|
212
|
+
const list = Array.isArray(items) ? [...items] : [];
|
|
213
|
+
if (!list.some((item) => validityRank(docOf(item)) > 0)) return list;
|
|
214
|
+
const [mode, raw] = String(process.env.UKIT_MEMORY_VALIDITY_RANK ?? 'mild').split(':');
|
|
215
|
+
if (mode === 'label') return list;
|
|
216
|
+
const rank = (item) => validityRank(docOf(item));
|
|
217
|
+
if (mode === 'hard') return list.sort((a, b) => rank(a) - rank(b));
|
|
218
|
+
const weight = Number.isFinite(Number(raw)) && raw !== undefined && raw !== '' ? Number(raw) : VALIDITY_WEIGHT;
|
|
219
|
+
const byIndex = list.map((item, i) => ({ item, i }));
|
|
220
|
+
const lane1 = byIndex.map((e) => pad(e.i));
|
|
221
|
+
const lane2 = [...byIndex].sort((a, b) => rank(a.item) - rank(b.item)).map((e) => pad(e.i));
|
|
222
|
+
return fuseRanked([lane1, lane2], { weights: [1, weight] })
|
|
223
|
+
.map((e) => list[Number(e.id)])
|
|
224
|
+
.sort((a, b) => Number(rank(a) === 2) - Number(rank(b) === 2));
|
|
225
|
+
}
|
|
226
|
+
|
|
164
227
|
function idf(totalDocs, docFreq) {
|
|
165
228
|
return Math.log(1 + (totalDocs - docFreq + 0.5) / (docFreq + 0.5));
|
|
166
229
|
}
|
|
@@ -176,6 +239,8 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
176
239
|
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
177
240
|
};
|
|
178
241
|
if (!Array.isArray(docs) || docs.length === 0 || queryTokens.length === 0) return empty;
|
|
242
|
+
docs = dropSuperseded(docs);
|
|
243
|
+
if (docs.length === 0) return empty;
|
|
179
244
|
|
|
180
245
|
const stats = [];
|
|
181
246
|
const docFreq = new Map();
|
|
@@ -222,7 +287,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
222
287
|
scored.sort((a, b) => b.score - a.score
|
|
223
288
|
|| String(a.doc.id ?? '').localeCompare(String(b.doc.id ?? '')));
|
|
224
289
|
const top = scored.length > 0 ? scored[0].score : 0;
|
|
225
|
-
|
|
290
|
+
// Temporal lane (3.5.3): validity is one more fusion lane over the docs that
|
|
291
|
+
// cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
|
|
292
|
+
const ranked = demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc)
|
|
293
|
+
.slice(0, limit);
|
|
226
294
|
return {
|
|
227
295
|
ranked,
|
|
228
296
|
candidateCount,
|
|
@@ -359,16 +427,30 @@ const SNIPPET_CHARS = 160;
|
|
|
359
427
|
|
|
360
428
|
// Shared hook lane: score ALL eligible docs (no newest-N pre-filter) together
|
|
361
429
|
// with the markdown source bullets, then cut. Async I/O only; never throws.
|
|
362
|
-
|
|
430
|
+
// `annotate(docs) -> Promise<docs>` is an optional caller-injected async step
|
|
431
|
+
// (source validity, 3.5.3); this file stays free of non-node imports. A failing
|
|
432
|
+
// or slow-to-throw annotate never affects recall.
|
|
433
|
+
export async function recallForHook({
|
|
434
|
+
projectRoot, query, docs = [], config = null, limit = 2, annotate = null,
|
|
435
|
+
} = {}) {
|
|
363
436
|
let markdown = [];
|
|
364
437
|
try {
|
|
365
438
|
markdown = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config));
|
|
366
439
|
} catch {
|
|
367
440
|
markdown = [];
|
|
368
441
|
}
|
|
442
|
+
let pool = [...docs, ...markdown];
|
|
443
|
+
if (typeof annotate === 'function') {
|
|
444
|
+
try {
|
|
445
|
+
const annotated = await annotate(pool);
|
|
446
|
+
if (Array.isArray(annotated) && annotated.length === pool.length) pool = annotated;
|
|
447
|
+
} catch {
|
|
448
|
+
// keep the unannotated pool
|
|
449
|
+
}
|
|
450
|
+
}
|
|
369
451
|
let result;
|
|
370
452
|
try {
|
|
371
|
-
result = rankDocs(
|
|
453
|
+
result = rankDocs(pool, query, { limit });
|
|
372
454
|
} catch {
|
|
373
455
|
result = rankDocs([], '', { limit });
|
|
374
456
|
}
|
|
@@ -419,5 +501,5 @@ async function emitRecallTelemetry(projectRoot, config, result) {
|
|
|
419
501
|
export function renderMarkdownDoc(doc, maxChars = SNIPPET_CHARS) {
|
|
420
502
|
const text = String(doc?.text ?? '');
|
|
421
503
|
const body = text.length > maxChars ? `${text.slice(0, maxChars - 3)}...` : text;
|
|
422
|
-
return `[${doc?.provenance ?? 'markdown'}] ${body}`;
|
|
504
|
+
return `[${doc?.provenance ?? 'markdown'}] ${body}${doc?.validity ? ` ${STALE_MARKER}` : ''}`;
|
|
423
505
|
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { withFileLock, writeJson, readJsonIfExists } from '../fileOps.js';
|
|
4
|
+
import { buildUserPaths } from '../userPaths.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Stale-entry pruning for the user-level project registry
|
|
8
|
+
* (`~/.ukit/storage/projects/registry.json`, owned by projectIdentity.js).
|
|
9
|
+
*
|
|
10
|
+
* An entry is stale only when EVERY root AND every git common dir provably no longer exists
|
|
11
|
+
* (a project can resolve through its git common dir after the worktree that registered it is gone):
|
|
12
|
+
* - stat of the path fails with ENOENT/ENOTDIR (any other error => keep), AND
|
|
13
|
+
* - the nearest mount-level ancestor is present and populated (an unmounted
|
|
14
|
+
* external/network volume must never make its projects look deleted), AND
|
|
15
|
+
* - the entry is not the project being registered, AND
|
|
16
|
+
* - no user-level memory record is attributed to the entry's project id.
|
|
17
|
+
* Mutation re-reads the registry under the same lock and uses the same atomic
|
|
18
|
+
* writer as registration; decisions made outside the lock are re-validated.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export const AUTO_PRUNE_MIN_ENTRIES = 500;
|
|
22
|
+
export const AUTO_PRUNE_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
23
|
+
export const AUTO_PRUNE_BUDGET_MS = 1500;
|
|
24
|
+
const STAT_CONCURRENCY = 32;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Pure: nearest mount-level ancestor whose presence is required before a missing
|
|
28
|
+
* root may be called deleted. Returns null when the path has no such ancestor
|
|
29
|
+
* (plain local path: the filesystem root is always there).
|
|
30
|
+
* `emptyIsUnmounted` tells the caller an empty mount-point dir is not proof.
|
|
31
|
+
*/
|
|
32
|
+
export function mountAncestorOf(rootPath, platform = process.platform) {
|
|
33
|
+
if (typeof rootPath !== 'string' || rootPath === '') return null;
|
|
34
|
+
if (platform === 'win32') {
|
|
35
|
+
const p = path.win32;
|
|
36
|
+
const parsed = p.parse(p.normalize(rootPath));
|
|
37
|
+
if (!parsed.root) return null;
|
|
38
|
+
// Drive root (C:\) or UNC share root (\\server\share\).
|
|
39
|
+
return { path: parsed.root, emptyIsUnmounted: false };
|
|
40
|
+
}
|
|
41
|
+
const parts = path.posix.normalize(rootPath).split('/').filter(Boolean);
|
|
42
|
+
const take = (n) => ({ path: `/${parts.slice(0, n).join('/')}`, emptyIsUnmounted: true });
|
|
43
|
+
if (parts[0] === 'Volumes' && parts.length >= 2) {
|
|
44
|
+
return { path: `/${parts.slice(0, 2).join('/')}`, emptyIsUnmounted: false };
|
|
45
|
+
}
|
|
46
|
+
if (parts[0] === 'mnt' && parts.length >= 2) return take(2);
|
|
47
|
+
if (parts[0] === 'media' && parts.length >= 3) return take(3);
|
|
48
|
+
if (parts[0] === 'run' && parts[1] === 'media' && parts.length >= 4) return take(4);
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Classify one root. `io` = { stat(p), readdir(p) } (injectable for tests).
|
|
54
|
+
* @returns {Promise<'missing'|'present'|'unproven'>}
|
|
55
|
+
*/
|
|
56
|
+
export async function classifyRoot(rootPath, { platform = process.platform, io, mountCache } = {}) {
|
|
57
|
+
const fsio = io ?? { stat: (p) => fs.stat(p), readdir: (p) => fs.readdir(p) };
|
|
58
|
+
try {
|
|
59
|
+
await fsio.stat(rootPath);
|
|
60
|
+
return 'present';
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (error?.code !== 'ENOENT' && error?.code !== 'ENOTDIR') return 'unproven';
|
|
63
|
+
}
|
|
64
|
+
const mount = mountAncestorOf(rootPath, platform);
|
|
65
|
+
if (!mount) return 'missing';
|
|
66
|
+
let ok = mountCache?.get(mount.path);
|
|
67
|
+
if (ok === undefined) {
|
|
68
|
+
ok = false;
|
|
69
|
+
try {
|
|
70
|
+
await fsio.stat(mount.path);
|
|
71
|
+
ok = mount.emptyIsUnmounted ? (await fsio.readdir(mount.path)).length > 0 : true;
|
|
72
|
+
} catch {
|
|
73
|
+
ok = false;
|
|
74
|
+
}
|
|
75
|
+
mountCache?.set(mount.path, ok);
|
|
76
|
+
}
|
|
77
|
+
return ok ? 'missing' : 'unproven';
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function entrySignature(entry) {
|
|
81
|
+
return JSON.stringify([entry?.roots, entry?.gitCommonDirs ?? []]);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function registryPaths(homeDir) {
|
|
85
|
+
const user = buildUserPaths(homeDir === undefined ? {} : { homeDir });
|
|
86
|
+
const regPath = path.join(user.storageRoot, 'projects', 'registry.json');
|
|
87
|
+
return { user, regPath, stampPath: path.join(path.dirname(regPath), 'registry.prune-stamp') };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function mapLimit(items, limit, fn, shouldStop) {
|
|
91
|
+
let next = 0;
|
|
92
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
93
|
+
while (next < items.length) {
|
|
94
|
+
if (shouldStop?.()) return;
|
|
95
|
+
const i = next++;
|
|
96
|
+
await fn(items[i], i);
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
await Promise.all(workers);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function rebuildAliasIndex(registry, removedAliases) {
|
|
103
|
+
const names = new Set(removedAliases);
|
|
104
|
+
for (const name of names) {
|
|
105
|
+
const ids = Object.entries(registry.projects)
|
|
106
|
+
.filter(([, entry]) => Array.isArray(entry?.aliases) && entry.aliases.includes(name))
|
|
107
|
+
.map(([id]) => id);
|
|
108
|
+
if (ids.length === 0) delete registry.aliasIndex[name];
|
|
109
|
+
else registry.aliasIndex[name] = ids.length > 1 ? 'ambiguous' : ids;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Prune stale registry entries.
|
|
115
|
+
* @param {object} o
|
|
116
|
+
* @param {string} [o.homeDir]
|
|
117
|
+
* @param {boolean} [o.dryRun] decide + count only, never write
|
|
118
|
+
* @param {string[]} [o.keepRoots] roots that must never be pruned (current project)
|
|
119
|
+
* @param {number} [o.budgetMs] stop stat-ing after this long (partial result)
|
|
120
|
+
* @param {string} [o.platform]
|
|
121
|
+
* @param {object} [o.io] injectable stat/readdir
|
|
122
|
+
* @returns {Promise<{status:string, total:number, checked:number, stale:number, removed:number,
|
|
123
|
+
* keptWithMemory:number, unproven:number, truncated:boolean}>}
|
|
124
|
+
*/
|
|
125
|
+
export async function pruneRegistry({
|
|
126
|
+
homeDir, dryRun = false, keepRoots = [], budgetMs = Infinity, platform = process.platform, io,
|
|
127
|
+
} = {}) {
|
|
128
|
+
const { user, regPath } = registryPaths(homeDir);
|
|
129
|
+
const result = { status: 'ok', total: 0, checked: 0, stale: 0, removed: 0, keptWithMemory: 0, unproven: 0, truncated: false };
|
|
130
|
+
|
|
131
|
+
let registry;
|
|
132
|
+
try {
|
|
133
|
+
registry = await readJsonIfExists(regPath);
|
|
134
|
+
} catch {
|
|
135
|
+
return { ...result, status: 'registry-corrupt' };
|
|
136
|
+
}
|
|
137
|
+
if (!registry || typeof registry.projects !== 'object' || registry.projects === null) {
|
|
138
|
+
return { ...result, status: registry ? 'registry-corrupt' : 'no-registry' };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// Conservative: read the RAW user-memory document (not normalized) so even records this build
|
|
142
|
+
// would skip still protect their project id. Project-scoped memory lives under the project's own
|
|
143
|
+
// `.ukit/` (gone with the root) and is unreachable; the user-level store is the only data keyed by id.
|
|
144
|
+
let userDoc;
|
|
145
|
+
try {
|
|
146
|
+
userDoc = await readJsonIfExists(user.memoryV2RecordsPath);
|
|
147
|
+
} catch {
|
|
148
|
+
return { ...result, status: 'user-memory-unreadable' };
|
|
149
|
+
}
|
|
150
|
+
const memoryIds = new Set(
|
|
151
|
+
(Array.isArray(userDoc?.records) ? userDoc.records : [])
|
|
152
|
+
.map((r) => r?.project_id).filter((id) => typeof id === 'string'),
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
const keep = new Set(keepRoots);
|
|
156
|
+
const entries = Object.entries(registry.projects);
|
|
157
|
+
result.total = entries.length;
|
|
158
|
+
const started = Date.now();
|
|
159
|
+
const mountCache = new Map();
|
|
160
|
+
const staleIds = new Map(); // id -> roots signature
|
|
161
|
+
const shouldStop = () => Date.now() - started > budgetMs;
|
|
162
|
+
|
|
163
|
+
await mapLimit(entries, STAT_CONCURRENCY, async ([id, entry]) => {
|
|
164
|
+
result.checked += 1;
|
|
165
|
+
const roots = entry?.roots;
|
|
166
|
+
if (!Array.isArray(roots) || roots.length === 0 || roots.some((r) => typeof r !== 'string')) return;
|
|
167
|
+
// gitCommonDirs: absent => none; present but malformed => keep (cannot prove anything).
|
|
168
|
+
const gitDirs = entry?.gitCommonDirs ?? [];
|
|
169
|
+
if (!Array.isArray(gitDirs) || gitDirs.some((g) => typeof g !== 'string')) return;
|
|
170
|
+
if (roots.some((r) => keep.has(r))) return;
|
|
171
|
+
for (const p of [...roots, ...gitDirs]) {
|
|
172
|
+
const verdict = await classifyRoot(p, { platform, io, mountCache });
|
|
173
|
+
if (verdict !== 'missing') {
|
|
174
|
+
if (verdict === 'unproven') result.unproven += 1;
|
|
175
|
+
return; // any present/unproven root or git directory keeps the entry
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (memoryIds.has(id)) { result.keptWithMemory += 1; return; }
|
|
179
|
+
staleIds.set(id, entrySignature(entry));
|
|
180
|
+
}, shouldStop);
|
|
181
|
+
|
|
182
|
+
result.truncated = result.checked < entries.length && shouldStop();
|
|
183
|
+
result.stale = staleIds.size;
|
|
184
|
+
if (dryRun || staleIds.size === 0) return result;
|
|
185
|
+
|
|
186
|
+
const outcome = await withFileLock(regPath, async () => {
|
|
187
|
+
const fresh = await readJsonIfExists(regPath).catch(() => null);
|
|
188
|
+
if (!fresh || typeof fresh.projects !== 'object' || fresh.projects === null) return 0;
|
|
189
|
+
if (typeof fresh.aliasIndex !== 'object' || fresh.aliasIndex === null) fresh.aliasIndex = {};
|
|
190
|
+
const removedAliases = [];
|
|
191
|
+
let removed = 0;
|
|
192
|
+
for (const [id, signature] of staleIds) {
|
|
193
|
+
const current = fresh.projects[id];
|
|
194
|
+
// Re-validate: entry must still exist unchanged (a concurrent writer may have re-bound it).
|
|
195
|
+
if (!current || entrySignature(current) !== signature) continue;
|
|
196
|
+
if (Array.isArray(current.aliases)) removedAliases.push(...current.aliases);
|
|
197
|
+
delete fresh.projects[id];
|
|
198
|
+
removed += 1;
|
|
199
|
+
}
|
|
200
|
+
if (removed === 0) return 0;
|
|
201
|
+
rebuildAliasIndex(fresh, removedAliases);
|
|
202
|
+
await writeJson(regPath, fresh);
|
|
203
|
+
return removed;
|
|
204
|
+
});
|
|
205
|
+
if (outcome === undefined) return { ...result, status: 'lock-busy' };
|
|
206
|
+
return { ...result, removed: outcome };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Bounded automatic prune used during registration. Never throws; async fs only;
|
|
211
|
+
* skipped below the entry threshold or when run within the last day.
|
|
212
|
+
*/
|
|
213
|
+
export async function maybeAutoPrune({ homeDir, entryCount, currentRoot, now = Date.now() } = {}) {
|
|
214
|
+
try {
|
|
215
|
+
if (!(entryCount >= AUTO_PRUNE_MIN_ENTRIES)) return null;
|
|
216
|
+
const { stampPath } = registryPaths(homeDir);
|
|
217
|
+
try {
|
|
218
|
+
const st = await fs.stat(stampPath);
|
|
219
|
+
if (now - st.mtimeMs < AUTO_PRUNE_INTERVAL_MS) return null;
|
|
220
|
+
} catch { /* no stamp yet */ }
|
|
221
|
+
// Stamp first: concurrent/failed runs do not retry within the interval.
|
|
222
|
+
await fs.mkdir(path.dirname(stampPath), { recursive: true });
|
|
223
|
+
await fs.writeFile(stampPath, `${new Date(now).toISOString()}\n`);
|
|
224
|
+
return await pruneRegistry({
|
|
225
|
+
homeDir, keepRoots: currentRoot ? [currentRoot] : [], budgetMs: AUTO_PRUNE_BUDGET_MS,
|
|
226
|
+
});
|
|
227
|
+
} catch {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
@@ -16,6 +16,8 @@ import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './r
|
|
|
16
16
|
import { resolveRecordFreshness } from './memoryFreshness.js';
|
|
17
17
|
import { getHeadSha } from '../codeintel/freshness.js';
|
|
18
18
|
import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
|
|
19
|
+
import { withValidity } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
|
|
20
|
+
import { STALE_MARKER } from './recallEngine.js';
|
|
19
21
|
|
|
20
22
|
function compactPhraseList(values, { limit = 1, fallback = 'n/a' } = {}) {
|
|
21
23
|
const normalized = [...new Set(
|
|
@@ -329,8 +331,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
|
|
|
329
331
|
if (!v2AllowsLegacyFallback(state)) {
|
|
330
332
|
// One engine (recallEngine.js BM25): eligibility first, then IDF over the
|
|
331
333
|
// eligible set + markdown bullets, floor, explicit empty result.
|
|
332
|
-
|
|
333
|
-
|
|
334
|
+
// Source validity (3.5.3): anchored items whose file changed/vanished are
|
|
335
|
+
// demoted and labelled. The CLI is the lazy-capture point for never-seen items.
|
|
336
|
+
const anchored = await withValidity(projectRoot, [...records, ...markdownDocs], { capture: true });
|
|
337
|
+
const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
|
|
338
|
+
query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
|
|
334
339
|
});
|
|
335
340
|
const results = await attachFreshness(ranked.hits, records, { projectRoot, watermarks, now });
|
|
336
341
|
emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
|
|
@@ -628,7 +633,9 @@ export async function getContextInjection(
|
|
|
628
633
|
// `path:line`; they are never records and carry no evidence/freshness.
|
|
629
634
|
const markdownRecord = (entry) => ({
|
|
630
635
|
id: entry.id, type: 'markdown', text: entry.snippet, provenance: entry.provenance, evidence: [],
|
|
636
|
+
...(entry.validity ? { validity: entry.validity } : {}),
|
|
631
637
|
});
|
|
638
|
+
const validityOf = new Map(rankedItems.filter((entry) => entry.validity).map((entry) => [entry.id, entry.validity]));
|
|
632
639
|
const markdownSelected = rankedItems.filter((entry) => entry.type === 'markdown').map(markdownRecord);
|
|
633
640
|
const selectedItems = v2Active
|
|
634
641
|
? dedupeItemsById(rankedItems
|
|
@@ -636,6 +643,7 @@ export async function getContextInjection(
|
|
|
636
643
|
? markdownRecord(entry)
|
|
637
644
|
: records.find((record) => record.id === entry.id)))
|
|
638
645
|
.filter(Boolean)
|
|
646
|
+
.map((record) => (validityOf.has(record.id) && !record.validity ? { ...record, validity: validityOf.get(record.id) } : record))
|
|
639
647
|
.filter((record) => record.type === 'markdown' || eligible(record, eligibleCtx, policy).ok))
|
|
640
648
|
.slice(0, Math.max(1, recallPlan.limit))
|
|
641
649
|
: [
|
|
@@ -697,7 +705,8 @@ export async function getContextInjection(
|
|
|
697
705
|
|
|
698
706
|
// FR-005: ' [stale]' appended only when freshness resolves stale; unknown
|
|
699
707
|
// adds no token.
|
|
700
|
-
const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '')
|
|
708
|
+
const staleTag = (record) => (freshnessById.get(record.id)?.state === 'stale' ? ' [stale]' : '')
|
|
709
|
+
+ (record.validity && record.type !== 'markdown' ? ` ${STALE_MARKER}` : '');
|
|
701
710
|
const rawBodyLines = v2Active
|
|
702
711
|
? selectedItems.map((record) => `- [${record.type}] ${buildRecordSnippet(record)}${staleTag(record)}`)
|
|
703
712
|
: selectedItems.map((item) => (item.type === 'markdown'
|
|
@@ -532,6 +532,7 @@ export function buildDefaultRuntimeConfig(overrides = {}) {
|
|
|
532
532
|
maxPatternCandidates: 50,
|
|
533
533
|
redactSecrets: true,
|
|
534
534
|
sources: ['docs/MEMORY.md'],
|
|
535
|
+
editHints: { enabled: true, mirrors: [], ignoreMirrors: [], coChange: false },
|
|
535
536
|
},
|
|
536
537
|
validation: {
|
|
537
538
|
enabled: true,
|
package/src/index/buildIndex.js
CHANGED
|
@@ -11,6 +11,8 @@ import { withAsyncLock } from '../../template_project/.claude/ukit/runtime/async
|
|
|
11
11
|
|
|
12
12
|
import { getArtifactPath, getIndexDir, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION, isLikelyTestFilePath, normalizeRelative } from './paths.js';
|
|
13
13
|
import { importsNeedAliasContext, loadImportAliasContextState, resolveImportSpecifier } from './importResolution.js';
|
|
14
|
+
import { ensureEditGraph } from '../../template_project/.claude/ukit/index/edit-graph.mjs';
|
|
15
|
+
import { MIRROR_PAIRS } from './impactCatalog.js';
|
|
14
16
|
import { clearIndexArtifactCache } from './queryIndex.js';
|
|
15
17
|
import { clearRelatedTestArtifactCache } from './relatedTests.js';
|
|
16
18
|
|
|
@@ -582,6 +584,7 @@ async function buildCodeIndexInner({ rootDir = process.cwd(), changedFiles = nul
|
|
|
582
584
|
[INDEX_ARTIFACTS.relations]: relations.length,
|
|
583
585
|
},
|
|
584
586
|
});
|
|
587
|
+
await refreshEditGraph(absoluteRoot, sourceFingerprint.digest, fileRecords, codeFiles, imports, importAliasContext, signal);
|
|
585
588
|
const preservedRuntimeCaches = canReuseFilesArtifact
|
|
586
589
|
&& canReuseAllParsedArtifacts
|
|
587
590
|
&& canReuseTestsMap
|
|
@@ -1241,6 +1244,7 @@ async function tryIncrementalIndexUpdate({ absoluteRoot, indexDir, changedPaths,
|
|
|
1241
1244
|
clearIndexArtifactCache(absoluteRoot);
|
|
1242
1245
|
clearRelatedTestArtifactCache(absoluteRoot);
|
|
1243
1246
|
}
|
|
1247
|
+
await refreshEditGraph(absoluteRoot, refreshedFingerprint.digest, mergedFileRecords, codeFiles, imports, importAliasContext, signal);
|
|
1244
1248
|
|
|
1245
1249
|
return {
|
|
1246
1250
|
indexDir,
|
|
@@ -2573,6 +2577,24 @@ function areSourceFingerprintsEqual(previousFingerprint, nextFingerprint) {
|
|
|
2573
2577
|
&& previousFingerprint?.digest === nextFingerprint?.digest;
|
|
2574
2578
|
}
|
|
2575
2579
|
|
|
2580
|
+
// Edit graph (mirror/test/import edges): rebuilt only when missing or its digest
|
|
2581
|
+
// differs from the index fingerprint. Never fails the index build.
|
|
2582
|
+
async function refreshEditGraph(rootDir, indexDigest, fileRecords, codeFiles, imports, aliasContext, signal) {
|
|
2583
|
+
await ensureEditGraph({
|
|
2584
|
+
rootDir,
|
|
2585
|
+
indexDigest,
|
|
2586
|
+
signal,
|
|
2587
|
+
inputs: () => {
|
|
2588
|
+
const resolved = buildResolvedImportMap(rootDir, imports, new Set(codeFiles.map((f) => f.filePath)), aliasContext);
|
|
2589
|
+
return {
|
|
2590
|
+
files: fileRecords,
|
|
2591
|
+
imports: resolved,
|
|
2592
|
+
declaredMirrors: MIRROR_PAIRS,
|
|
2593
|
+
};
|
|
2594
|
+
},
|
|
2595
|
+
});
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2576
2598
|
async function writeArtifact(rootDir, artifactName, payload) {
|
|
2577
2599
|
const artifactPath = getArtifactPath(rootDir, artifactName);
|
|
2578
2600
|
const temporaryPath = `${artifactPath}.tmp-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
|
@@ -2,6 +2,7 @@ import fs from 'node:fs/promises';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
|
|
4
4
|
import { getArtifactPath, INDEX_ARTIFACTS, isLikelyTestFilePath } from './paths.js';
|
|
5
|
+
import { loadEditGraph, neighboursOf } from '../../template_project/.claude/ukit/index/edit-graph.mjs';
|
|
5
6
|
import { classifyImpactRisk, findMirrorCounterparts } from './impactCatalog.js';
|
|
6
7
|
import { inferRelatedTestsFromArtifacts, loadRelatedTestArtifacts } from './relatedTests.js';
|
|
7
8
|
import { importsNeedAliasContext, loadImportAliasContext, resolveImportSpecifier } from './importResolution.js';
|
|
@@ -108,7 +109,16 @@ export async function resolveImpactContext({
|
|
|
108
109
|
const callItems = safeItems(callsArtifact);
|
|
109
110
|
const importItems = safeItems(importsArtifact);
|
|
110
111
|
const risk = classifyImpactRisk(normalizedChangedFiles);
|
|
111
|
-
const
|
|
112
|
+
const editGraph = await loadEditGraph(absoluteRoot);
|
|
113
|
+
const mirrorCounterparts = [...findMirrorCounterparts(normalizedChangedFiles)];
|
|
114
|
+
for (const changed of normalizedChangedFiles) {
|
|
115
|
+
for (const mirror of neighboursOf(editGraph, changed).mirrors) {
|
|
116
|
+
if (!mirrorCounterparts.some((m) => m.filePath === mirror.path && m.source === changed)) {
|
|
117
|
+
mirrorCounterparts.push({ filePath: mirror.path, reason: `edit-graph ${mirror.kind} mirror`, source: changed });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
mirrorCounterparts.splice(budget.maxMirrors);
|
|
112
122
|
|
|
113
123
|
const changedFileSet = new Set(normalizedChangedFiles);
|
|
114
124
|
const changedSymbolSet = new Set(normalizedChangedSymbols);
|