@ngockhoale/ukit 3.5.3 → 3.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/manifests/documentation.yaml +12 -1
- package/manifests/platform.full.yaml +12 -0
- package/package.json +27 -1
- package/src/cli/commands/doctor.js +41 -0
- package/src/cli/commands/indexTools.js +10 -4
- package/src/cli/commands/memory.js +77 -6
- package/src/core/memory/episodes.js +106 -7
- package/src/core/memory/memoryCheck.js +241 -0
- package/src/core/memory/memoryCheckRun.js +27 -0
- package/src/core/memory/mutateMemory.js +37 -1
- package/src/core/memory/observations.js +192 -0
- package/src/core/memory/recallEngine.js +120 -7
- package/src/core/memory/records.js +23 -0
- package/src/core/memory/writeGate.js +137 -0
- package/src/core/runtimeConfig.js +11 -1
- package/src/index/buildIndex.js +26 -0
- package/src/index/paths.js +3 -1
- package/src/index/queryIndex.js +225 -10
- package/src/index/resolveContext.js +1 -1
- package/template_project/.claude/hooks/edit-hint.mjs +64 -17
- package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
- package/template_project/.claude/hooks/session-episode.sh +9 -4
- package/template_project/.claude/settings.json +2 -2
- package/template_project/.claude/ukit/index/lib/index-core.mjs +255 -12
- package/template_project/.claude/ukit/index/query-index.mjs +14 -3
- package/template_project/.claude/ukit/index/route-task.mjs +4 -1
- package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +120 -7
- package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
- package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
- package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
- package/template_project/ukit/storage/config.json +4 -2
- package/src/core/agentRuntime/completionGate.js +0 -179
- package/src/core/agentRuntime/context.js +0 -571
- package/src/core/agentRuntime/qualityComparison.js +0 -112
- package/src/core/agentRuntime/resourcePolicy.js +0 -78
- package/src/core/agentRuntime/runtimeSupport.js +0 -255
- package/src/core/codeintel/manifest.js +0 -176
- package/src/core/experiments/deliberation.js +0 -327
- package/src/core/experiments/dynamicWorkflow.js +0 -507
- package/src/core/handoffDocValidator.js +0 -154
- package/src/core/memory/deltaOverlays.js +0 -493
- package/src/core/memory/learningCandidates.js +0 -347
- package/src/core/observability/emit/sessionBoot.js +0 -407
- package/src/core/observability/evaluation/replay.js +0 -149
- package/src/core/observability/evaluation/scorecard.js +0 -465
- package/src/core/observability/schema/compatibility.js +0 -180
- package/src/core/observability/support/renderer.js +0 -130
- package/src/core/reviewPanelAggregate.js +0 -204
- package/src/core/taskBudgetValidator.js +0 -251
- package/src/core/taskProgressGuard.js +0 -227
- package/src/decision/reviewVerdict.js +0 -339
- package/src/decision/runtimeFilter.js +0 -150
- package/src/decision/runtimeScheduler.js +0 -239
- package/src/index/contextPack.js +0 -285
- package/src/index/impactContext.js +0 -360
- package/src/index/reviewPolicy.js +0 -368
- package/src/index/sensitivePath.js +0 -21
|
@@ -23,7 +23,12 @@ import fsp from 'node:fs/promises';
|
|
|
23
23
|
import path from 'node:path';
|
|
24
24
|
|
|
25
25
|
export const RECALL_ENGINE_VERSION = 'bm25-1';
|
|
26
|
-
|
|
26
|
+
// Capped core + every detail file of docs/memory/ (3.5.4). An entry ending in
|
|
27
|
+
// `/*.md` names one directory level, expanded by expandMemorySources; a missing
|
|
28
|
+
// file or directory contributes nothing.
|
|
29
|
+
export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md', 'docs/memory/*.md']);
|
|
30
|
+
const DIR_GLOB_RE = /^[^*?[\]{}]+\/\*\.md$/;
|
|
31
|
+
const MAX_DIR_FILES = 32;
|
|
27
32
|
|
|
28
33
|
const K1 = 1.2;
|
|
29
34
|
const B = 0.75;
|
|
@@ -37,6 +42,8 @@ const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
|
|
|
37
42
|
// ... and the IDF mass of the matched tokens reaches this fraction of the
|
|
38
43
|
// largest IDF the corpus can produce (a token present in exactly one doc).
|
|
39
44
|
const MIN_IDF_MASS_RATIO = 0.5;
|
|
45
|
+
// Episodes only: matched IDF mass must reach this fraction of the whole query's.
|
|
46
|
+
const EPISODE_MIN_COVERAGE = 0.5;
|
|
40
47
|
// Results scoring below this fraction of the best hit are noise tail.
|
|
41
48
|
const RELATIVE_CUTOFF = 0.35;
|
|
42
49
|
// Query-dependent type prior: multiplies the lexical score, so it can only
|
|
@@ -46,6 +53,37 @@ const RELATIVE_CUTOFF = 0.35;
|
|
|
46
53
|
const TYPE_PRIOR = Object.freeze({
|
|
47
54
|
project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
|
|
48
55
|
});
|
|
56
|
+
// Core/detail twins (3.5.4): a capped-core one-liner ends in "Detail: <file>" and
|
|
57
|
+
// its verbatim note sits in that file. Both score on the same words, so one fact
|
|
58
|
+
// would spend two result slots. collapseTwins keeps one of each pair that is
|
|
59
|
+
// present in the same ranked list: `keep` 'detail' (default: the full note) or 'core'.
|
|
60
|
+
const DETAIL_POINTER_RE = /\bDetail:\s+(\S+?\.md)\b/;
|
|
61
|
+
const TWIN_MIN_OVERLAP = 0.5;
|
|
62
|
+
export function collapseTwins(entries, keep = 'detail') {
|
|
63
|
+
if (keep === 'off' || !Array.isArray(entries) || entries.length < 2) return entries;
|
|
64
|
+
const drop = new Set();
|
|
65
|
+
for (const core of entries) {
|
|
66
|
+
const doc = core.doc;
|
|
67
|
+
if (doc?.type !== 'markdown' || drop.has(core)) continue;
|
|
68
|
+
const m = DETAIL_POINTER_RE.exec(doc.text ?? '');
|
|
69
|
+
if (!m) continue;
|
|
70
|
+
const coreTokens = new Set(tokenize(doc.text));
|
|
71
|
+
if (coreTokens.size === 0) continue;
|
|
72
|
+
let best = null;
|
|
73
|
+
let bestOverlap = 0;
|
|
74
|
+
for (const other of entries) {
|
|
75
|
+
if (other === core || drop.has(other) || other.doc?.type !== 'markdown') continue;
|
|
76
|
+
if (!String(other.doc.provenance ?? '').startsWith(`${m[1]}:`)) continue;
|
|
77
|
+
const otherTokens = new Set(tokenize(other.doc.text));
|
|
78
|
+
let shared = 0;
|
|
79
|
+
for (const t of coreTokens) if (otherTokens.has(t)) shared += 1;
|
|
80
|
+
const overlap = shared / coreTokens.size;
|
|
81
|
+
if (overlap > bestOverlap) { bestOverlap = overlap; best = other; }
|
|
82
|
+
}
|
|
83
|
+
if (best && bestOverlap >= TWIN_MIN_OVERLAP) drop.add(keep === 'core' ? best : core);
|
|
84
|
+
}
|
|
85
|
+
return drop.size === 0 ? entries : entries.filter((e) => !drop.has(e));
|
|
86
|
+
}
|
|
49
87
|
const MAX_SOURCES = 8;
|
|
50
88
|
const MAX_SOURCE_BYTES = 512 * 1024;
|
|
51
89
|
const MIN_BULLET_CHARS = 12;
|
|
@@ -233,7 +271,7 @@ function idf(totalDocs, docFreq) {
|
|
|
233
271
|
// abstained }
|
|
234
272
|
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
235
273
|
// `query` is a string or a pre-tokenized array. Never throws.
|
|
236
|
-
export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
274
|
+
export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
237
275
|
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
238
276
|
const empty = {
|
|
239
277
|
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
@@ -261,6 +299,8 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
261
299
|
// IDF cannot discriminate in a tiny corpus, so the multi-token requirement
|
|
262
300
|
// only applies once there are enough documents for it to mean something.
|
|
263
301
|
const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
|
|
302
|
+
let queryMass = 0;
|
|
303
|
+
for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
|
|
264
304
|
|
|
265
305
|
const scored = [];
|
|
266
306
|
let candidateCount = 0;
|
|
@@ -280,6 +320,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
280
320
|
candidateCount += 1;
|
|
281
321
|
if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
|
|
282
322
|
if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
|
|
323
|
+
// An episode is a long raw prompt: it holds many common words, so two of them
|
|
324
|
+
// clear the absolute floor above while the query's real subject is absent.
|
|
325
|
+
// It must also cover a fraction of the query's own IDF mass.
|
|
326
|
+
if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
|
|
283
327
|
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
284
328
|
}
|
|
285
329
|
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|
|
@@ -289,8 +333,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
|
|
|
289
333
|
const top = scored.length > 0 ? scored[0].score : 0;
|
|
290
334
|
// Temporal lane (3.5.3): validity is one more fusion lane over the docs that
|
|
291
335
|
// cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
|
|
292
|
-
const ranked =
|
|
293
|
-
.
|
|
336
|
+
const ranked = collapseTwins(
|
|
337
|
+
demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc),
|
|
338
|
+
twins,
|
|
339
|
+
).slice(0, limit);
|
|
294
340
|
return {
|
|
295
341
|
ranked,
|
|
296
342
|
candidateCount,
|
|
@@ -312,6 +358,7 @@ export function normalizeMemorySources(config) {
|
|
|
312
358
|
const rel = entry.trim().replace(/\\/g, '/');
|
|
313
359
|
if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
|
|
314
360
|
if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
|
|
361
|
+
if (rel.includes('*') && !DIR_GLOB_RE.test(rel)) continue;
|
|
315
362
|
seen.add(rel);
|
|
316
363
|
out.push(rel);
|
|
317
364
|
if (out.length >= MAX_SOURCES) break;
|
|
@@ -319,6 +366,44 @@ export function normalizeMemorySources(config) {
|
|
|
319
366
|
return out;
|
|
320
367
|
}
|
|
321
368
|
|
|
369
|
+
// matchesMemorySource(sources, rel) -> boolean. Sync, no I/O: is `rel` one of the
|
|
370
|
+
// (possibly `dir/*.md`) entries? Lets a hook skip the directory listing for edits
|
|
371
|
+
// that are nowhere near a memory source.
|
|
372
|
+
export function matchesMemorySource(sources, rel) {
|
|
373
|
+
for (const entry of sources) {
|
|
374
|
+
if (entry === rel) return true;
|
|
375
|
+
if (entry.endsWith('/*.md') && DIR_GLOB_RE.test(entry)) {
|
|
376
|
+
const dir = entry.slice(0, -4);
|
|
377
|
+
if (rel.startsWith(dir) && !rel.slice(dir.length).includes('/') && /\.md$/i.test(rel)) return true;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
return false;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// expandMemorySources(projectRoot, sources) -> concrete rel .md paths. `dir/*.md`
|
|
384
|
+
// lists that one directory level (async, sorted, at most MAX_DIR_FILES, dot-files
|
|
385
|
+
// skipped); a missing directory yields nothing. Explicit entries keep their order.
|
|
386
|
+
export async function expandMemorySources(projectRoot, sources) {
|
|
387
|
+
const out = [];
|
|
388
|
+
const seen = new Set();
|
|
389
|
+
const push = (rel) => { if (!seen.has(rel)) { seen.add(rel); out.push(rel); } };
|
|
390
|
+
for (const entry of sources) {
|
|
391
|
+
if (!entry.includes('*')) { push(entry); continue; }
|
|
392
|
+
if (!DIR_GLOB_RE.test(entry)) continue;
|
|
393
|
+
const dir = entry.slice(0, -4);
|
|
394
|
+
let names = [];
|
|
395
|
+
try {
|
|
396
|
+
const items = await fsp.readdir(path.resolve(projectRoot, dir), { withFileTypes: true });
|
|
397
|
+
names = items.filter((d) => (d.isFile() || d.isSymbolicLink()) && /\.md$/i.test(d.name) && !d.name.startsWith('.')).map((d) => d.name);
|
|
398
|
+
} catch {
|
|
399
|
+
names = [];
|
|
400
|
+
}
|
|
401
|
+
names.sort();
|
|
402
|
+
for (const name of names.slice(0, MAX_DIR_FILES)) push(`${dir}${name}`);
|
|
403
|
+
}
|
|
404
|
+
return out;
|
|
405
|
+
}
|
|
406
|
+
|
|
322
407
|
function cleanMarkdown(text) {
|
|
323
408
|
return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
|
|
324
409
|
}
|
|
@@ -328,6 +413,22 @@ function cleanMarkdown(text) {
|
|
|
328
413
|
// indexing it too would return the same rule twice (record + bullet).
|
|
329
414
|
const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
|
|
330
415
|
const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
|
|
416
|
+
const MD_ID_TAG_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
|
|
417
|
+
const MD_SUPERSEDES_TAG_RE = /\[supersedes:\s*([a-z0-9][a-z0-9._,\s-]*)\]/gi;
|
|
418
|
+
|
|
419
|
+
// Resolve slug tags to doc ids across every loaded source: the replacing doc
|
|
420
|
+
// gets `supersedes: [md:<rel>:<line>]` so dropSuperseded() removes the old one.
|
|
421
|
+
// Returns copies only for docs that carry tags; cached docs are never mutated.
|
|
422
|
+
function resolveMarkdownSupersession(docs) {
|
|
423
|
+
const bySlug = new Map();
|
|
424
|
+
for (const doc of docs) if (doc.slug && !bySlug.has(doc.slug)) bySlug.set(doc.slug, doc.id);
|
|
425
|
+
if (bySlug.size === 0) return docs;
|
|
426
|
+
return docs.map((doc) => {
|
|
427
|
+
if (!doc.supersedesSlugs) return doc;
|
|
428
|
+
const ids = doc.supersedesSlugs.map((s) => bySlug.get(s)).filter((id) => id && id !== doc.id);
|
|
429
|
+
return ids.length > 0 ? { ...doc, supersedes: ids } : doc;
|
|
430
|
+
});
|
|
431
|
+
}
|
|
331
432
|
|
|
332
433
|
// Top-level bullets ("- " / "* ") with their indented continuation lines.
|
|
333
434
|
export function parseMarkdownBullets(content, relPath) {
|
|
@@ -339,9 +440,21 @@ export function parseMarkdownBullets(content, relPath) {
|
|
|
339
440
|
let current = null;
|
|
340
441
|
const flush = () => {
|
|
341
442
|
if (!current) return;
|
|
342
|
-
|
|
443
|
+
// Supersession tags (3.5.4): `[id: slug]` names a bullet, `[supersedes: a, b]`
|
|
444
|
+
// says it replaces the bullets so named. Tags are stripped from the text.
|
|
445
|
+
let slug = null;
|
|
446
|
+
const replacesSlugs = [];
|
|
447
|
+
const raw = current.parts.join(' ')
|
|
448
|
+
.replace(MD_ID_TAG_RE, (_m, s) => { slug ??= s.toLowerCase(); return ' '; })
|
|
449
|
+
.replace(MD_SUPERSEDES_TAG_RE, (_m, list) => {
|
|
450
|
+
for (const s of list.split(/[\s,]+/)) if (s) replacesSlugs.push(s.toLowerCase());
|
|
451
|
+
return ' ';
|
|
452
|
+
});
|
|
453
|
+
const text = cleanMarkdown(raw);
|
|
343
454
|
if (text.length >= MIN_BULLET_CHARS) {
|
|
344
455
|
docs.push({
|
|
456
|
+
...(slug ? { slug } : {}),
|
|
457
|
+
...(replacesSlugs.length > 0 ? { supersedesSlugs: replacesSlugs } : {}),
|
|
345
458
|
id: `md:${relPath}:${current.line}`,
|
|
346
459
|
type: 'markdown',
|
|
347
460
|
source: 'markdown',
|
|
@@ -398,7 +511,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
|
|
|
398
511
|
const root = path.resolve(projectRoot);
|
|
399
512
|
const docs = [];
|
|
400
513
|
let realRoot = null;
|
|
401
|
-
for (const rel of sources) {
|
|
514
|
+
for (const rel of await expandMemorySources(root, sources)) {
|
|
402
515
|
const abs = path.resolve(root, rel);
|
|
403
516
|
if (abs !== root && !abs.startsWith(root + path.sep)) continue;
|
|
404
517
|
try {
|
|
@@ -420,7 +533,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
|
|
|
420
533
|
markdownCache.delete(abs);
|
|
421
534
|
}
|
|
422
535
|
}
|
|
423
|
-
return docs;
|
|
536
|
+
return resolveMarkdownSupersession(docs);
|
|
424
537
|
}
|
|
425
538
|
|
|
426
539
|
const SNIPPET_CHARS = 160;
|
|
@@ -8,6 +8,7 @@ export const RECORD_TYPES = Object.freeze([
|
|
|
8
8
|
'derived_fact',
|
|
9
9
|
'episode',
|
|
10
10
|
'procedure',
|
|
11
|
+
'observation',
|
|
11
12
|
]);
|
|
12
13
|
|
|
13
14
|
export const RECORD_SCOPES = Object.freeze([
|
|
@@ -66,6 +67,27 @@ function newId() {
|
|
|
66
67
|
return `mem_${crypto.randomBytes(6).toString('hex')}`;
|
|
67
68
|
}
|
|
68
69
|
|
|
70
|
+
// 3.5.4: an observation is a consolidated note backed by >= 2 distinct episodes.
|
|
71
|
+
// Structural check only (no store lookup); existence is validateObservation's job.
|
|
72
|
+
export const MIN_OBSERVATION_EVIDENCE = 2;
|
|
73
|
+
export function observationEvidenceErrors(record) {
|
|
74
|
+
const ev = record?.meta?.evidence;
|
|
75
|
+
if (!ev || typeof ev !== 'object' || Array.isArray(ev) || !Array.isArray(ev.episodes)) {
|
|
76
|
+
return ['observation requires meta.evidence = { episodes: [ids], count }'];
|
|
77
|
+
}
|
|
78
|
+
const errors = [];
|
|
79
|
+
const ids = ev.episodes;
|
|
80
|
+
if (ids.some((id) => typeof id !== 'string' || !RECORD_ID_RE.test(id))) {
|
|
81
|
+
errors.push('observation evidence episodes must be record ids');
|
|
82
|
+
}
|
|
83
|
+
const distinct = new Set(ids).size;
|
|
84
|
+
if (distinct < MIN_OBSERVATION_EVIDENCE) {
|
|
85
|
+
errors.push(`observation needs at least ${MIN_OBSERVATION_EVIDENCE} distinct evidence episodes`);
|
|
86
|
+
}
|
|
87
|
+
if (ev.count !== distinct) errors.push('observation evidence count must equal the distinct episode count');
|
|
88
|
+
return errors;
|
|
89
|
+
}
|
|
90
|
+
|
|
69
91
|
/**
|
|
70
92
|
* validateRecord(record) → { ok: boolean, errors: string[] }
|
|
71
93
|
*/
|
|
@@ -147,6 +169,7 @@ export function validateRecord(record) {
|
|
|
147
169
|
}
|
|
148
170
|
}
|
|
149
171
|
}
|
|
172
|
+
if (record.type === 'observation') errors.push(...observationEvidenceErrors(record));
|
|
150
173
|
if (record.valid_from != null
|
|
151
174
|
&& (typeof record.valid_from !== 'number' || !Number.isFinite(record.valid_from))) {
|
|
152
175
|
errors.push('valid_from must be epoch ms | null');
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
// writeGate.js — deterministic admission gate for record writes (3.5.4).
|
|
2
|
+
// No LLM. Called by mutateMemory on `add` / `update` after the schema build and
|
|
3
|
+
// the secret guard; returns a typed rejection with a one-line reason.
|
|
4
|
+
//
|
|
5
|
+
// code reason prefix rule
|
|
6
|
+
// too-large too-large: record text over RECORD_TEXT_MAX chars
|
|
7
|
+
// no-evidence no-evidence: no source evidence (see hasSourceEvidence)
|
|
8
|
+
// near-duplicate near-duplicate: lexical near-duplicate of a live record
|
|
9
|
+
//
|
|
10
|
+
// (schema -> `invalid-record`, secret -> `secret-detected` stay where they were
|
|
11
|
+
// in mutateMemory; the codes above are the gate's own.) Stored items are never
|
|
12
|
+
// rejected retroactively: the gate only looks at the incoming write.
|
|
13
|
+
//
|
|
14
|
+
// Source evidence (project lane), any one of:
|
|
15
|
+
// - evidence[] entry kind 'file' whose locator is an existing path inside the project
|
|
16
|
+
// - evidence[] entry kind 'ledger' whose locator is an existing episode id (store or archive)
|
|
17
|
+
// - evidence[] entry kind 'command' whose locator names a commit: `commit:<sha>` or a bare 7-40 hex sha
|
|
18
|
+
// - an observation with meta.evidence.episodes that validateObservationEvidence accepts
|
|
19
|
+
// kinds 'manual' and 'import' are claims, not evidence, and do not count.
|
|
20
|
+
// Machine-generated writers (GENERATED_CREATORS) skip evidence and duplicate
|
|
21
|
+
// checks (their text is templated from a ledger); size still applies.
|
|
22
|
+
import fsp from 'node:fs/promises';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
|
|
25
|
+
import { GENERATED_CREATORS, RECORD_TEXT_MAX, findNearDuplicate } from './memoryCheck.js';
|
|
26
|
+
import { tokenize } from './recallEngine.js';
|
|
27
|
+
import { guardWritePayload } from './writeGuard.js';
|
|
28
|
+
import { loadEpisodeIdsFor, validateObservationEvidence } from './observations.js';
|
|
29
|
+
|
|
30
|
+
const COMMIT_RE = /^(?:commit:)?[0-9a-f]{7,40}$/i;
|
|
31
|
+
|
|
32
|
+
async function fileExists(projectRoot, locator) {
|
|
33
|
+
if (typeof projectRoot !== 'string' || typeof locator !== 'string' || !locator) return false;
|
|
34
|
+
const root = path.resolve(projectRoot);
|
|
35
|
+
const abs = path.resolve(root, locator);
|
|
36
|
+
if (abs !== root && !abs.startsWith(root + path.sep)) return false;
|
|
37
|
+
try {
|
|
38
|
+
await fsp.stat(abs);
|
|
39
|
+
return true;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** hasSourceEvidence(record, {projectRoot}) -> Promise<{ok, detail?}> */
|
|
46
|
+
export async function hasSourceEvidence(record, { projectRoot } = {}) {
|
|
47
|
+
if (record?.type === 'observation') {
|
|
48
|
+
const res = await validateObservationEvidence(record, { projectRoot });
|
|
49
|
+
return res.ok ? { ok: true } : { ok: false, detail: res.reason };
|
|
50
|
+
}
|
|
51
|
+
let episodeIds = null;
|
|
52
|
+
for (const ev of Array.isArray(record?.evidence) ? record.evidence : []) {
|
|
53
|
+
if (!ev || typeof ev.locator !== 'string') continue;
|
|
54
|
+
if (ev.kind === 'file' && await fileExists(projectRoot, ev.locator)) return { ok: true };
|
|
55
|
+
if (ev.kind === 'command' && COMMIT_RE.test(ev.locator)) return { ok: true };
|
|
56
|
+
if (ev.kind === 'ledger') {
|
|
57
|
+
episodeIds ??= await loadEpisodeIdsFor(projectRoot).catch(() => new Set());
|
|
58
|
+
if (episodeIds.has(ev.locator)) return { ok: true };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return { ok: false, detail: 'add evidence: a file path that exists, a commit sha, or an episode id' };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* checkProposalText({ text, existing, config, projectId }) -> { ok: true } | { ok, code, reason, labels? }
|
|
66
|
+
* Text-only gate for `ukit memory propose` (a human approves candidates, so no
|
|
67
|
+
* evidence rule): size cap, secret scan, near-duplicate of `existing` ([{id,text}],
|
|
68
|
+
* pending + approved candidates). Honours memory.writeGate 'off'.
|
|
69
|
+
*/
|
|
70
|
+
export function checkProposalText({ text, existing = [], config = null, projectId = null } = {}) {
|
|
71
|
+
const mode = config?.memory?.writeGate;
|
|
72
|
+
if (mode === false || mode === 'off') return { ok: true };
|
|
73
|
+
const body = String(text ?? '');
|
|
74
|
+
if (body.length > RECORD_TEXT_MAX) {
|
|
75
|
+
return {
|
|
76
|
+
ok: false, code: 'too-large',
|
|
77
|
+
reason: `too-large: text is ${body.length} chars (max ${RECORD_TEXT_MAX}); move the detail to docs/memory/ and keep one line here`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
const guarded = guardWritePayload({ text: body }, { config, projectId });
|
|
81
|
+
if (guarded?.ok === false) {
|
|
82
|
+
return { ok: false, code: 'secret-detected', reason: guarded.reason ?? 'secret-detected', labels: guarded.labels };
|
|
83
|
+
}
|
|
84
|
+
const dup = findNearDuplicate(body, existing, tokenize);
|
|
85
|
+
if (dup) {
|
|
86
|
+
return {
|
|
87
|
+
ok: false, code: 'near-duplicate',
|
|
88
|
+
reason: `near-duplicate: ${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; approve or reject that candidate instead`,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
return { ok: true };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const isLive = (r) => r && r.status !== 'archived' && !r.superseded_by;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* checkRecordWrite({ record, records, projectRoot, op, replaces }) ->
|
|
98
|
+
* { ok: true } | { ok: false, code, reason }
|
|
99
|
+
* record: the built (validated) record or, for update, the would-be merged record.
|
|
100
|
+
* records: current store contents. replaces: ids this write supersedes or updates
|
|
101
|
+
* (excluded from the duplicate comparison). op 'update' only enforces the rules
|
|
102
|
+
* the patch can break (size, duplicate of changed text, evidence it supplies).
|
|
103
|
+
*/
|
|
104
|
+
export async function checkRecordWrite({
|
|
105
|
+
record, records = [], projectRoot = null, op = 'add', replaces = [], patch = null, strict = false,
|
|
106
|
+
} = {}) {
|
|
107
|
+
const text = String(record?.text ?? '');
|
|
108
|
+
const touchesText = op === 'add' || (patch && 'text' in patch);
|
|
109
|
+
if (touchesText && text.length > RECORD_TEXT_MAX) {
|
|
110
|
+
return {
|
|
111
|
+
ok: false, code: 'too-large',
|
|
112
|
+
reason: `too-large: text is ${text.length} chars (max ${RECORD_TEXT_MAX}); move the detail to docs/memory/ and keep one line here`,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
// Evidence and near-duplicate rules are for agent-authored notes (strict).
|
|
116
|
+
// Legacy callers that write bare records (tests, tools, restore flows) pass
|
|
117
|
+
// the size cap only; generated writers are exempt even when strict.
|
|
118
|
+
if (!strict || GENERATED_CREATORS.has(record?.created_by)) return { ok: true };
|
|
119
|
+
|
|
120
|
+
const touchesEvidence = op === 'add' || (patch && ('evidence' in patch || 'meta' in patch || patch.type === 'observation'));
|
|
121
|
+
if (touchesEvidence && projectRoot) {
|
|
122
|
+
const res = await hasSourceEvidence(record, { projectRoot });
|
|
123
|
+
if (!res.ok) return { ok: false, code: 'no-evidence', reason: `no-evidence: ${res.detail}` };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (touchesText) {
|
|
127
|
+
const skip = new Set([record.id, ...replaces, ...(Array.isArray(record.supersedes) ? record.supersedes : [])]);
|
|
128
|
+
const dup = findNearDuplicate(text, records.filter((r) => isLive(r) && !skip.has(r.id)), tokenize);
|
|
129
|
+
if (dup) {
|
|
130
|
+
return {
|
|
131
|
+
ok: false, code: 'near-duplicate',
|
|
132
|
+
reason: `near-duplicate: ${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; update it, or pass supersedes: ["${dup.id}"] to replace it`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return { ok: true };
|
|
137
|
+
}
|
|
@@ -531,7 +531,14 @@ export function buildDefaultRuntimeConfig(overrides = {}) {
|
|
|
531
531
|
maxArchivedSessions: 50,
|
|
532
532
|
maxPatternCandidates: 50,
|
|
533
533
|
redactSecrets: true,
|
|
534
|
-
|
|
534
|
+
// Capped core + every docs/memory/*.md detail file (3.5.4; one directory
|
|
535
|
+
// level, expanded by recallEngine.expandMemorySources). A missing directory
|
|
536
|
+
// contributes nothing, so a project with only docs/MEMORY.md behaves as before.
|
|
537
|
+
sources: ['docs/MEMORY.md', 'docs/memory/*.md'],
|
|
538
|
+
// Record write gate (3.5.4): 'strict' (default) = size cap, secrets, source
|
|
539
|
+
// evidence and near-duplicate checks on every non-generated record write;
|
|
540
|
+
// 'off' = no gate. scope user needs no file evidence. See writeGate.js.
|
|
541
|
+
writeGate: 'strict',
|
|
535
542
|
editHints: { enabled: true, mirrors: [], ignoreMirrors: [], coChange: false },
|
|
536
543
|
},
|
|
537
544
|
validation: {
|
|
@@ -1195,6 +1202,9 @@ export function validateRuntimeConfig(config) {
|
|
|
1195
1202
|
|| config.memory.sources.some((entry) => typeof entry !== 'string'))) {
|
|
1196
1203
|
errors.push('memory.sources must be an array of relative .md paths.');
|
|
1197
1204
|
}
|
|
1205
|
+
if (config.memory.writeGate !== undefined && !['strict', 'off', false].includes(config.memory.writeGate)) {
|
|
1206
|
+
errors.push('memory.writeGate must be "strict" or "off".');
|
|
1207
|
+
}
|
|
1198
1208
|
}
|
|
1199
1209
|
|
|
1200
1210
|
if (!isPlainObject(config.validation)) {
|
package/src/index/buildIndex.js
CHANGED
|
@@ -1682,6 +1682,8 @@ function computeSignature(content, lineStartIndex) {
|
|
|
1682
1682
|
return `${collapsed.slice(0, 199)}…`;
|
|
1683
1683
|
}
|
|
1684
1684
|
|
|
1685
|
+
const CLASS_METHOD_SKIP = new Set(['if', 'for', 'while', 'switch', 'catch', 'function', 'return', 'with', 'constructor', 'super']);
|
|
1686
|
+
|
|
1685
1687
|
export function extractSymbols(filePath, content) {
|
|
1686
1688
|
const symbols = [];
|
|
1687
1689
|
const lineStarts = computeLineStarts(content);
|
|
@@ -1750,6 +1752,30 @@ export function extractSymbols(filePath, content) {
|
|
|
1750
1752
|
addMatches(/defineComponent\(\s*\{[\s\S]*?name\s*:\s*['"]([^'"]+)['"]/g, 'component-name');
|
|
1751
1753
|
addMatches(/export\s+default\s*\{[\s\S]*?name\s*:\s*['"]([^'"]+)['"]/g, 'component-name');
|
|
1752
1754
|
|
|
1755
|
+
// Unexported declarations (3.5.4). Line-anchored, so `export function` / `export class`
|
|
1756
|
+
// (indexed above as exported) never double-count. Local types rank below exported ones.
|
|
1757
|
+
addMatches(/^[ \t]*(?:async\s+)?function\s*\*?\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*[(<]/gm, 'local-function');
|
|
1758
|
+
addMatches(/^[ \t]*(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*(?:async\s+)?function\b/gm, 'callable');
|
|
1759
|
+
addMatches(/^[ \t]*(?:abstract\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'local-class');
|
|
1760
|
+
if (/\.(?:ts|tsx|mts|cts|vue)$/.test(filePath)) {
|
|
1761
|
+
addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?(?:const\s+)?(?:interface|enum)\s+([A-Za-z_$][A-Za-z0-9_$]*)/gm, 'type');
|
|
1762
|
+
addMatches(/^[ \t]*(?:export\s+)?(?:declare\s+)?type\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*(?:<[^=\n]*>)?\s*=/gm, 'type');
|
|
1763
|
+
}
|
|
1764
|
+
// Class methods: indented `name(args) {` lines inside a balanced class body.
|
|
1765
|
+
for (const classMatch of content.matchAll(/(?:^|[^\w$.])class\s+[A-Za-z_$][A-Za-z0-9_$]*(?:\s+extends\s+[\w$.]+(?:\([^)]*\))?)?\s*\{/g)) {
|
|
1766
|
+
const bodyStart = classMatch.index + classMatch[0].length;
|
|
1767
|
+
const classBody = sliceBalancedBlock(content, bodyStart);
|
|
1768
|
+
if (!classBody) {
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
for (const methodMatch of classBody.matchAll(/^[ \t]+(?:(?:static|async|get|set)\s+)*\*?\s*(#?[A-Za-z_$][A-Za-z0-9_$]*)\s*\([^()]*\)\s*\{/gm)) {
|
|
1772
|
+
if (CLASS_METHOD_SKIP.has(methodMatch[1])) {
|
|
1773
|
+
continue;
|
|
1774
|
+
}
|
|
1775
|
+
addSymbol(methodMatch[1], 'method', bodyStart + methodMatch.index + methodMatch[0].indexOf(methodMatch[1]));
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1753
1779
|
// Global-registered frontend functions: reachable from anywhere without an
|
|
1754
1780
|
// import edge, so impact analysis must treat them as verified dependencies.
|
|
1755
1781
|
addMatches(/\b(?:window|globalThis|global)\s*\.\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*=/g, 'global');
|
package/src/index/paths.js
CHANGED
|
@@ -19,7 +19,9 @@ export const INDEX_ARTIFACTS = {
|
|
|
19
19
|
// SQL 'column' rows with parent/schema, and calls.json gains per-object SQL
|
|
20
20
|
// reference records — a v8 cached artifact must not be reused for unchanged
|
|
21
21
|
// files or the new extractions silently vanish.
|
|
22
|
-
|
|
22
|
+
// v10: symbols.json gains unexported function/class ('local-function', 'local-class'),
|
|
23
|
+
// function-expression consts, class 'method' rows and TS interface/enum/type ('type') rows.
|
|
24
|
+
export const INDEX_SCHEMA_VERSION = 10;
|
|
23
25
|
|
|
24
26
|
export function getIndexDir(rootDir) {
|
|
25
27
|
return path.join(rootDir, '.cache', 'index');
|