@ngockhoale/ukit 3.5.3 → 3.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CHANGELOG.md +36 -0
  2. package/manifests/documentation.yaml +12 -1
  3. package/manifests/platform.full.yaml +12 -0
  4. package/package.json +29 -2
  5. package/src/cli/commands/doctor.js +41 -0
  6. package/src/cli/commands/indexTools.js +10 -4
  7. package/src/cli/commands/memory.js +88 -6
  8. package/src/core/memory/episodes.js +106 -7
  9. package/src/core/memory/memoryCheck.js +241 -0
  10. package/src/core/memory/memoryCheckRun.js +39 -0
  11. package/src/core/memory/memoryHit.js +31 -15
  12. package/src/core/memory/mutateMemory.js +37 -1
  13. package/src/core/memory/observations.js +192 -0
  14. package/src/core/memory/recallEngine.js +159 -10
  15. package/src/core/memory/recallLanes.js +182 -0
  16. package/src/core/memory/records.js +23 -0
  17. package/src/core/memory/retrieval.js +13 -1
  18. package/src/core/memory/topicCommand.js +75 -0
  19. package/src/core/memory/topicPages.js +545 -0
  20. package/src/core/memory/writeGate.js +137 -0
  21. package/src/core/runtimeConfig.js +56 -1
  22. package/src/decision/registry.js +17 -0
  23. package/src/index/buildIndex.js +312 -5
  24. package/src/index/languageTools.js +100 -0
  25. package/src/index/paths.js +4 -1
  26. package/src/index/queryIndex.js +356 -14
  27. package/src/index/resolveContext.js +1 -1
  28. package/template_project/.claude/hooks/edit-hint.mjs +64 -17
  29. package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
  30. package/template_project/.claude/hooks/session-episode.sh +9 -4
  31. package/template_project/.claude/settings.json +2 -2
  32. package/template_project/.claude/ukit/index/lib/index-core.mjs +769 -18
  33. package/template_project/.claude/ukit/index/query-index.mjs +14 -3
  34. package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
  35. package/template_project/.claude/ukit/index/route-task.mjs +4 -1
  36. package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
  37. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
  38. package/template_project/.claude/ukit/runtime/memory-recall.mjs +159 -10
  39. package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
  40. package/template_project/.claude/ukit/runtime/memory-validity.mjs +28 -1
  41. package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
  42. package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
  43. package/template_project/ukit/storage/config.json +21 -2
  44. package/manifests/instructionRules.yaml +0 -442
  45. package/src/core/agentRuntime/completionGate.js +0 -179
  46. package/src/core/agentRuntime/context.js +0 -571
  47. package/src/core/agentRuntime/qualityComparison.js +0 -112
  48. package/src/core/agentRuntime/resourcePolicy.js +0 -78
  49. package/src/core/agentRuntime/runtimeSupport.js +0 -255
  50. package/src/core/codeintel/manifest.js +0 -176
  51. package/src/core/experiments/deliberation.js +0 -327
  52. package/src/core/experiments/dynamicWorkflow.js +0 -507
  53. package/src/core/handoffDocValidator.js +0 -154
  54. package/src/core/memory/deltaOverlays.js +0 -493
  55. package/src/core/memory/learningCandidates.js +0 -347
  56. package/src/core/observability/emit/sessionBoot.js +0 -407
  57. package/src/core/observability/evaluation/replay.js +0 -149
  58. package/src/core/observability/evaluation/scorecard.js +0 -465
  59. package/src/core/observability/schema/compatibility.js +0 -180
  60. package/src/core/observability/support/renderer.js +0 -130
  61. package/src/core/reviewPanelAggregate.js +0 -204
  62. package/src/core/taskBudgetValidator.js +0 -251
  63. package/src/core/taskProgressGuard.js +0 -227
  64. package/src/decision/reviewVerdict.js +0 -339
  65. package/src/decision/runtimeFilter.js +0 -150
  66. package/src/decision/runtimeScheduler.js +0 -239
  67. package/src/index/contextPack.js +0 -285
  68. package/src/index/impactContext.js +0 -360
  69. package/src/index/reviewPolicy.js +0 -368
  70. package/src/index/sensitivePath.js +0 -21
@@ -23,7 +23,12 @@ import fsp from 'node:fs/promises';
23
23
  import path from 'node:path';
24
24
 
25
25
  export const RECALL_ENGINE_VERSION = 'bm25-1';
26
- export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md']);
26
+ // Capped core + every detail file of docs/memory/ (3.5.4). An entry ending in
27
+ // `/*.md` names one directory level, expanded by expandMemorySources; a missing
28
+ // file or directory contributes nothing.
29
+ export const DEFAULT_MEMORY_SOURCES = Object.freeze(['docs/MEMORY.md', 'docs/memory/*.md']);
30
+ const DIR_GLOB_RE = /^[^*?[\]{}]+\/\*\.md$/;
31
+ const MAX_DIR_FILES = 32;
27
32
 
28
33
  const K1 = 1.2;
29
34
  const B = 0.75;
@@ -37,6 +42,8 @@ const isCuratedDoc = (doc) => doc?.type !== 'markdown' && !isTemplateText(doc);
37
42
  // ... and the IDF mass of the matched tokens reaches this fraction of the
38
43
  // largest IDF the corpus can produce (a token present in exactly one doc).
39
44
  const MIN_IDF_MASS_RATIO = 0.5;
45
+ // Episodes only: matched IDF mass must reach this fraction of the whole query's.
46
+ const EPISODE_MIN_COVERAGE = 0.5;
40
47
  // Results scoring below this fraction of the best hit are noise tail.
41
48
  const RELATIVE_CUTOFF = 0.35;
42
49
  // Query-dependent type prior: multiplies the lexical score, so it can only
@@ -46,6 +53,37 @@ const RELATIVE_CUTOFF = 0.35;
46
53
  const TYPE_PRIOR = Object.freeze({
47
54
  project_rule: 1.2, procedure: 1.1, derived_fact: 1.1, markdown: 1.1,
48
55
  });
56
+ // Core/detail twins (3.5.4): a capped-core one-liner ends in "Detail: <file>" and
57
+ // its verbatim note sits in that file. Both score on the same words, so one fact
58
+ // would spend two result slots. collapseTwins keeps one of each pair that is
59
+ // present in the same ranked list: `keep` 'detail' (default: the full note) or 'core'.
60
+ const DETAIL_POINTER_RE = /\bDetail:\s+(\S+?\.md)\b/;
61
+ const TWIN_MIN_OVERLAP = 0.5;
62
+ export function collapseTwins(entries, keep = 'detail') {
63
+ if (keep === 'off' || !Array.isArray(entries) || entries.length < 2) return entries;
64
+ const drop = new Set();
65
+ for (const core of entries) {
66
+ const doc = core.doc;
67
+ if (doc?.type !== 'markdown' || drop.has(core)) continue;
68
+ const m = DETAIL_POINTER_RE.exec(doc.text ?? '');
69
+ if (!m) continue;
70
+ const coreTokens = new Set(tokenize(doc.text));
71
+ if (coreTokens.size === 0) continue;
72
+ let best = null;
73
+ let bestOverlap = 0;
74
+ for (const other of entries) {
75
+ if (other === core || drop.has(other) || other.doc?.type !== 'markdown') continue;
76
+ if (!String(other.doc.provenance ?? '').startsWith(`${m[1]}:`)) continue;
77
+ const otherTokens = new Set(tokenize(other.doc.text));
78
+ let shared = 0;
79
+ for (const t of coreTokens) if (otherTokens.has(t)) shared += 1;
80
+ const overlap = shared / coreTokens.size;
81
+ if (overlap > bestOverlap) { bestOverlap = overlap; best = other; }
82
+ }
83
+ if (best && bestOverlap >= TWIN_MIN_OVERLAP) drop.add(keep === 'core' ? best : core);
84
+ }
85
+ return drop.size === 0 ? entries : entries.filter((e) => !drop.has(e));
86
+ }
49
87
  const MAX_SOURCES = 8;
50
88
  const MAX_SOURCE_BYTES = 512 * 1024;
51
89
  const MIN_BULLET_CHARS = 12;
@@ -117,9 +155,17 @@ function stem(token) {
117
155
 
118
156
  // Tokens of `text`: whole tokens plus their separator-split parts
119
157
  // (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
120
- export function tokenize(text) {
158
+ // A camelCase identifier also yields its words (`deriveDeterministicBucket`
159
+ // -> derive, deterministic, bucket); the whole identifier stays a token too.
160
+ const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
161
+ const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
162
+
163
+ // Query side only (`{ camel: true }`): in a long note the split words of its many
164
+ // identifiers (`postEditReviewPolicy`) would add common-word matches.
165
+ export function tokenize(text, { camel = false } = {}) {
121
166
  const out = [];
122
- for (const raw of normalize(text).split(/\s+/)) {
167
+ const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
168
+ for (const raw of normalize(source).split(/\s+/)) {
123
169
  if (!raw) continue;
124
170
  const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
125
171
  const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
@@ -131,6 +177,21 @@ export function tokenize(text) {
131
177
  return out;
132
178
  }
133
179
 
180
+ // File names in a query (`name.ext`): a query that names a file is only
181
+ // answered by a note that names it too and covers the rest of the question.
182
+ const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
183
+ const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
184
+ const FILE_REST_COVERAGE = 0.3;
185
+ const FILE_REST_MIN_TOKENS = 2;
186
+ function queryFileSpec(query) {
187
+ if (typeof query !== 'string') return null;
188
+ const files = query.match(QUERY_FILE_RE);
189
+ if (!files) return null;
190
+ const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
191
+ const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
192
+ return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
193
+ }
194
+
134
195
  function episodeBody(text) {
135
196
  return String(text ?? '')
136
197
  .replace(/^\s*Session\s+\S+?:\s*/i, '')
@@ -233,8 +294,8 @@ function idf(totalDocs, docFreq) {
233
294
  // abstained }
234
295
  // `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
235
296
  // `query` is a string or a pre-tokenized array. Never throws.
236
- export function rankDocs(docs, query, { limit = 10 } = {}) {
237
- const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
297
+ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
298
+ const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
238
299
  const empty = {
239
300
  ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
240
301
  };
@@ -261,6 +322,12 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
261
322
  // IDF cannot discriminate in a tiny corpus, so the multi-token requirement
262
323
  // only applies once there are enough documents for it to mean something.
263
324
  const wantMatched = total < SMALL_CORPUS_DOCS ? 1 : Math.min(MIN_MATCHED_TOKENS, queryTokens.length);
325
+ let queryMass = 0;
326
+ for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
327
+
328
+ const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
329
+ let restMass = 0;
330
+ if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
264
331
 
265
332
  const scored = [];
266
333
  let candidateCount = 0;
@@ -280,6 +347,19 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
280
347
  candidateCount += 1;
281
348
  if (total >= SMALL_CORPUS_DOCS && idfMass < MIN_IDF_MASS_RATIO * idfMax) continue;
282
349
  if (total >= SMALL_CORPUS_DOCS && matched < wantMatched && !isCuratedDoc(doc)) continue;
350
+ // An episode is a long raw prompt: it holds many common words, so two of them
351
+ // clear the absolute floor above while the query's real subject is absent.
352
+ // It must also cover a fraction of the query's own IDF mass.
353
+ if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
354
+ if (fileSpec) {
355
+ let named = false;
356
+ for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
357
+ let restHit = 0;
358
+ for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
359
+ // One leftover word ("what does X check") is too little to judge coverage by.
360
+ const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
361
+ if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
362
+ }
283
363
  scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
284
364
  }
285
365
  // Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
@@ -289,8 +369,10 @@ export function rankDocs(docs, query, { limit = 10 } = {}) {
289
369
  const top = scored.length > 0 ? scored[0].score : 0;
290
370
  // Temporal lane (3.5.3): validity is one more fusion lane over the docs that
291
371
  // cleared the floor and cutoff (see demoteByValidity); nothing is dropped.
292
- const ranked = demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc)
293
- .slice(0, limit);
372
+ const ranked = collapseTwins(
373
+ demoteByValidity(scored.filter((entry) => entry.score >= RELATIVE_CUTOFF * top), (entry) => entry.doc),
374
+ twins,
375
+ ).slice(0, limit);
294
376
  return {
295
377
  ranked,
296
378
  candidateCount,
@@ -312,6 +394,7 @@ export function normalizeMemorySources(config) {
312
394
  const rel = entry.trim().replace(/\\/g, '/');
313
395
  if (!rel || path.isAbsolute(rel) || rel.split('/').includes('..')) continue;
314
396
  if (!/\.md$/i.test(rel) || seen.has(rel)) continue;
397
+ if (rel.includes('*') && !DIR_GLOB_RE.test(rel)) continue;
315
398
  seen.add(rel);
316
399
  out.push(rel);
317
400
  if (out.length >= MAX_SOURCES) break;
@@ -319,6 +402,44 @@ export function normalizeMemorySources(config) {
319
402
  return out;
320
403
  }
321
404
 
405
+ // matchesMemorySource(sources, rel) -> boolean. Sync, no I/O: is `rel` one of the
406
+ // (possibly `dir/*.md`) entries? Lets a hook skip the directory listing for edits
407
+ // that are nowhere near a memory source.
408
+ export function matchesMemorySource(sources, rel) {
409
+ for (const entry of sources) {
410
+ if (entry === rel) return true;
411
+ if (entry.endsWith('/*.md') && DIR_GLOB_RE.test(entry)) {
412
+ const dir = entry.slice(0, -4);
413
+ if (rel.startsWith(dir) && !rel.slice(dir.length).includes('/') && /\.md$/i.test(rel)) return true;
414
+ }
415
+ }
416
+ return false;
417
+ }
418
+
419
+ // expandMemorySources(projectRoot, sources) -> concrete rel .md paths. `dir/*.md`
420
+ // lists that one directory level (async, sorted, at most MAX_DIR_FILES, dot-files
421
+ // skipped); a missing directory yields nothing. Explicit entries keep their order.
422
+ export async function expandMemorySources(projectRoot, sources) {
423
+ const out = [];
424
+ const seen = new Set();
425
+ const push = (rel) => { if (!seen.has(rel)) { seen.add(rel); out.push(rel); } };
426
+ for (const entry of sources) {
427
+ if (!entry.includes('*')) { push(entry); continue; }
428
+ if (!DIR_GLOB_RE.test(entry)) continue;
429
+ const dir = entry.slice(0, -4);
430
+ let names = [];
431
+ try {
432
+ const items = await fsp.readdir(path.resolve(projectRoot, dir), { withFileTypes: true });
433
+ names = items.filter((d) => (d.isFile() || d.isSymbolicLink()) && /\.md$/i.test(d.name) && !d.name.startsWith('.')).map((d) => d.name);
434
+ } catch {
435
+ names = [];
436
+ }
437
+ names.sort();
438
+ for (const name of names.slice(0, MAX_DIR_FILES)) push(`${dir}${name}`);
439
+ }
440
+ return out;
441
+ }
442
+
322
443
  function cleanMarkdown(text) {
323
444
  return text.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\*\*|__/g, '').replace(/`/g, '').replace(/\s+/g, ' ').trim();
324
445
  }
@@ -328,6 +449,22 @@ function cleanMarkdown(text) {
328
449
  // indexing it too would return the same rule twice (record + bullet).
329
450
  const LEARNED_START_RE = /^\s*<!--\s*ukit-learned:start\s*-->/;
330
451
  const LEARNED_END_RE = /^\s*<!--\s*ukit-learned:end\s*-->/;
452
+ const MD_ID_TAG_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
453
+ const MD_SUPERSEDES_TAG_RE = /\[supersedes:\s*([a-z0-9][a-z0-9._,\s-]*)\]/gi;
454
+
455
+ // Resolve slug tags to doc ids across every loaded source: the replacing doc
456
+ // gets `supersedes: [md:<rel>:<line>]` so dropSuperseded() removes the old one.
457
+ // Returns copies only for docs that carry tags; cached docs are never mutated.
458
+ function resolveMarkdownSupersession(docs) {
459
+ const bySlug = new Map();
460
+ for (const doc of docs) if (doc.slug && !bySlug.has(doc.slug)) bySlug.set(doc.slug, doc.id);
461
+ if (bySlug.size === 0) return docs;
462
+ return docs.map((doc) => {
463
+ if (!doc.supersedesSlugs) return doc;
464
+ const ids = doc.supersedesSlugs.map((s) => bySlug.get(s)).filter((id) => id && id !== doc.id);
465
+ return ids.length > 0 ? { ...doc, supersedes: ids } : doc;
466
+ });
467
+ }
331
468
 
332
469
  // Top-level bullets ("- " / "* ") with their indented continuation lines.
333
470
  export function parseMarkdownBullets(content, relPath) {
@@ -339,9 +476,21 @@ export function parseMarkdownBullets(content, relPath) {
339
476
  let current = null;
340
477
  const flush = () => {
341
478
  if (!current) return;
342
- const text = cleanMarkdown(current.parts.join(' '));
479
+ // Supersession tags (3.5.4): `[id: slug]` names a bullet, `[supersedes: a, b]`
480
+ // says it replaces the bullets so named. Tags are stripped from the text.
481
+ let slug = null;
482
+ const replacesSlugs = [];
483
+ const raw = current.parts.join(' ')
484
+ .replace(MD_ID_TAG_RE, (_m, s) => { slug ??= s.toLowerCase(); return ' '; })
485
+ .replace(MD_SUPERSEDES_TAG_RE, (_m, list) => {
486
+ for (const s of list.split(/[\s,]+/)) if (s) replacesSlugs.push(s.toLowerCase());
487
+ return ' ';
488
+ });
489
+ const text = cleanMarkdown(raw);
343
490
  if (text.length >= MIN_BULLET_CHARS) {
344
491
  docs.push({
492
+ ...(slug ? { slug } : {}),
493
+ ...(replacesSlugs.length > 0 ? { supersedesSlugs: replacesSlugs } : {}),
345
494
  id: `md:${relPath}:${current.line}`,
346
495
  type: 'markdown',
347
496
  source: 'markdown',
@@ -398,7 +547,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
398
547
  const root = path.resolve(projectRoot);
399
548
  const docs = [];
400
549
  let realRoot = null;
401
- for (const rel of sources) {
550
+ for (const rel of await expandMemorySources(root, sources)) {
402
551
  const abs = path.resolve(root, rel);
403
552
  if (abs !== root && !abs.startsWith(root + path.sep)) continue;
404
553
  try {
@@ -420,7 +569,7 @@ export async function loadMarkdownDocs(projectRoot, sources = DEFAULT_MEMORY_SOU
420
569
  markdownCache.delete(abs);
421
570
  }
422
571
  }
423
- return docs;
572
+ return resolveMarkdownSupersession(docs);
424
573
  }
425
574
 
426
575
  const SNIPPET_CHARS = 160;
@@ -0,0 +1,182 @@
1
+ // Optional recall lanes (3.5.5), CLI path only, both off unless enabled.
2
+ //
3
+ // vector - embedding cosine over the eligible docs, fused with the BM25
4
+ // order by RRF (recallEngine.fuseRanked). Provider = the
5
+ // codeIntel.embedding seam (hashed by default, `auto` takes a loader).
6
+ // rerank - the top BM25 candidates are put to `unic-decision` (the owner's
7
+ // local decision model, reached through the decision-plane client)
8
+ // as one yes/no relevance question each; confirmed candidates move
9
+ // up, nothing is dropped. Hard timeout, any failure keeps the BM25
10
+ // order.
11
+ //
12
+ // Neither lane can turn an abstention into a result: both run only on a
13
+ // non-empty BM25 result. Neither runs on a hook path: the hook engine
14
+ // (recallEngine.js / memory-recall.mjs) never imports this file, so no network
15
+ // code can sit inside an armed hook deadline. Never throws.
16
+
17
+ import { cosineSimilarity, createEmbeddingProvider } from '../codeintel/vectorProvider.js';
18
+ import { createDecisionClient } from '../../decision/client.js';
19
+ import { resolveConfigStage } from '../runtimeConfig.js';
20
+ import { fuseRanked } from './recallEngine.js';
21
+ import { hitForDoc } from './memoryHit.js';
22
+
23
+ export const VECTOR_WEIGHT = 0.6;
24
+ export const VECTOR_TOP_K = 10;
25
+ export const VECTOR_MIN_COSINE = 0.2;
26
+ export const RERANK_TOP_N = 6;
27
+ export const RERANK_TIMEOUT_MS = 1500;
28
+ export const RERANK_DECISION_KEY = 'memory.relevance.v1';
29
+ const SNIPPET_CHARS = 200;
30
+ const QUERY_CHARS = 200;
31
+
32
+ const envFlag = (name) => {
33
+ const raw = process.env[name];
34
+ return raw === '1' ? true : raw === '0' ? false : null;
35
+ };
36
+
37
+ export function resolveLaneFlags(config) {
38
+ const recall = config?.memory?.recall ?? {};
39
+ const vectorEnv = envFlag('UKIT_MEMORY_VECTOR');
40
+ const rerankEnv = envFlag('UKIT_MEMORY_RERANK');
41
+ const rerankStage = resolveConfigStage(config, 'memory.recall.rerank.stage');
42
+ return {
43
+ vector: vectorEnv ?? recall.vector?.enabled === true,
44
+ rerank: rerankEnv ?? (rerankStage === 'canary' || rerankStage === 'default'),
45
+ rerankStage,
46
+ vectorWeight: Number.isFinite(recall.vector?.weight) ? recall.vector.weight : VECTOR_WEIGHT,
47
+ vectorMinCosine: Number.isFinite(recall.vector?.minCosine) ? recall.vector.minCosine : VECTOR_MIN_COSINE,
48
+ rerankTopN: Number.isInteger(recall.rerank?.topN) && recall.rerank.topN > 0
49
+ ? Math.min(recall.rerank.topN, 8) : RERANK_TOP_N,
50
+ rerankTimeoutMs: Number.isFinite(recall.rerank?.timeoutMs) && recall.rerank.timeoutMs > 0
51
+ ? recall.rerank.timeoutMs : RERANK_TIMEOUT_MS,
52
+ };
53
+ }
54
+
55
+ const providers = new Map();
56
+ function providerFor(projectRoot, config) {
57
+ const embedding = config?.codeIntel?.embedding;
58
+ const key = `${projectRoot}\n${embedding?.provider ?? 'hashed'}\n${embedding?.dimensions ?? ''}`;
59
+ let provider = providers.get(key);
60
+ if (!provider) {
61
+ provider = createEmbeddingProvider({ projectRoot, config });
62
+ providers.set(key, provider);
63
+ }
64
+ return provider;
65
+ }
66
+
67
+ const docText = (doc) => String(doc?.text ?? doc?.summary ?? '');
68
+
69
+ // Ids of the docs nearest the query by cosine, best first (floor + top-K).
70
+ export async function vectorRanking(docs, query, { provider, minCosine = VECTOR_MIN_COSINE, topK = VECTOR_TOP_K } = {}) {
71
+ try {
72
+ if (!provider || typeof provider.embed !== 'function') return [];
73
+ const queryVec = await provider.embed(String(query ?? ''));
74
+ const scored = [];
75
+ for (const doc of docs) {
76
+ const cos = cosineSimilarity(queryVec, await provider.embed(docText(doc)));
77
+ if (cos >= minCosine) scored.push({ id: doc.id, cos });
78
+ }
79
+ scored.sort((a, b) => b.cos - a.cos || String(a.id).localeCompare(String(b.id)));
80
+ return scored.slice(0, topK).map((entry) => entry.id);
81
+ } catch {
82
+ return [];
83
+ }
84
+ }
85
+
86
+ // RRF fusion of the BM25 hit order with the vector order. Vector-only docs
87
+ // enter as hits; BM25 hits keep their own score.
88
+ export function fuseVector(hits, vectorIds, docsById, { weight = VECTOR_WEIGHT, limit = 10 } = {}) {
89
+ if (!Array.isArray(vectorIds) || vectorIds.length === 0) return hits;
90
+ const byId = new Map(hits.map((hit) => [hit.id, hit]));
91
+ const fused = fuseRanked([hits.map((h) => h.id), vectorIds], { weights: [1, weight] });
92
+ const out = [];
93
+ for (const { id } of fused) {
94
+ const hit = byId.get(id) ?? (docsById.get(id) ? hitForDoc(docsById.get(id), 0, ['vector']) : null);
95
+ if (hit) out.push(hit);
96
+ }
97
+ return out.slice(0, Math.max(limit, 1));
98
+ }
99
+
100
+ const oneLine = (text, max) => String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, max);
101
+
102
+ export function buildRerankBatch(query, hits) {
103
+ const questions = hits.map((hit, i) => ({
104
+ decisionKey: `${RERANK_DECISION_KEY}#c${i + 1}`,
105
+ schemaVersion: 1,
106
+ family: 'memory',
107
+ owner: 'memoryRecall',
108
+ kind: 'noul',
109
+ instruction: `Does this project note answer the question? Question: ${oneLine(query, QUERY_CHARS)} | Note: ${oneLine(hit.snippet ?? hit.summary, SNIPPET_CHARS)}`,
110
+ candidates: [],
111
+ }));
112
+ return { statePacket: { task: 'memory-relevance', candidateCount: hits.length }, questions };
113
+ }
114
+
115
+ function withTimeout(promise, ms) {
116
+ let timer;
117
+ const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ status: 'timeout', answers: [] }), ms); });
118
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
119
+ }
120
+
121
+ // Stable partition: candidates the model confirms first, the rest in their
122
+ // BM25 order, everything past topN untouched. `meta.status` is the client's
123
+ // batch status; `meta.applied` is true only when a valid answer moved something.
124
+ export async function rerankHits(hits, query, {
125
+ client, topN = RERANK_TOP_N, timeoutMs = RERANK_TIMEOUT_MS,
126
+ } = {}) {
127
+ const meta = { status: 'skipped', applied: false };
128
+ try {
129
+ if (!client || !Array.isArray(hits) || hits.length < 2) return { hits, meta };
130
+ const head = hits.slice(0, topN);
131
+ const batch = buildRerankBatch(query, head);
132
+ const result = await withTimeout(
133
+ client.requestBatch({ ...batch, deadlineMs: timeoutMs }),
134
+ timeoutMs,
135
+ );
136
+ meta.status = result?.status ?? 'invalid';
137
+ const verdict = new Map();
138
+ for (const answer of result?.answers ?? []) {
139
+ // noul answers decode to P(true) in [0,1]; >= 0.5 confirms the candidate.
140
+ if (answer?.validationStatus !== 'valid' || !Number.isFinite(answer.value)) continue;
141
+ const idx = Number(String(answer.decisionKey).split('#c')[1]);
142
+ if (Number.isInteger(idx) && idx >= 1 && idx <= head.length) verdict.set(idx - 1, answer.value >= 0.5);
143
+ }
144
+ if (verdict.size === 0) return { hits, meta };
145
+ const yes = head.filter((_, i) => verdict.get(i) === true);
146
+ const rest = head.filter((_, i) => verdict.get(i) !== true);
147
+ const reordered = [...yes, ...rest, ...hits.slice(topN)];
148
+ meta.applied = reordered.some((hit, i) => hit !== hits[i]);
149
+ return { hits: reordered, meta };
150
+ } catch {
151
+ meta.status = 'error';
152
+ return { hits, meta };
153
+ }
154
+ }
155
+
156
+ // Entry point for retrieval.search(). `hits` is the BM25 result; empty stays empty.
157
+ export async function applyRecallLanes({
158
+ config, query, docs, hits, limit = 10, projectRoot, deps = {},
159
+ } = {}) {
160
+ const flags = resolveLaneFlags(config);
161
+ if ((!flags.vector && !flags.rerank) || !Array.isArray(hits) || hits.length === 0) {
162
+ return { hits, lanes: {} };
163
+ }
164
+ const lanes = {};
165
+ let out = hits;
166
+ if (flags.vector) {
167
+ const provider = deps.provider ?? providerFor(projectRoot, config);
168
+ const ids = await vectorRanking(docs ?? [], query, { provider, minCosine: flags.vectorMinCosine });
169
+ const docsById = new Map((docs ?? []).map((doc) => [doc.id, doc]));
170
+ out = fuseVector(out, ids, docsById, { weight: flags.vectorWeight, limit });
171
+ lanes.vector = { candidates: ids.length };
172
+ }
173
+ if (flags.rerank) {
174
+ const client = deps.client ?? createDecisionClient({ config, projectRoot, transport: deps.transport });
175
+ const reranked = await rerankHits(out, query, {
176
+ client, topN: flags.rerankTopN, timeoutMs: flags.rerankTimeoutMs,
177
+ });
178
+ out = reranked.hits;
179
+ lanes.rerank = reranked.meta;
180
+ }
181
+ return { hits: out, lanes };
182
+ }
@@ -8,6 +8,7 @@ export const RECORD_TYPES = Object.freeze([
8
8
  'derived_fact',
9
9
  'episode',
10
10
  'procedure',
11
+ 'observation',
11
12
  ]);
12
13
 
13
14
  export const RECORD_SCOPES = Object.freeze([
@@ -66,6 +67,27 @@ function newId() {
66
67
  return `mem_${crypto.randomBytes(6).toString('hex')}`;
67
68
  }
68
69
 
70
+ // 3.5.4: an observation is a consolidated note backed by >= 2 distinct episodes.
71
+ // Structural check only (no store lookup); existence is validateObservation's job.
72
+ export const MIN_OBSERVATION_EVIDENCE = 2;
73
+ export function observationEvidenceErrors(record) {
74
+ const ev = record?.meta?.evidence;
75
+ if (!ev || typeof ev !== 'object' || Array.isArray(ev) || !Array.isArray(ev.episodes)) {
76
+ return ['observation requires meta.evidence = { episodes: [ids], count }'];
77
+ }
78
+ const errors = [];
79
+ const ids = ev.episodes;
80
+ if (ids.some((id) => typeof id !== 'string' || !RECORD_ID_RE.test(id))) {
81
+ errors.push('observation evidence episodes must be record ids');
82
+ }
83
+ const distinct = new Set(ids).size;
84
+ if (distinct < MIN_OBSERVATION_EVIDENCE) {
85
+ errors.push(`observation needs at least ${MIN_OBSERVATION_EVIDENCE} distinct evidence episodes`);
86
+ }
87
+ if (ev.count !== distinct) errors.push('observation evidence count must equal the distinct episode count');
88
+ return errors;
89
+ }
90
+
69
91
  /**
70
92
  * validateRecord(record) → { ok: boolean, errors: string[] }
71
93
  */
@@ -147,6 +169,7 @@ export function validateRecord(record) {
147
169
  }
148
170
  }
149
171
  }
172
+ if (record.type === 'observation') errors.push(...observationEvidenceErrors(record));
150
173
  if (record.valid_from != null
151
174
  && (typeof record.valid_from !== 'number' || !Number.isFinite(record.valid_from))) {
152
175
  errors.push('valid_from must be epoch ms | null');
@@ -13,6 +13,8 @@ import { resolveMemoryStage, appendRolloutReceipt, latencyBandForMs } from './me
13
13
  import { normalize, tokenize } from './recordIndex.js';
14
14
  import { buildMemoryHit, rankHitsDetailed } from './memoryHit.js';
15
15
  import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './recallEngine.js';
16
+ import { loadTopicDocs, topicsRecallEnabled } from './topicPages.js';
17
+ import { applyRecallLanes } from './recallLanes.js';
16
18
  import { resolveRecordFreshness } from './memoryFreshness.js';
17
19
  import { getHeadSha } from '../codeintel/freshness.js';
18
20
  import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
@@ -327,6 +329,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
327
329
  const markdownDocs = scope === 'all' || scope === 'project' || scope === 'repo'
328
330
  ? await loadMarkdownDocs(projectRoot, normalizeMemorySources(config)).catch(() => [])
329
331
  : [];
332
+ // 3.5.5 topic pages: recall documents of their own kind, only when enabled
333
+ // (memory.topics.recall / UKIT_MEMORY_TOPICS); a stale page carries validity.
334
+ if (markdownDocs.length > 0 && topicsRecallEnabled(config)) {
335
+ markdownDocs.push(...await loadTopicDocs(projectRoot).catch(() => []));
336
+ }
330
337
 
331
338
  if (!v2AllowsLegacyFallback(state)) {
332
339
  // One engine (recallEngine.js BM25): eligibility first, then IDF over the
@@ -337,7 +344,12 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
337
344
  const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
338
345
  query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
339
346
  });
340
- const results = await attachFreshness(ranked.hits, records, { projectRoot, watermarks, now });
347
+ // Optional vector / rerank lanes (3.5.5, off by default, CLI path only; an
348
+ // empty BM25 result stays empty so abstention is untouched).
349
+ const laned = await applyRecallLanes({
350
+ config, query, docs: ranked.docs, hits: ranked.hits, limit, projectRoot,
351
+ });
352
+ const results = await attachFreshness(laned.hits, records, { projectRoot, watermarks, now });
341
353
  emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
342
354
  return results;
343
355
  }
@@ -0,0 +1,75 @@
1
+ // `ukit memory topic build <dir>... | list | show <dir> | check [--json]` (3.5.5).
2
+ // Thin CLI runner over topicPages.js: gathers the memory notes that name paths
3
+ // (markdown bullets + active records) and writes/reads docs/memory/topics pages.
4
+ import { loadRuntimeConfig } from '../runtimeConfig.js';
5
+ import { loadSensitiveAllowlist, scanText } from '../sensitiveValueScanner.js';
6
+ import { loadRecords } from './storeV2.js';
7
+ import { loadMarkdownDocs, normalizeMemorySources } from './recallEngine.js';
8
+ import {
9
+ checkTopics, listTopics, readTopicPage, topicRel, validTopicDir, writeTopicPage,
10
+ } from './topicPages.js';
11
+
12
+ async function gatherNotes(projectRoot, config) {
13
+ const notes = [];
14
+ try {
15
+ for (const d of await loadMarkdownDocs(projectRoot, normalizeMemorySources(config))) notes.push({ ref: d.provenance, text: d.text });
16
+ } catch { /* no markdown notes */ }
17
+ try {
18
+ for (const r of await loadRecords(projectRoot)) {
19
+ if (r?.superseded_by || r?.status === 'archived' || r?.type === 'episode') continue;
20
+ notes.push({ ref: `record:${r.id}`, text: String(r.text ?? ''), type: r.type });
21
+ }
22
+ } catch { /* no records */ }
23
+ return notes;
24
+ }
25
+
26
+ export async function runTopicCommand(projectRoot, args = []) {
27
+ const [action, ...rest] = args;
28
+ const dirs = rest.filter((a) => !a.startsWith('--'));
29
+ const json = rest.includes('--json');
30
+ if (action === 'build') {
31
+ if (dirs.length === 0) return { ok: false, output: 'Usage: ukit memory topic build <dir> [<dir>...]' };
32
+ const config = await loadRuntimeConfig(projectRoot).catch(() => ({}));
33
+ const notes = await gatherNotes(projectRoot, config);
34
+ const rows = [];
35
+ let ok = true;
36
+ for (const dir of dirs) {
37
+ if (!validTopicDir(dir)) { ok = false; rows.push(`[UKit] rejected: ${dir} is not a repo-relative directory`); continue; }
38
+ try {
39
+ const built = await writeTopicPage(projectRoot, dir, notes);
40
+ rows.push(`[UKit] topic built: ${built.rel} (${built.count} files, ${built.bytes} bytes)`);
41
+ } catch (error) {
42
+ ok = false;
43
+ rows.push(`[UKit] topic not built: ${error.message}`);
44
+ }
45
+ }
46
+ return { ok, output: rows.join('\n') };
47
+ }
48
+ if (action === 'list') {
49
+ const topics = await listTopics(projectRoot);
50
+ if (json) return { ok: true, output: JSON.stringify(topics, null, 2) };
51
+ if (topics.length === 0) return { ok: true, output: '[UKit] no topic pages. Build one with `ukit memory topic build <dir>`.' };
52
+ return { ok: true, output: topics.map((t) => `${t.rel} ${t.dir} ${t.missing ? 'MISSING-DIR' : t.stale ? 'STALE' : 'current'}${t.filled ? ' agent-notes' : ''} ${t.bytes}B`).join('\n') };
53
+ }
54
+ if (action === 'show') {
55
+ if (dirs.length !== 1) return { ok: false, output: 'Usage: ukit memory topic show <dir>' };
56
+ const page = await readTopicPage(projectRoot, topicRel(dirs[0]));
57
+ if (!page) return { ok: false, output: `[UKit] no topic page for ${dirs[0]} (expected ${topicRel(dirs[0])})` };
58
+ const label = page.missing
59
+ ? '[UKit] STALE: the described directory no longer exists. Do not rely on this page.\n'
60
+ : page.stale ? '[UKit] STALE: the directory changed since this page was built. Verify before relying on it; rebuild with `ukit memory topic build`.\n' : '';
61
+ return { ok: true, output: `${label}${page.text}` };
62
+ }
63
+ if (action === 'check') {
64
+ const config = await loadRuntimeConfig(projectRoot).catch(() => ({}));
65
+ const allowlistHashes = loadSensitiveAllowlist(config);
66
+ const res = await checkTopics(projectRoot, { scan: (text) => scanText(text, { allowlistHashes }) });
67
+ const ok = res.violations.length === 0;
68
+ if (json) return { ok, output: JSON.stringify({ ok, ...res }, null, 2) };
69
+ const rows = [`[UKit] topic check: ${ok ? 'OK' : 'VIOLATIONS'} - ${res.pages} page(s), ${res.warnings.length} stale`,
70
+ ...res.violations.map((v) => ` violation ${v.file}:${v.line} ${v.code} - ${v.message}`),
71
+ ...res.warnings.map((v) => ` warning ${v.file} ${v.code} - ${v.message}`)];
72
+ return { ok, output: rows.join('\n') };
73
+ }
74
+ return { ok: false, output: 'Usage: ukit memory topic build <dir>... | list [--json] | show <dir> | check [--json]' };
75
+ }