@ngockhoale/ukit 3.5.4 → 3.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/manifests/engineConformance.yaml +0 -13
  3. package/manifests/platform.full.yaml +25 -0
  4. package/package.json +3 -9
  5. package/src/cli/commands/indexTools.js +1 -0
  6. package/src/cli/commands/memory.js +44 -4
  7. package/src/core/applyPlan.js +25 -72
  8. package/src/core/diffPlan.js +38 -9
  9. package/src/core/managedTemplateMerge.js +146 -0
  10. package/src/core/memory/memoryCheckRun.js +13 -1
  11. package/src/core/memory/memoryFlags.js +5 -6
  12. package/src/core/memory/memoryHit.js +31 -15
  13. package/src/core/memory/policy.js +5 -0
  14. package/src/core/memory/recallEngine.js +39 -3
  15. package/src/core/memory/recallLanes.js +182 -0
  16. package/src/core/memory/retrieval.js +13 -1
  17. package/src/core/memory/store.js +17 -0
  18. package/src/core/memory/topicCommand.js +75 -0
  19. package/src/core/memory/topicPages.js +545 -0
  20. package/src/core/metadata.js +12 -8
  21. package/src/core/runInstallPipeline.js +16 -10
  22. package/src/core/runtimeConfig.js +59 -83
  23. package/src/core/uninstall.js +8 -1
  24. package/src/decision/registry.js +33 -16
  25. package/src/decision/runtimeDecide.js +5 -5
  26. package/src/index/buildIndex.js +287 -6
  27. package/src/index/languageTools.js +100 -0
  28. package/src/index/paths.js +2 -1
  29. package/src/index/queryIndex.js +174 -10
  30. package/src/index/routeResolver.js +2 -2
  31. package/src/index/semanticLane.js +269 -0
  32. package/src/learning/lessonClass.js +110 -0
  33. package/src/learning/patternProposals.js +111 -7
  34. package/src/learning/reeditMiner.js +104 -0
  35. package/src/learning/selfImprove.js +9 -1
  36. package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
  37. package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
  38. package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
  39. package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
  40. package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
  41. package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
  42. package/template_project/.claude/ukit/index/query-index.mjs +9 -1
  43. package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
  44. package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
  45. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
  46. package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
  47. package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
  48. package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
  49. package/template_project/ukit/storage/config.json +19 -34
  50. package/manifests/instructionRules.yaml +0 -442
  51. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
  52. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
  53. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
  54. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
  55. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
  56. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
  57. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
  58. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
  59. package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
  60. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
  61. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
  62. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
  63. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
  64. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
  65. package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
  66. package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
  67. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
  68. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
@@ -3,6 +3,7 @@
3
3
  import { loadRuntimeConfig } from '../runtimeConfig.js';
4
4
  import { loadSensitiveAllowlist, scanText } from '../sensitiveValueScanner.js';
5
5
  import { checkMemorySources } from './memoryCheck.js';
6
+ import { checkTopics } from './topicPages.js';
6
7
  import { expandMemorySources, normalizeMemorySources, parseMarkdownBullets, tokenize } from './recallEngine.js';
7
8
 
8
9
  export async function runMemoryCheck(projectRoot, args = []) {
@@ -17,11 +18,22 @@ export async function runMemoryCheck(projectRoot, args = []) {
17
18
  scan: (text) => scanText(text, { allowlistHashes }),
18
19
  files: files.length > 0 ? files : null,
19
20
  });
21
+ // 3.5.5: topic pages (docs/memory/topics) share the check: size cap, line cap, secrets are
22
+ // violations, a stale fingerprint is a warning. Skipped when the run is scoped with --file.
23
+ if (files.length === 0) {
24
+ const topics = await checkTopics(projectRoot, { scan: (text) => scanText(text, { allowlistHashes }) });
25
+ res.violations.push(...topics.violations);
26
+ res.warnings.push(...topics.warnings);
27
+ res.stats.topicPages = topics.pages;
28
+ if (topics.violations.length > 0) res.ok = false;
29
+ }
20
30
  if (json) return { ok: res.ok, output: JSON.stringify(res, null, 2) };
21
31
  const rows = [
22
32
  `[UKit] memory check: ${res.ok ? 'OK' : 'VIOLATIONS'} — ${res.stats.files} file(s), ${res.stats.bullets} note(s), core ${res.stats.coreLines} lines / ${res.stats.coreBytes} bytes`,
23
33
  ...res.violations.map((v) => ` violation ${v.file}:${v.line} ${v.code} — ${v.message}`),
24
34
  ];
25
- if (res.warnings.length > 0) rows.push(` ${res.warnings.length} warning(s) (notes naming no existing file); \`--json\` lists them`);
35
+ for (const w of res.warnings.filter((x) => String(x.code).startsWith('topic-'))) rows.push(` warning ${w.file} ${w.code} - ${w.message}`);
36
+ const noteWarnings = res.warnings.filter((x) => !String(x.code).startsWith('topic-'));
37
+ if (noteWarnings.length > 0) rows.push(` ${noteWarnings.length} warning(s) (notes naming no existing file); \`--json\` lists them`);
26
38
  return { ok: res.ok, output: rows.join('\n') };
27
39
  }
@@ -4,12 +4,11 @@
4
4
  // resolveConfigStage machinery: off → shadow → canary → default. Flags only
5
5
  // ever REDUCE capability — malformed/missing resolves 'off', killSwitch is
6
6
  // absolute, and 'canary' without an opted-in projectId resolves 'off'.
7
- // The `decision` plane is reserved for the memoryV2 candidate-classification
8
- // lane (registry key `learn.candidate-class.v1`, family 'learn'). Note the
9
- // decision-plane stage namespace is separate: promotion gates that key via
10
- // `decisionPlane.families.learn.stage`, NOT `memoryV2.decision.stage` — this
11
- // flag only guards the memoryV2 lane itself. Zero consumers today — do not
12
- // remove; user configs may pin its stage key.
7
+ // The `decision` plane is reserved for the memoryV2 decision lane and has no
8
+ // consumer today — do not remove; user configs may pin its stage key. Lesson
9
+ // classification (registry key `learn.candidate-class.v1`, family 'learn',
10
+ // owner src/learning/lessonClass.js) is a separate namespace: it is gated by
11
+ // `decisionPlane.families.learn.stage`, NOT `memoryV2.decision.stage`.
13
12
  //
14
13
  // Shadow receipts are the only telemetry: bounded JSONL of counts/codes/
15
14
  // latency bands — never record text, prompt bodies, or secrets.
@@ -126,15 +126,11 @@ export function buildMarkdownHit(doc, { score = 0 } = {}) {
126
126
  // and an explicit empty result. Type weights, the evidence bonus and the
127
127
  // half-life recency bonus were removed from the score: they only ever added a
128
128
  // query-independent constant that surfaced content-free episodes.
129
- export function rankHitsDetailed(records, {
130
- query = '',
131
- queryTokens,
132
- eligibleCtx = {},
133
- policy = {},
134
- scope = 'all',
135
- limit = 10,
136
- now,
137
- extraDocs = [],
129
+ // The eligible document set (records through the policy gate and requested
130
+ // scope, plus `extraDocs`): the exact corpus BM25 ranks, reused by the optional
131
+ // vector lane so both lanes see the same candidates.
132
+ export function collectEligibleDocs(records, {
133
+ eligibleCtx = {}, policy = {}, scope = 'all', now, extraDocs = [],
138
134
  } = {}) {
139
135
  const nowMs = typeof now === 'number' ? now : Date.now();
140
136
  const ctx = { ...eligibleCtx, now: nowMs };
@@ -148,14 +144,34 @@ export function rankHitsDetailed(records, {
148
144
  docs.push(record);
149
145
  }
150
146
  for (const doc of extraDocs) docs.push(doc);
147
+ return docs;
148
+ }
149
+
150
+ // Hit for one ranked doc (markdown bullet or record).
151
+ export function hitForDoc(doc, score, extraReasons = []) {
152
+ return doc.type === 'markdown'
153
+ ? buildMarkdownHit(doc, { score })
154
+ : buildMemoryHit(doc, {
155
+ score,
156
+ reasons: ['lexical', ...extraReasons, `type:${doc.type}`, ...(hasLocatorEvidence(doc) ? ['evidence'] : [])],
157
+ });
158
+ }
159
+
160
+ export function rankHitsDetailed(records, {
161
+ query = '',
162
+ queryTokens,
163
+ eligibleCtx = {},
164
+ policy = {},
165
+ scope = 'all',
166
+ limit = 10,
167
+ now,
168
+ extraDocs = [],
169
+ } = {}) {
170
+ const docs = collectEligibleDocs(records, { eligibleCtx, policy, scope, now, extraDocs });
151
171
  const result = rankDocs(docs, Array.isArray(queryTokens) ? queryTokens : query, { limit });
152
- const hits = result.ranked.map((entry) => (entry.doc.type === 'markdown'
153
- ? buildMarkdownHit(entry.doc, { score: entry.score })
154
- : buildMemoryHit(entry.doc, {
155
- score: entry.score,
156
- reasons: ['lexical', `type:${entry.doc.type}`, ...(hasLocatorEvidence(entry.doc) ? ['evidence'] : [])],
157
- })));
172
+ const hits = result.ranked.map((entry) => hitForDoc(entry.doc, entry.score));
158
173
  return {
174
+ docs,
159
175
  hits,
160
176
  candidateCount: result.candidateCount,
161
177
  queryTokenCount: result.queryTokenCount,
@@ -70,6 +70,11 @@ export function eligible(record, context = {}, policy = {}) {
70
70
  if (NON_AUTHORITY_TRUST_TIERS.includes(record.trust_tier)) {
71
71
  return deny('non-authority-trust-tier');
72
72
  }
73
+ // Records written before candidates got trust_tier 'candidate' sit as
74
+ // active/verified with meta.legacyStatus 'pending' — still unreviewed.
75
+ if (record.meta?.legacyStatus === 'pending') {
76
+ return deny('pending-approval');
77
+ }
73
78
 
74
79
  if (record.scope === 'user') {
75
80
  if (context.includeUser !== true) {
@@ -155,9 +155,17 @@ function stem(token) {
155
155
 
156
156
  // Tokens of `text`: whole tokens plus their separator-split parts
157
157
  // (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
158
- export function tokenize(text) {
158
+ // A camelCase identifier also yields its words (`deriveDeterministicBucket`
159
+ // -> derive, deterministic, bucket); the whole identifier stays a token too.
160
+ const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
161
+ const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
162
+
163
+ // Query side only (`{ camel: true }`): in a long note the split words of its many
164
+ // identifiers (`postEditReviewPolicy`) would add common-word matches.
165
+ export function tokenize(text, { camel = false } = {}) {
159
166
  const out = [];
160
- for (const raw of normalize(text).split(/\s+/)) {
167
+ const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
168
+ for (const raw of normalize(source).split(/\s+/)) {
161
169
  if (!raw) continue;
162
170
  const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
163
171
  const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
@@ -169,6 +177,21 @@ export function tokenize(text) {
169
177
  return out;
170
178
  }
171
179
 
180
+ // File names in a query (`name.ext`): a query that names a file is only
181
+ // answered by a note that names it too and covers the rest of the question.
182
+ const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
183
+ const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
184
+ const FILE_REST_COVERAGE = 0.3;
185
+ const FILE_REST_MIN_TOKENS = 2;
186
+ function queryFileSpec(query) {
187
+ if (typeof query !== 'string') return null;
188
+ const files = query.match(QUERY_FILE_RE);
189
+ if (!files) return null;
190
+ const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
191
+ const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
192
+ return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
193
+ }
194
+
172
195
  function episodeBody(text) {
173
196
  return String(text ?? '')
174
197
  .replace(/^\s*Session\s+\S+?:\s*/i, '')
@@ -272,7 +295,7 @@ function idf(totalDocs, docFreq) {
272
295
  // `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
273
296
  // `query` is a string or a pre-tokenized array. Never throws.
274
297
  export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
275
- const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
298
+ const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
276
299
  const empty = {
277
300
  ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
278
301
  };
@@ -302,6 +325,10 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
302
325
  let queryMass = 0;
303
326
  for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
304
327
 
328
+ const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
329
+ let restMass = 0;
330
+ if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
331
+
305
332
  const scored = [];
306
333
  let candidateCount = 0;
307
334
  for (const { doc, tf, length } of stats) {
@@ -324,6 +351,15 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
324
351
  // clear the absolute floor above while the query's real subject is absent.
325
352
  // It must also cover a fraction of the query's own IDF mass.
326
353
  if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
354
+ if (fileSpec) {
355
+ let named = false;
356
+ for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
357
+ let restHit = 0;
358
+ for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
359
+ // One leftover word ("what does X check") is too little to judge coverage by.
360
+ const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
361
+ if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
362
+ }
327
363
  scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
328
364
  }
329
365
  // Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
@@ -0,0 +1,182 @@
1
+ // Optional recall lanes (3.5.5), CLI path only, both off unless enabled.
2
+ //
3
+ // vector - embedding cosine over the eligible docs, fused with the BM25
4
+ // order by RRF (recallEngine.fuseRanked). Provider = the
5
+ // codeIntel.embedding seam (hashed by default, `auto` takes a loader).
6
+ // rerank - the top BM25 candidates are put to `unic-decision` (the owner's
7
+ // local decision model, reached through the decision-plane client)
8
+ // as one yes/no relevance question each; confirmed candidates move
9
+ // up, nothing is dropped. Hard timeout, any failure keeps the BM25
10
+ // order.
11
+ //
12
+ // Neither lane can turn an abstention into a result: both run only on a
13
+ // non-empty BM25 result. Neither runs on a hook path: the hook engine
14
+ // (recallEngine.js / memory-recall.mjs) never imports this file, so no network
15
+ // code can sit inside an armed hook deadline. Never throws.
16
+
17
+ import { cosineSimilarity, createEmbeddingProvider } from '../codeintel/vectorProvider.js';
18
+ import { createDecisionClient } from '../../decision/client.js';
19
+ import { resolveConfigStage } from '../runtimeConfig.js';
20
+ import { fuseRanked } from './recallEngine.js';
21
+ import { hitForDoc } from './memoryHit.js';
22
+
23
+ export const VECTOR_WEIGHT = 0.6;
24
+ export const VECTOR_TOP_K = 10;
25
+ export const VECTOR_MIN_COSINE = 0.2;
26
+ export const RERANK_TOP_N = 6;
27
+ export const RERANK_TIMEOUT_MS = 1500;
28
+ export const RERANK_DECISION_KEY = 'memory.relevance.v1';
29
+ const SNIPPET_CHARS = 200;
30
+ const QUERY_CHARS = 200;
31
+
32
+ const envFlag = (name) => {
33
+ const raw = process.env[name];
34
+ return raw === '1' ? true : raw === '0' ? false : null;
35
+ };
36
+
37
+ export function resolveLaneFlags(config) {
38
+ const recall = config?.memory?.recall ?? {};
39
+ const vectorEnv = envFlag('UKIT_MEMORY_VECTOR');
40
+ const rerankEnv = envFlag('UKIT_MEMORY_RERANK');
41
+ const rerankStage = resolveConfigStage(config, 'memory.recall.rerank.stage');
42
+ return {
43
+ vector: vectorEnv ?? recall.vector?.enabled === true,
44
+ rerank: rerankEnv ?? (rerankStage === 'canary' || rerankStage === 'default'),
45
+ rerankStage,
46
+ vectorWeight: Number.isFinite(recall.vector?.weight) ? recall.vector.weight : VECTOR_WEIGHT,
47
+ vectorMinCosine: Number.isFinite(recall.vector?.minCosine) ? recall.vector.minCosine : VECTOR_MIN_COSINE,
48
+ rerankTopN: Number.isInteger(recall.rerank?.topN) && recall.rerank.topN > 0
49
+ ? Math.min(recall.rerank.topN, 8) : RERANK_TOP_N,
50
+ rerankTimeoutMs: Number.isFinite(recall.rerank?.timeoutMs) && recall.rerank.timeoutMs > 0
51
+ ? recall.rerank.timeoutMs : RERANK_TIMEOUT_MS,
52
+ };
53
+ }
54
+
55
+ const providers = new Map();
56
+ function providerFor(projectRoot, config) {
57
+ const embedding = config?.codeIntel?.embedding;
58
+ const key = `${projectRoot}\n${embedding?.provider ?? 'hashed'}\n${embedding?.dimensions ?? ''}`;
59
+ let provider = providers.get(key);
60
+ if (!provider) {
61
+ provider = createEmbeddingProvider({ projectRoot, config });
62
+ providers.set(key, provider);
63
+ }
64
+ return provider;
65
+ }
66
+
67
+ const docText = (doc) => String(doc?.text ?? doc?.summary ?? '');
68
+
69
+ // Ids of the docs nearest the query by cosine, best first (floor + top-K).
70
+ export async function vectorRanking(docs, query, { provider, minCosine = VECTOR_MIN_COSINE, topK = VECTOR_TOP_K } = {}) {
71
+ try {
72
+ if (!provider || typeof provider.embed !== 'function') return [];
73
+ const queryVec = await provider.embed(String(query ?? ''));
74
+ const scored = [];
75
+ for (const doc of docs) {
76
+ const cos = cosineSimilarity(queryVec, await provider.embed(docText(doc)));
77
+ if (cos >= minCosine) scored.push({ id: doc.id, cos });
78
+ }
79
+ scored.sort((a, b) => b.cos - a.cos || String(a.id).localeCompare(String(b.id)));
80
+ return scored.slice(0, topK).map((entry) => entry.id);
81
+ } catch {
82
+ return [];
83
+ }
84
+ }
85
+
86
+ // RRF fusion of the BM25 hit order with the vector order. Vector-only docs
87
+ // enter as hits; BM25 hits keep their own score.
88
+ export function fuseVector(hits, vectorIds, docsById, { weight = VECTOR_WEIGHT, limit = 10 } = {}) {
89
+ if (!Array.isArray(vectorIds) || vectorIds.length === 0) return hits;
90
+ const byId = new Map(hits.map((hit) => [hit.id, hit]));
91
+ const fused = fuseRanked([hits.map((h) => h.id), vectorIds], { weights: [1, weight] });
92
+ const out = [];
93
+ for (const { id } of fused) {
94
+ const hit = byId.get(id) ?? (docsById.get(id) ? hitForDoc(docsById.get(id), 0, ['vector']) : null);
95
+ if (hit) out.push(hit);
96
+ }
97
+ return out.slice(0, Math.max(limit, 1));
98
+ }
99
+
100
+ const oneLine = (text, max) => String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, max);
101
+
102
+ export function buildRerankBatch(query, hits) {
103
+ const questions = hits.map((hit, i) => ({
104
+ decisionKey: `${RERANK_DECISION_KEY}#c${i + 1}`,
105
+ schemaVersion: 1,
106
+ family: 'memory',
107
+ owner: 'memoryRecall',
108
+ kind: 'noul',
109
+ instruction: `Does this project note answer the question? Question: ${oneLine(query, QUERY_CHARS)} | Note: ${oneLine(hit.snippet ?? hit.summary, SNIPPET_CHARS)}`,
110
+ candidates: [],
111
+ }));
112
+ return { statePacket: { task: 'memory-relevance', candidateCount: hits.length }, questions };
113
+ }
114
+
115
+ function withTimeout(promise, ms) {
116
+ let timer;
117
+ const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ status: 'timeout', answers: [] }), ms); });
118
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
119
+ }
120
+
121
+ // Stable partition: candidates the model confirms first, the rest in their
122
+ // BM25 order, everything past topN untouched. `meta.status` is the client's
123
+ // batch status; `meta.applied` is true only when a valid answer moved something.
124
+ export async function rerankHits(hits, query, {
125
+ client, topN = RERANK_TOP_N, timeoutMs = RERANK_TIMEOUT_MS,
126
+ } = {}) {
127
+ const meta = { status: 'skipped', applied: false };
128
+ try {
129
+ if (!client || !Array.isArray(hits) || hits.length < 2) return { hits, meta };
130
+ const head = hits.slice(0, topN);
131
+ const batch = buildRerankBatch(query, head);
132
+ const result = await withTimeout(
133
+ client.requestBatch({ ...batch, deadlineMs: timeoutMs }),
134
+ timeoutMs,
135
+ );
136
+ meta.status = result?.status ?? 'invalid';
137
+ const verdict = new Map();
138
+ for (const answer of result?.answers ?? []) {
139
+ // noul answers decode to P(true) in [0,1]; >= 0.5 confirms the candidate.
140
+ if (answer?.validationStatus !== 'valid' || !Number.isFinite(answer.value)) continue;
141
+ const idx = Number(String(answer.decisionKey).split('#c')[1]);
142
+ if (Number.isInteger(idx) && idx >= 1 && idx <= head.length) verdict.set(idx - 1, answer.value >= 0.5);
143
+ }
144
+ if (verdict.size === 0) return { hits, meta };
145
+ const yes = head.filter((_, i) => verdict.get(i) === true);
146
+ const rest = head.filter((_, i) => verdict.get(i) !== true);
147
+ const reordered = [...yes, ...rest, ...hits.slice(topN)];
148
+ meta.applied = reordered.some((hit, i) => hit !== hits[i]);
149
+ return { hits: reordered, meta };
150
+ } catch {
151
+ meta.status = 'error';
152
+ return { hits, meta };
153
+ }
154
+ }
155
+
156
+ // Entry point for retrieval.search(). `hits` is the BM25 result; empty stays empty.
157
+ export async function applyRecallLanes({
158
+ config, query, docs, hits, limit = 10, projectRoot, deps = {},
159
+ } = {}) {
160
+ const flags = resolveLaneFlags(config);
161
+ if ((!flags.vector && !flags.rerank) || !Array.isArray(hits) || hits.length === 0) {
162
+ return { hits, lanes: {} };
163
+ }
164
+ const lanes = {};
165
+ let out = hits;
166
+ if (flags.vector) {
167
+ const provider = deps.provider ?? providerFor(projectRoot, config);
168
+ const ids = await vectorRanking(docs ?? [], query, { provider, minCosine: flags.vectorMinCosine });
169
+ const docsById = new Map((docs ?? []).map((doc) => [doc.id, doc]));
170
+ out = fuseVector(out, ids, docsById, { weight: flags.vectorWeight, limit });
171
+ lanes.vector = { candidates: ids.length };
172
+ }
173
+ if (flags.rerank) {
174
+ const client = deps.client ?? createDecisionClient({ config, projectRoot, transport: deps.transport });
175
+ const reranked = await rerankHits(out, query, {
176
+ client, topN: flags.rerankTopN, timeoutMs: flags.rerankTimeoutMs,
177
+ });
178
+ out = reranked.hits;
179
+ lanes.rerank = reranked.meta;
180
+ }
181
+ return { hits: out, lanes };
182
+ }
@@ -13,6 +13,8 @@ import { resolveMemoryStage, appendRolloutReceipt, latencyBandForMs } from './me
13
13
  import { normalize, tokenize } from './recordIndex.js';
14
14
  import { buildMemoryHit, rankHitsDetailed } from './memoryHit.js';
15
15
  import { loadMarkdownDocs, normalizeMemorySources, renderMarkdownDoc } from './recallEngine.js';
16
+ import { loadTopicDocs, topicsRecallEnabled } from './topicPages.js';
17
+ import { applyRecallLanes } from './recallLanes.js';
16
18
  import { resolveRecordFreshness } from './memoryFreshness.js';
17
19
  import { getHeadSha } from '../codeintel/freshness.js';
18
20
  import { emitRetrievalQuery, emitMemoryRetrieved } from './memoryEmit.js';
@@ -327,6 +329,11 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
327
329
  const markdownDocs = scope === 'all' || scope === 'project' || scope === 'repo'
328
330
  ? await loadMarkdownDocs(projectRoot, normalizeMemorySources(config)).catch(() => [])
329
331
  : [];
332
+ // 3.5.5 topic pages: recall documents of their own kind, only when enabled
333
+ // (memory.topics.recall / UKIT_MEMORY_TOPICS); a stale page carries validity.
334
+ if (markdownDocs.length > 0 && topicsRecallEnabled(config)) {
335
+ markdownDocs.push(...await loadTopicDocs(projectRoot).catch(() => []));
336
+ }
330
337
 
331
338
  if (!v2AllowsLegacyFallback(state)) {
332
339
  // One engine (recallEngine.js BM25): eligibility first, then IDF over the
@@ -337,7 +344,12 @@ export async function search(query, scope = 'all', limit = 10, { projectRoot, pr
337
344
  const ranked = rankHitsDetailed(anchored.slice(0, records.length), {
338
345
  query, eligibleCtx, policy, scope, limit, now, extraDocs: anchored.slice(records.length),
339
346
  });
340
- const results = await attachFreshness(ranked.hits, records, { projectRoot, watermarks, now });
347
+ // Optional vector / rerank lanes (3.5.5, off by default, CLI path only; an
348
+ // empty BM25 result stays empty so abstention is untouched).
349
+ const laned = await applyRecallLanes({
350
+ config, query, docs: ranked.docs, hits: ranked.hits, limit, projectRoot,
351
+ });
352
+ const results = await attachFreshness(laned.hits, records, { projectRoot, watermarks, now });
341
353
  emitQuery({ lane: 'v2', indexed: false, candidateCount: ranked.candidateCount, results });
342
354
  return results;
343
355
  }
@@ -608,6 +608,8 @@ function patternCandidateFromRecord(record) {
608
608
  };
609
609
  }
610
610
 
611
+ const DAY_MS = 24 * 60 * 60 * 1000;
612
+
611
613
  async function proposePatternCandidateV2(projectRoot, projectId, candidate, { homeDir } = {}) {
612
614
  const v2 = await storeV2();
613
615
  const normalizedText = normalizePatternText(candidate?.text);
@@ -643,6 +645,19 @@ async function proposePatternCandidateV2(projectRoot, projectId, candidate, { ho
643
645
  confidence: 0.4,
644
646
  createdBy: 'propose',
645
647
  projectId,
648
+ // Unreviewed until `memory approve`: candidate tier keeps it out of every
649
+ // recall path; approval flips it to verified and clears the expiry.
650
+ // Only mined lessons ask for an expiry (candidate.ttlDays); a hand-written
651
+ // `memory propose` candidate stays pending until a person decides.
652
+ trustTier: 'candidate',
653
+ validUntil: Number(candidate?.ttlDays) > 0 ? Date.now() + Number(candidate.ttlDays) * DAY_MS : null,
654
+ evidence: (Array.isArray(candidate?.evidence) ? candidate.evidence : []).map((ev) => ({
655
+ kind: ev.kind,
656
+ locator: ev.locator,
657
+ observed_at: Date.now(),
658
+ fingerprint: null,
659
+ trust_tier: 'verified',
660
+ })),
646
661
  meta: { ...entry, legacyStatus: 'pending' },
647
662
  }, { homeDir });
648
663
  return entry;
@@ -684,6 +699,8 @@ async function resolvePatternCandidateV2(projectRoot, projectId, candidateId, de
684
699
  provenance: `${target.provenance ?? 'pattern-candidate'};promoted-from:${target.id}`,
685
700
  // Approval is a trust transition — the promoted record is authoritative.
686
701
  trust_tier: 'verified',
702
+ // A promoted rule must not inherit a mined lesson's expiry.
703
+ valid_until: null,
687
704
  meta: {
688
705
  ...target.meta,
689
706
  legacyStatus: 'approved',
@@ -0,0 +1,75 @@
1
+ // `ukit memory topic build <dir>... | list | show <dir> | check [--json]` (3.5.5).
2
+ // Thin CLI runner over topicPages.js: gathers the memory notes that name paths
3
+ // (markdown bullets + active records) and writes/reads docs/memory/topics pages.
4
+ import { loadRuntimeConfig } from '../runtimeConfig.js';
5
+ import { loadSensitiveAllowlist, scanText } from '../sensitiveValueScanner.js';
6
+ import { loadRecords } from './storeV2.js';
7
+ import { loadMarkdownDocs, normalizeMemorySources } from './recallEngine.js';
8
+ import {
9
+ checkTopics, listTopics, readTopicPage, topicRel, validTopicDir, writeTopicPage,
10
+ } from './topicPages.js';
11
+
12
+ async function gatherNotes(projectRoot, config) {
13
+ const notes = [];
14
+ try {
15
+ for (const d of await loadMarkdownDocs(projectRoot, normalizeMemorySources(config))) notes.push({ ref: d.provenance, text: d.text });
16
+ } catch { /* no markdown notes */ }
17
+ try {
18
+ for (const r of await loadRecords(projectRoot)) {
19
+ if (r?.superseded_by || r?.status === 'archived' || r?.type === 'episode') continue;
20
+ notes.push({ ref: `record:${r.id}`, text: String(r.text ?? ''), type: r.type });
21
+ }
22
+ } catch { /* no records */ }
23
+ return notes;
24
+ }
25
+
26
+ export async function runTopicCommand(projectRoot, args = []) {
27
+ const [action, ...rest] = args;
28
+ const dirs = rest.filter((a) => !a.startsWith('--'));
29
+ const json = rest.includes('--json');
30
+ if (action === 'build') {
31
+ if (dirs.length === 0) return { ok: false, output: 'Usage: ukit memory topic build <dir> [<dir>...]' };
32
+ const config = await loadRuntimeConfig(projectRoot).catch(() => ({}));
33
+ const notes = await gatherNotes(projectRoot, config);
34
+ const rows = [];
35
+ let ok = true;
36
+ for (const dir of dirs) {
37
+ if (!validTopicDir(dir)) { ok = false; rows.push(`[UKit] rejected: ${dir} is not a repo-relative directory`); continue; }
38
+ try {
39
+ const built = await writeTopicPage(projectRoot, dir, notes);
40
+ rows.push(`[UKit] topic built: ${built.rel} (${built.count} files, ${built.bytes} bytes)`);
41
+ } catch (error) {
42
+ ok = false;
43
+ rows.push(`[UKit] topic not built: ${error.message}`);
44
+ }
45
+ }
46
+ return { ok, output: rows.join('\n') };
47
+ }
48
+ if (action === 'list') {
49
+ const topics = await listTopics(projectRoot);
50
+ if (json) return { ok: true, output: JSON.stringify(topics, null, 2) };
51
+ if (topics.length === 0) return { ok: true, output: '[UKit] no topic pages. Build one with `ukit memory topic build <dir>`.' };
52
+ return { ok: true, output: topics.map((t) => `${t.rel} ${t.dir} ${t.missing ? 'MISSING-DIR' : t.stale ? 'STALE' : 'current'}${t.filled ? ' agent-notes' : ''} ${t.bytes}B`).join('\n') };
53
+ }
54
+ if (action === 'show') {
55
+ if (dirs.length !== 1) return { ok: false, output: 'Usage: ukit memory topic show <dir>' };
56
+ const page = await readTopicPage(projectRoot, topicRel(dirs[0]));
57
+ if (!page) return { ok: false, output: `[UKit] no topic page for ${dirs[0]} (expected ${topicRel(dirs[0])})` };
58
+ const label = page.missing
59
+ ? '[UKit] STALE: the described directory no longer exists. Do not rely on this page.\n'
60
+ : page.stale ? '[UKit] STALE: the directory changed since this page was built. Verify before relying on it; rebuild with `ukit memory topic build`.\n' : '';
61
+ return { ok: true, output: `${label}${page.text}` };
62
+ }
63
+ if (action === 'check') {
64
+ const config = await loadRuntimeConfig(projectRoot).catch(() => ({}));
65
+ const allowlistHashes = loadSensitiveAllowlist(config);
66
+ const res = await checkTopics(projectRoot, { scan: (text) => scanText(text, { allowlistHashes }) });
67
+ const ok = res.violations.length === 0;
68
+ if (json) return { ok, output: JSON.stringify({ ok, ...res }, null, 2) };
69
+ const rows = [`[UKit] topic check: ${ok ? 'OK' : 'VIOLATIONS'} - ${res.pages} page(s), ${res.warnings.length} stale`,
70
+ ...res.violations.map((v) => ` violation ${v.file}:${v.line} ${v.code} - ${v.message}`),
71
+ ...res.warnings.map((v) => ` warning ${v.file} ${v.code} - ${v.message}`)];
72
+ return { ok, output: rows.join('\n') };
73
+ }
74
+ return { ok: false, output: 'Usage: ukit memory topic build <dir>... | list [--json] | show <dir> | check [--json]' };
75
+ }