@ngockhoale/ukit 3.5.4 → 3.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/manifests/engineConformance.yaml +0 -13
  3. package/manifests/platform.full.yaml +25 -0
  4. package/package.json +3 -9
  5. package/src/cli/commands/indexTools.js +1 -0
  6. package/src/cli/commands/memory.js +44 -4
  7. package/src/core/applyPlan.js +25 -72
  8. package/src/core/diffPlan.js +38 -9
  9. package/src/core/managedTemplateMerge.js +146 -0
  10. package/src/core/memory/memoryCheckRun.js +13 -1
  11. package/src/core/memory/memoryFlags.js +5 -6
  12. package/src/core/memory/memoryHit.js +31 -15
  13. package/src/core/memory/policy.js +5 -0
  14. package/src/core/memory/recallEngine.js +39 -3
  15. package/src/core/memory/recallLanes.js +182 -0
  16. package/src/core/memory/retrieval.js +13 -1
  17. package/src/core/memory/store.js +17 -0
  18. package/src/core/memory/topicCommand.js +75 -0
  19. package/src/core/memory/topicPages.js +545 -0
  20. package/src/core/metadata.js +12 -8
  21. package/src/core/runInstallPipeline.js +16 -10
  22. package/src/core/runtimeConfig.js +59 -83
  23. package/src/core/uninstall.js +8 -1
  24. package/src/decision/registry.js +33 -16
  25. package/src/decision/runtimeDecide.js +5 -5
  26. package/src/index/buildIndex.js +287 -6
  27. package/src/index/languageTools.js +100 -0
  28. package/src/index/paths.js +2 -1
  29. package/src/index/queryIndex.js +174 -10
  30. package/src/index/routeResolver.js +2 -2
  31. package/src/index/semanticLane.js +269 -0
  32. package/src/learning/lessonClass.js +110 -0
  33. package/src/learning/patternProposals.js +111 -7
  34. package/src/learning/reeditMiner.js +104 -0
  35. package/src/learning/selfImprove.js +9 -1
  36. package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
  37. package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
  38. package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
  39. package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
  40. package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
  41. package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
  42. package/template_project/.claude/ukit/index/query-index.mjs +9 -1
  43. package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
  44. package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
  45. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
  46. package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
  47. package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
  48. package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
  49. package/template_project/ukit/storage/config.json +19 -34
  50. package/manifests/instructionRules.yaml +0 -442
  51. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
  52. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
  53. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
  54. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
  55. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
  56. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
  57. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
  58. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
  59. package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
  60. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
  61. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
  62. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
  63. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
  64. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
  65. package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
  66. package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
  67. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
  68. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
@@ -0,0 +1,269 @@
1
+ // Real-embedding lane for `ukit index query` (opt-in; no dependency; global fetch only).
2
+ //
3
+ // PRIVACY: building vectors SENDS FILE TEXT (path, symbol names, leading comment and,
4
+ // at most ~240 characters per file) to the endpoint in
5
+ // UKIT_INDEX_EMBED_URL, and queries send the question text. With no URL configured nothing
6
+ // here runs and nothing leaves the machine. Sensitive paths (.env, keys, credentials ...)
7
+ // are never embedded; only files already present in the index are considered.
8
+ //
9
+ // Byte-identical mirror: template_project/.claude/ukit/index/lib/semantic-lane.mjs
10
+ // (tests/index/semanticLane.test.js enforces it). Keep this file dependency-free.
11
+ //
12
+ // Env: UKIT_INDEX_EMBED_URL (base URL ending in /v1), UKIT_INDEX_EMBED_MODEL,
13
+ // UKIT_INDEX_EMBED_KEY (optional Bearer).
14
+ // Build is an explicit command (embed-index.mjs), never part of index build/refresh, never in
15
+ // a hook. Query use is reachable only through queryCodeIndex({ semantic: true }), which only
16
+ // the CLI query path passes.
17
+ import fs from 'node:fs/promises';
18
+ import path from 'node:path';
19
+ import crypto from 'node:crypto';
20
+
21
+ export const SEMANTIC_MANIFEST = 'semantic.json';
22
+ export const SEMANTIC_BIN = 'semantic.bin';
23
+ // A CLI query waits at most this long for the embedding of the question. Measured on the
24
+ // reference endpoint: warm question embedding ~0.45 s, 64 descriptors ~24 s (serial service).
25
+ export const QUERY_TIMEOUT_MS = 1500;
26
+ // One descriptor per file, ~60 tokens. Endpoint throughput is ~170-200 tokens/s and serial, so
27
+ // file bodies (>1 h for 2.4k files) are not embedded; descriptors build in minutes.
28
+ export const DOC_CHARS = 240;
29
+ const NAMES_CHARS = 90;
30
+ const LEAD_CHARS = 100;
31
+ const QUERY_INSTRUCTION = 'Instruct: Given a question about a code repository, retrieve the source file that answers it\nQuery: ';
32
+
33
+ export function embedConfig(env = process.env) {
34
+ // Accepts the base (.../v1) or the full endpoint (.../v1/embeddings).
35
+ const url = String(env.UKIT_INDEX_EMBED_URL ?? '').trim().replace(/\/+$/, '').replace(/\/embeddings$/i, '');
36
+ if (!url) return null;
37
+ return { url, model: String(env.UKIT_INDEX_EMBED_MODEL ?? '').trim() || 'unic-embedding', key: String(env.UKIT_INDEX_EMBED_KEY ?? '').trim() };
38
+ }
39
+
40
+ function isSensitive(value) {
41
+ const segments = String(value ?? '').replace(/\\/g, '/').split('/');
42
+ const base = segments[segments.length - 1] ?? '';
43
+ return base.toLowerCase().startsWith('.env')
44
+ || /\.(?:pem|key|p12|pfx|jks|keystore)$/i.test(base)
45
+ || /^(?:credentials?|\.npmrc|\.netrc|\.pypirc)(?:\.[^/]*)?$/i.test(base)
46
+ || /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.[^/]*)?$/i.test(base)
47
+ || segments.includes('.ssh') || segments.includes('.aws') || segments.includes('.gnupg');
48
+ }
49
+
50
+ const sha = (text) => crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
51
+
52
+ // POST /embeddings; returns arrays in input order or throws. Validates shape.
53
+ export async function embedTexts(texts, cfg, { timeoutMs = 60000 } = {}) {
54
+ const headers = { 'content-type': 'application/json' };
55
+ if (cfg.key) headers.authorization = `Bearer ${cfg.key}`;
56
+ const response = await fetch(`${cfg.url}/embeddings`, {
57
+ method: 'POST',
58
+ headers,
59
+ body: JSON.stringify({ model: cfg.model, input: texts }),
60
+ signal: AbortSignal.timeout(timeoutMs),
61
+ });
62
+ if (!response.ok) throw new Error(`embedding endpoint HTTP ${response.status}`);
63
+ const data = (await response.json())?.data;
64
+ if (!Array.isArray(data) || data.length !== texts.length) throw new Error('embedding response shape mismatch');
65
+ const rows = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding);
66
+ const dims = rows[0]?.length;
67
+ if (!dims || rows.some((r) => !Array.isArray(r) || r.length !== dims || r.some((x) => !Number.isFinite(x)))) {
68
+ throw new Error('embedding response vectors invalid');
69
+ }
70
+ return rows;
71
+ }
72
+
73
+ function normalizeTo(vec, dims) {
74
+ const out = vec.slice(0, dims);
75
+ let n = 0;
76
+ for (const x of out) n += x * x;
77
+ n = Math.sqrt(n) || 1;
78
+ return out.map((x) => x / n);
79
+ }
80
+
81
+ // Descriptor for one file: path + exported/declared symbol names + leading doc comment,
82
+ // capped at DOC_CHARS. The path is never truncated; names and comment share the remainder.
83
+ export function buildDocumentText({ filePath, symbols = [], content = '' }) {
84
+ const names = [...new Set(symbols)].join(' ').slice(0, NAMES_CHARS);
85
+ const lines = String(content).split('\n');
86
+ const lead = [];
87
+ for (const line of lines.slice(0, 80)) {
88
+ const t = line.trim();
89
+ if (!t || t.startsWith('#!') || t === '---' || t.startsWith("'use ")) { if (lead.length) break; continue; }
90
+ if (/^(\/\/|\/\*|\*|#|<!--|"""|--)/.test(t)) lead.push(t.replace(/^(\/\/+|\/\*+|\*+\/?|#+|<!--|-->|"""|--)\s?/, ''));
91
+ else break;
92
+ }
93
+ const leading = (lead.length ? lead.join(' ') : '').slice(0, LEAD_CHARS);
94
+ const text = `${filePath}\n${names}\n${leading}`;
95
+ return text.length > DOC_CHARS ? text.slice(0, DOC_CHARS) : text;
96
+ }
97
+
98
+ async function readJson(file) {
99
+ try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; }
100
+ }
101
+
102
+ async function loadStore(indexDir) {
103
+ const manifest = await readJson(path.join(indexDir, SEMANTIC_MANIFEST));
104
+ if (!manifest || manifest.v !== 1 || !Array.isArray(manifest.order)) return null;
105
+ let bin;
106
+ try { bin = await fs.readFile(path.join(indexDir, SEMANTIC_BIN)); } catch { return null; }
107
+ const rec = 4 + manifest.dims;
108
+ if (bin.length !== rec * manifest.order.length) return null;
109
+ return { manifest, bin, rec };
110
+ }
111
+
112
+ // Explicit build step. Embeds only documents whose hash (document text + model) is not in the
113
+ // existing store. Returns stats. Never called from index build/refresh or hooks.
114
+ export async function buildSemanticVectors({
115
+ rootDir = process.cwd(), env = process.env, dims = 0,
116
+ batchSize = 64, onProgress = null,
117
+ } = {}) {
118
+ const cfg = embedConfig(env);
119
+ if (!cfg) throw new Error('UKIT_INDEX_EMBED_URL is not set; semantic vectors are opt-in');
120
+ const root = path.resolve(rootDir);
121
+ const indexDir = path.join(root, '.cache', 'index');
122
+ const filesArtifact = await readJson(path.join(indexDir, 'files.json'));
123
+ const symbolsArtifact = await readJson(path.join(indexDir, 'symbols.json'));
124
+ if (!filesArtifact?.items) throw new Error('index missing; run the index build first');
125
+ const symbolsByFile = new Map();
126
+ for (const s of symbolsArtifact?.items ?? []) {
127
+ if (!symbolsByFile.has(s.filePath)) symbolsByFile.set(s.filePath, []);
128
+ symbolsByFile.get(s.filePath).push(s.name);
129
+ }
130
+ // Reuse vectors from a store built with the same model and truncation.
131
+ const old = await loadStore(indexDir);
132
+ const reusable = old && old.manifest.model === cfg.model
133
+ && (!dims || old.manifest.dims === dims) ? old : null;
134
+ const oldByHash = new Map();
135
+ if (reusable) reusable.manifest.order.forEach(([, h], i) => oldByHash.set(h, reusable.bin.subarray(i * reusable.rec, (i + 1) * reusable.rec)));
136
+
137
+ const docs = [];
138
+ for (const item of filesArtifact.items) {
139
+ const filePath = item.filePath;
140
+ if (typeof filePath !== 'string' || isSensitive(filePath)) continue;
141
+ let content = '';
142
+ try { content = await fs.readFile(path.join(root, filePath), 'utf8'); } catch { continue; }
143
+ const text = buildDocumentText({ filePath, symbols: symbolsByFile.get(filePath) ?? [], content });
144
+ docs.push({ filePath, text, hash: sha(`${cfg.model}\0${text}`) });
145
+ }
146
+ const missing = [...new Map(docs.filter((d) => !oldByHash.has(d.hash)).map((d) => [d.hash, d])).values()];
147
+ const fresh = new Map();
148
+ let outDims = dims || reusable?.manifest.dims || 0;
149
+ // Requests are strictly sequential: the embedding service is serial, so parallel requests
150
+ // only queue. Partial progress is saved: a failed batch leaves its files without vectors and
151
+ // the error is rethrown after the store is written, so a rerun only embeds what is missing.
152
+ const failures = [];
153
+ try {
154
+ for (let i = 0; i < missing.length; i += batchSize) {
155
+ const batch = missing.slice(i, i + batchSize);
156
+ const rows = await embedTexts(batch.map((d) => d.text), cfg, { timeoutMs: 180000 });
157
+ if (!outDims) outDims = rows[0].length;
158
+ if (rows[0].length < outDims) throw new Error('endpoint returned fewer dimensions than requested');
159
+ batch.forEach((d, k) => fresh.set(d.hash, rows[k]));
160
+ onProgress?.({ done: fresh.size, total: missing.length });
161
+ }
162
+ } catch (e) { failures.push(e); }
163
+ if (!outDims) outDims = old?.manifest.dims ?? 0;
164
+ if (!outDims && docs.length) throw new Error('no vectors produced');
165
+
166
+ const rec = 4 + outDims;
167
+ const order = docs.filter((d) => fresh.has(d.hash) || oldByHash.has(d.hash)).map((d) => [d.filePath, d.hash]);
168
+ const bin = Buffer.alloc(rec * order.length);
169
+ order.forEach(([, hash], i) => {
170
+ const off = i * rec;
171
+ const prior = oldByHash.get(hash);
172
+ if (prior) { prior.copy(bin, off); return; }
173
+ const v = normalizeTo(fresh.get(hash), outDims);
174
+ const maxAbs = Math.max(...v.map(Math.abs)) || 1;
175
+ bin.writeFloatLE(maxAbs / 127, off);
176
+ for (let k = 0; k < outDims; k += 1) bin.writeInt8(Math.max(-127, Math.min(127, Math.round(v[k] / maxAbs * 127))), off + 4 + k);
177
+ });
178
+ const manifest = { v: 1, model: cfg.model, dims: outDims, createdAt: new Date().toISOString(), order };
179
+ const tmp = `.tmp-${process.pid}`;
180
+ await fs.writeFile(path.join(indexDir, SEMANTIC_BIN + tmp), bin);
181
+ await fs.writeFile(path.join(indexDir, SEMANTIC_MANIFEST + tmp), JSON.stringify(manifest));
182
+ await fs.rename(path.join(indexDir, SEMANTIC_BIN + tmp), path.join(indexDir, SEMANTIC_BIN));
183
+ await fs.rename(path.join(indexDir, SEMANTIC_MANIFEST + tmp), path.join(indexDir, SEMANTIC_MANIFEST));
184
+ if (failures.length) throw new Error(`partial store saved (${order.length}/${docs.length} files); rerun to resume: ${failures[0]?.message ?? failures[0]}`);
185
+ return { files: docs.length, embedded: fresh.size, reused: docs.length - missing.length, dims: outDims, bytes: bin.length, model: cfg.model };
186
+ }
187
+
188
+ // Cache-key part: changes when the vector store is built, rebuilt or removed, so a result cached
189
+ // before `embed-index` ran never outlives the vectors. Never throws.
190
+ export async function semanticStoreSignature(rootDir) {
191
+ try {
192
+ const stat = await fs.stat(path.join(path.resolve(rootDir), '.cache', 'index', SEMANTIC_MANIFEST));
193
+ return `${stat.mtimeMs}:${stat.size}`;
194
+ } catch {
195
+ return 'none';
196
+ }
197
+ }
198
+
199
+ const STORE_CACHE = new Map();
200
+
201
+ // Query-side ranking. Returns [{ filePath, similarity }] (best first) or null on ANY problem
202
+ // (lane off, no vectors, model/dimension mismatch, timeout, HTTP error). Never throws.
203
+ export async function semanticCandidates({
204
+ rootDir, query, env = process.env, timeoutMs = QUERY_TIMEOUT_MS, limit = 100,
205
+ queryInstruction = QUERY_INSTRUCTION, accept = () => true, weight = () => 1,
206
+ } = {}) {
207
+ try {
208
+ const cfg = embedConfig(env);
209
+ if (!cfg) return null;
210
+ const indexDir = path.join(path.resolve(rootDir), '.cache', 'index');
211
+ const stat = await fs.stat(path.join(indexDir, SEMANTIC_MANIFEST));
212
+ const key = `${indexDir}:${stat.mtimeMs}:${stat.size}`;
213
+ let store = STORE_CACHE.get(key);
214
+ if (!store) { store = await loadStore(indexDir); if (!store) return null; STORE_CACHE.clear(); STORE_CACHE.set(key, store); }
215
+ const { manifest, bin, rec } = store;
216
+ if (manifest.model !== cfg.model) return null;
217
+ const [raw] = await embedTexts([`${queryInstruction}${query}`], cfg, { timeoutMs });
218
+ if (raw.length < manifest.dims) return null;
219
+ const q = normalizeTo(raw, manifest.dims);
220
+ const scored = [];
221
+ for (let i = 0; i < manifest.order.length; i += 1) {
222
+ const filePath = manifest.order[i][0];
223
+ if (!accept(filePath)) continue;
224
+ const off = i * rec;
225
+ let dot = 0;
226
+ for (let k = 0; k < manifest.dims; k += 1) dot += q[k] * bin.readInt8(off + 4 + k);
227
+ scored.push({ filePath, similarity: dot * bin.readFloatLE(off) * weight(filePath) });
228
+ }
229
+ scored.sort((a, b) => b.similarity - a.similarity || a.filePath.localeCompare(b.filePath));
230
+ return scored.slice(0, limit);
231
+ } catch {
232
+ return null;
233
+ }
234
+ }
235
+
236
+ // Plain-language gate: >= 4 words, no path separators or code punctuation, no quotes/colons,
237
+ // no trailing period (a finished statement is error-message shaped), no error-leading word, and
238
+ // no identifier-shaped token (camelCase, snake_case, kebab with digits, dotted name, file
239
+ // extension, token containing a digit, ALL-CAPS code such as ENOENT). Non-ASCII text
240
+ // (Vietnamese) and ordinary lowercase sentences pass; everything else keeps the lexical result.
241
+ const ERROR_LEADS = /^(?:error|errors|failed|failure|cannot|can't|unable|exception|invalid|unexpected|uncaught|warning|fatal|missing|undefined|null|typeerror|referenceerror)\b/i;
242
+ function identifierShaped(token) {
243
+ const t = token.replace(/^[^\w]+|[^\w]+$/g, '');
244
+ if (!t) return false;
245
+ return /[a-z][A-Z]/.test(t) || /_/.test(t) || /\d/.test(t) || /\w\.\w/.test(t) || /\w-\w*\d/.test(t)
246
+ || /^[A-Z]{3,}$/.test(t) || /^[A-Z][a-z]+[A-Z]/.test(t) || /\.(?:js|mjs|cjs|ts|tsx|jsx|json|md|py|go|rs|java|rb|yml|yaml|sh)\b/i.test(token);
247
+ }
248
+ export function looksLikeQuestion(query) {
249
+ const q = String(query ?? '').trim();
250
+ const words = q.split(/\s+/).filter(Boolean);
251
+ if (words.length < 4 || /[\\/]|[(){}\[\]=;<>@#$%^&*|~`'"]|::|=>|:/.test(q.replace(/[^\x00-\x7f]/g, ''))) return false;
252
+ if (/[^\x20-\x7e]/.test(q)) return true;
253
+ if (/\.\s*$/.test(q) || ERROR_LEADS.test(q) || words.some(identifierShaped)) return false;
254
+ return true;
255
+ }
256
+
257
+ // Combine: weighted RRF over the lexical ranking and the semantic list. Semantic-only files
258
+ // are injected with a floor score. Returns null (caller keeps lexical) when no candidates.
259
+ export function fuseSemantic({ ranked, candidates, rrfK = 5, weight = 1, depth = 100 }) {
260
+ if (!candidates?.length) return null;
261
+ const rrf = new Map();
262
+ ranked.slice(0, depth).forEach((e, i) => rrf.set(e.filePath, 1 / (rrfK + i + 1)));
263
+ candidates.slice(0, depth).forEach((c, i) => rrf.set(c.filePath, (rrf.get(c.filePath) ?? 0) + weight / (rrfK + i + 1)));
264
+ const byPath = new Map(ranked.map((e) => [e.filePath, e]));
265
+ const floor = ranked.length ? ranked[ranked.length - 1].score : 0.01;
266
+ return [...rrf.entries()]
267
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
268
+ .map(([filePath]) => byPath.get(filePath) ?? { filePath, score: Math.min(0.01, floor), reasons: ['semantic:vector'], tests: [] });
269
+ }
@@ -3,6 +3,7 @@ import fs from 'node:fs/promises';
3
3
  import path from 'node:path';
4
4
  import crypto from 'node:crypto';
5
5
  import * as indexCore from './lib/index-core.mjs';
6
+ import { embedConfig, semanticStoreSignature } from './lib/semantic-lane.mjs';
6
7
  import {
7
8
  DEFAULT_RECENT_CACHE_MAX_ENTRIES,
8
9
  readRecentCacheEntry,
@@ -42,6 +43,8 @@ if (!query && !symbolQuery) {
42
43
  query,
43
44
  limit: effectiveLimit,
44
45
  showOutline,
46
+ // Lane on: key on the vector store's mtime+size (or "none") so building vectors invalidates.
47
+ semantic: embedConfig() ? `1:${await semanticStoreSignature(rootDir)}` : 0,
45
48
  });
46
49
  const cached = await readRecentCacheEntry(cachePath, requestKey, {
47
50
  maxEntries: DEFAULT_RECENT_CACHE_MAX_ENTRIES,
@@ -52,11 +55,14 @@ if (!query && !symbolQuery) {
52
55
  rootDir,
53
56
  query,
54
57
  limit: effectiveLimit,
58
+ // CLI-only: the opt-in embedding lane (needs UKIT_INDEX_EMBED_URL); never passed from hooks.
59
+ semantic: true,
55
60
  });
56
61
  // The status rides on the array as a non-enumerable prop, so the cache stores it explicitly.
57
62
  const contentLaneTruncated = cached ? (cached.contentLaneTruncated ?? []) : [...(results.contentLaneTruncated ?? [])];
58
63
 
59
- if (!cached) {
64
+ // A lane-on query that fell back to keywords (timeout/HTTP error/no vectors) is never persisted.
65
+ if (!cached && !results.semanticFellBack) {
60
66
  await writeRecentCacheEntry(cachePath, {
61
67
  requestKey,
62
68
  updatedAt: Date.now(),
@@ -244,12 +250,14 @@ function buildQueryRequestKey({
244
250
  query = '',
245
251
  limit = 5,
246
252
  showOutline = true,
253
+ semantic = 0,
247
254
  } = {}) {
248
255
  return buildCompactMachineKey('query-v2', {
249
256
  indexGeneratedAtMs,
250
257
  query: String(query || '').trim(),
251
258
  limit: Number(limit),
252
259
  showOutline: Boolean(showOutline),
260
+ ...(semantic ? { semantic } : {}),
253
261
  });
254
262
  }
255
263
 
@@ -74,6 +74,17 @@ if (changedFiles.length > 0 && !force) {
74
74
  }
75
75
  }
76
76
 
77
+ // 3.5.5: keep existing topic pages current. Rebuilds only pages that already exist
78
+ // and are stale (agent block and cited notes preserved); never creates pages; never
79
+ // fails the refresh. Runs here (git hook, backgrounded + deadline-armed), not in a hook path.
80
+ try {
81
+ if (process.env.UKIT_MEMORY_TOPICS !== '0') {
82
+ const { refreshTopics } = await import('../runtime/memory-topics.mjs');
83
+ const res = await refreshTopics(rootDir);
84
+ if (res.rebuilt.length > 0) console.log(`topics-rebuilt: ${res.rebuilt.length}`);
85
+ }
86
+ } catch { /* topic pages are optional */ }
87
+
77
88
  function getRootDir(argv) {
78
89
  const value = readFlagValue(argv, '--root');
79
90
  return value ? path.resolve(value) : process.cwd();
@@ -421,8 +421,8 @@ export const ROUTE_SHADOW_QUESTIONS = Object.freeze([
421
421
  // capability.impact.v1 — noul kind; no threshold baseline exists on
422
422
  // routeSummary (capabilityPolicy.recommended is unpopulated upstream), so
423
423
  // the question has no comparison value.
424
- // learn.candidate-class.v1 — reserved for the memoryV2 `decision` plane
425
- // (memoryFlags.js), not the route boundary.
424
+ // learn.candidate-class.v1 — the `learn` family classifies mined lesson candidates
425
+ // (gated by decisionPlane.families.learn.stage, off by default); it is not a route-boundary question.
426
426
  ]);
427
427
 
428
428
  // Only families whose resolved stage is not 'off' get a question.
@@ -6,6 +6,7 @@ import {
6
6
  demoteByValidity, dropSuperseded, fuseRanked, loadMarkdownDocs, normalizeMemorySources, rankDocs, STALE_MARKER,
7
7
  } from './memory-recall.mjs';
8
8
  import { withValidity } from './memory-validity.mjs';
9
+ import { findTopicFor } from './memory-topics.mjs';
9
10
  import { eligible, resolveProjectIdentityMirror } from './memory-policy.mjs';
10
11
  import { resolveMemoryStage } from './memory-flags.mjs';
11
12
 
@@ -113,10 +114,12 @@ export function recallForTarget({ target, neighbours, docs, query = '', limit =
113
114
  const flat = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
114
115
  const KIND_LABEL = { twin: 'identical at last index', declared: 'declared', drifted: 'differs now' };
115
116
 
116
- export function renderEditHint({ target, neighbours, memory = [], maxChars = 600 } = {}) {
117
+ export function renderEditHint({ target, neighbours, memory = [], maxChars = 600, topic = null } = {}) {
117
118
  try {
118
119
  const n = neighbours ?? {};
119
120
  const lines = [];
121
+ // 3.5.5: pointer to the module's topic page (path only; the page is read on request).
122
+ if (topic?.rel) lines.push(`module page: ${flat(topic.rel)} (read it for the module overview; \`ukit memory topic list\` shows if it is stale)`);
120
123
  const mirrors = (Array.isArray(n.mirrors) ? n.mirrors : []).map((m) => (typeof m === 'string' ? { path: m, kind: 'declared' } : m))
121
124
  .filter((m) => m?.path).sort((a, b) => String(a.path).localeCompare(String(b.path))).slice(0, 2);
122
125
  for (const m of mirrors) lines.push(`mirror (${KIND_LABEL[m.kind] ?? 'declared'}): ${flat(m.path)}`);
@@ -250,7 +253,11 @@ export async function editHintFor({
250
253
  const drop = new Set(Array.isArray(exclude?.memoryIds) ? exclude.memoryIds : []);
251
254
  const docs = [...records, ...markdown].filter((d) => !drop.has(d.id));
252
255
  const { ranked } = recallForTarget({ target: rel, neighbours, docs, query });
253
- const text = renderEditHint({ target: rel, neighbours, memory: ranked });
256
+ // Topic pointer: one header read per ancestor dir, async, never hashes the module;
257
+ // skipped when the memory budget already missed so the graph part stays fast.
258
+ const topic = memoryMissed || cfg?.memory?.topics?.hint === false || signal?.aborted
259
+ ? null : await findTopicFor(projectRoot, rel, { checkStale: false });
260
+ const text = renderEditHint({ target: rel, neighbours, memory: ranked, topic });
254
261
  if (!text) return { text: '', neighbours, memoryIds: [], memoryMissed };
255
262
  return { text, neighbours, memoryIds: ranked.slice(0, 2).map((e) => e.doc.id), memoryMissed };
256
263
  } catch {
@@ -80,6 +80,11 @@ export function eligible(record, context = {}, policy = {}) {
80
80
  if (NON_AUTHORITY_TRUST_TIERS.includes(record.trust_tier)) {
81
81
  return deny('non-authority-trust-tier');
82
82
  }
83
+ // Records written before candidates got trust_tier 'candidate' sit as
84
+ // active/verified with meta.legacyStatus 'pending' — still unreviewed.
85
+ if (record.meta?.legacyStatus === 'pending') {
86
+ return deny('pending-approval');
87
+ }
83
88
 
84
89
  if (record.scope === 'user') {
85
90
  if (context.includeUser !== true) {
@@ -155,9 +155,17 @@ function stem(token) {
155
155
 
156
156
  // Tokens of `text`: whole tokens plus their separator-split parts
157
157
  // (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
158
- export function tokenize(text) {
158
+ // A camelCase identifier also yields its words (`deriveDeterministicBucket`
159
+ // -> derive, deterministic, bucket); the whole identifier stays a token too.
160
+ const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
161
+ const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
162
+
163
+ // Query side only (`{ camel: true }`): in a long note the split words of its many
164
+ // identifiers (`postEditReviewPolicy`) would add common-word matches.
165
+ export function tokenize(text, { camel = false } = {}) {
159
166
  const out = [];
160
- for (const raw of normalize(text).split(/\s+/)) {
167
+ const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
168
+ for (const raw of normalize(source).split(/\s+/)) {
161
169
  if (!raw) continue;
162
170
  const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
163
171
  const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
@@ -169,6 +177,21 @@ export function tokenize(text) {
169
177
  return out;
170
178
  }
171
179
 
180
+ // File names in a query (`name.ext`): a query that names a file is only
181
+ // answered by a note that names it too and covers the rest of the question.
182
+ const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
183
+ const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
184
+ const FILE_REST_COVERAGE = 0.3;
185
+ const FILE_REST_MIN_TOKENS = 2;
186
+ function queryFileSpec(query) {
187
+ if (typeof query !== 'string') return null;
188
+ const files = query.match(QUERY_FILE_RE);
189
+ if (!files) return null;
190
+ const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
191
+ const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
192
+ return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
193
+ }
194
+
172
195
  function episodeBody(text) {
173
196
  return String(text ?? '')
174
197
  .replace(/^\s*Session\s+\S+?:\s*/i, '')
@@ -272,7 +295,7 @@ function idf(totalDocs, docFreq) {
272
295
  // `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
273
296
  // `query` is a string or a pre-tokenized array. Never throws.
274
297
  export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
275
- const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
298
+ const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
276
299
  const empty = {
277
300
  ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
278
301
  };
@@ -302,6 +325,10 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
302
325
  let queryMass = 0;
303
326
  for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
304
327
 
328
+ const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
329
+ let restMass = 0;
330
+ if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
331
+
305
332
  const scored = [];
306
333
  let candidateCount = 0;
307
334
  for (const { doc, tf, length } of stats) {
@@ -324,6 +351,15 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
324
351
  // clear the absolute floor above while the query's real subject is absent.
325
352
  // It must also cover a fraction of the query's own IDF mass.
326
353
  if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
354
+ if (fileSpec) {
355
+ let named = false;
356
+ for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
357
+ let restHit = 0;
358
+ for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
359
+ // One leftover word ("what does X check") is too little to judge coverage by.
360
+ const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
361
+ if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
362
+ }
327
363
  scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
328
364
  }
329
365
  // Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved