@ngockhoale/ukit 3.5.4 → 3.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/manifests/engineConformance.yaml +0 -13
- package/manifests/platform.full.yaml +25 -0
- package/package.json +3 -9
- package/src/cli/commands/indexTools.js +1 -0
- package/src/cli/commands/memory.js +44 -4
- package/src/core/applyPlan.js +25 -72
- package/src/core/diffPlan.js +38 -9
- package/src/core/managedTemplateMerge.js +146 -0
- package/src/core/memory/memoryCheckRun.js +13 -1
- package/src/core/memory/memoryFlags.js +5 -6
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/policy.js +5 -0
- package/src/core/memory/recallEngine.js +39 -3
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/store.js +17 -0
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/metadata.js +12 -8
- package/src/core/runInstallPipeline.js +16 -10
- package/src/core/runtimeConfig.js +59 -83
- package/src/core/uninstall.js +8 -1
- package/src/decision/registry.js +33 -16
- package/src/decision/runtimeDecide.js +5 -5
- package/src/index/buildIndex.js +287 -6
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +2 -1
- package/src/index/queryIndex.js +174 -10
- package/src/index/routeResolver.js +2 -2
- package/src/index/semanticLane.js +269 -0
- package/src/learning/lessonClass.js +110 -0
- package/src/learning/patternProposals.js +111 -7
- package/src/learning/reeditMiner.js +104 -0
- package/src/learning/selfImprove.js +9 -1
- package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
- package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
- package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
- package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
- package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
- package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
- package/template_project/.claude/ukit/index/query-index.mjs +9 -1
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/ukit/storage/config.json +19 -34
- package/manifests/instructionRules.yaml +0 -442
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// Real-embedding lane for `ukit index query` (opt-in; no dependency; global fetch only).
|
|
2
|
+
//
|
|
3
|
+
// PRIVACY: building vectors SENDS FILE TEXT (path, symbol names, leading comment and,
|
|
4
|
+
// at most ~240 characters per file) to the endpoint in
|
|
5
|
+
// UKIT_INDEX_EMBED_URL, and queries send the question text. With no URL configured nothing
|
|
6
|
+
// here runs and nothing leaves the machine. Sensitive paths (.env, keys, credentials ...)
|
|
7
|
+
// are never embedded; only files already present in the index are considered.
|
|
8
|
+
//
|
|
9
|
+
// Byte-identical mirror: template_project/.claude/ukit/index/lib/semantic-lane.mjs
|
|
10
|
+
// (tests/index/semanticLane.test.js enforces it). Keep this file dependency-free.
|
|
11
|
+
//
|
|
12
|
+
// Env: UKIT_INDEX_EMBED_URL (base URL ending in /v1), UKIT_INDEX_EMBED_MODEL,
|
|
13
|
+
// UKIT_INDEX_EMBED_KEY (optional Bearer).
|
|
14
|
+
// Build is an explicit command (embed-index.mjs), never part of index build/refresh, never in
|
|
15
|
+
// a hook. Query use is reachable only through queryCodeIndex({ semantic: true }), which only
|
|
16
|
+
// the CLI query path passes.
|
|
17
|
+
import fs from 'node:fs/promises';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import crypto from 'node:crypto';
|
|
20
|
+
|
|
21
|
+
export const SEMANTIC_MANIFEST = 'semantic.json';
|
|
22
|
+
export const SEMANTIC_BIN = 'semantic.bin';
|
|
23
|
+
// A CLI query waits at most this long for the embedding of the question. Measured on the
|
|
24
|
+
// reference endpoint: warm question embedding ~0.45 s, 64 descriptors ~24 s (serial service).
|
|
25
|
+
export const QUERY_TIMEOUT_MS = 1500;
|
|
26
|
+
// One descriptor per file, ~60 tokens. Endpoint throughput is ~170-200 tokens/s and serial, so
|
|
27
|
+
// file bodies (>1 h for 2.4k files) are not embedded; descriptors build in minutes.
|
|
28
|
+
export const DOC_CHARS = 240;
|
|
29
|
+
const NAMES_CHARS = 90;
|
|
30
|
+
const LEAD_CHARS = 100;
|
|
31
|
+
const QUERY_INSTRUCTION = 'Instruct: Given a question about a code repository, retrieve the source file that answers it\nQuery: ';
|
|
32
|
+
|
|
33
|
+
export function embedConfig(env = process.env) {
|
|
34
|
+
// Accepts the base (.../v1) or the full endpoint (.../v1/embeddings).
|
|
35
|
+
const url = String(env.UKIT_INDEX_EMBED_URL ?? '').trim().replace(/\/+$/, '').replace(/\/embeddings$/i, '');
|
|
36
|
+
if (!url) return null;
|
|
37
|
+
return { url, model: String(env.UKIT_INDEX_EMBED_MODEL ?? '').trim() || 'unic-embedding', key: String(env.UKIT_INDEX_EMBED_KEY ?? '').trim() };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function isSensitive(value) {
|
|
41
|
+
const segments = String(value ?? '').replace(/\\/g, '/').split('/');
|
|
42
|
+
const base = segments[segments.length - 1] ?? '';
|
|
43
|
+
return base.toLowerCase().startsWith('.env')
|
|
44
|
+
|| /\.(?:pem|key|p12|pfx|jks|keystore)$/i.test(base)
|
|
45
|
+
|| /^(?:credentials?|\.npmrc|\.netrc|\.pypirc)(?:\.[^/]*)?$/i.test(base)
|
|
46
|
+
|| /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.[^/]*)?$/i.test(base)
|
|
47
|
+
|| segments.includes('.ssh') || segments.includes('.aws') || segments.includes('.gnupg');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const sha = (text) => crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
|
|
51
|
+
|
|
52
|
+
// POST /embeddings; returns arrays in input order or throws. Validates shape.
|
|
53
|
+
export async function embedTexts(texts, cfg, { timeoutMs = 60000 } = {}) {
|
|
54
|
+
const headers = { 'content-type': 'application/json' };
|
|
55
|
+
if (cfg.key) headers.authorization = `Bearer ${cfg.key}`;
|
|
56
|
+
const response = await fetch(`${cfg.url}/embeddings`, {
|
|
57
|
+
method: 'POST',
|
|
58
|
+
headers,
|
|
59
|
+
body: JSON.stringify({ model: cfg.model, input: texts }),
|
|
60
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
61
|
+
});
|
|
62
|
+
if (!response.ok) throw new Error(`embedding endpoint HTTP ${response.status}`);
|
|
63
|
+
const data = (await response.json())?.data;
|
|
64
|
+
if (!Array.isArray(data) || data.length !== texts.length) throw new Error('embedding response shape mismatch');
|
|
65
|
+
const rows = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding);
|
|
66
|
+
const dims = rows[0]?.length;
|
|
67
|
+
if (!dims || rows.some((r) => !Array.isArray(r) || r.length !== dims || r.some((x) => !Number.isFinite(x)))) {
|
|
68
|
+
throw new Error('embedding response vectors invalid');
|
|
69
|
+
}
|
|
70
|
+
return rows;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function normalizeTo(vec, dims) {
|
|
74
|
+
const out = vec.slice(0, dims);
|
|
75
|
+
let n = 0;
|
|
76
|
+
for (const x of out) n += x * x;
|
|
77
|
+
n = Math.sqrt(n) || 1;
|
|
78
|
+
return out.map((x) => x / n);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Descriptor for one file: path + exported/declared symbol names + leading doc comment,
|
|
82
|
+
// capped at DOC_CHARS. The path is never truncated; names and comment share the remainder.
|
|
83
|
+
export function buildDocumentText({ filePath, symbols = [], content = '' }) {
|
|
84
|
+
const names = [...new Set(symbols)].join(' ').slice(0, NAMES_CHARS);
|
|
85
|
+
const lines = String(content).split('\n');
|
|
86
|
+
const lead = [];
|
|
87
|
+
for (const line of lines.slice(0, 80)) {
|
|
88
|
+
const t = line.trim();
|
|
89
|
+
if (!t || t.startsWith('#!') || t === '---' || t.startsWith("'use ")) { if (lead.length) break; continue; }
|
|
90
|
+
if (/^(\/\/|\/\*|\*|#|<!--|"""|--)/.test(t)) lead.push(t.replace(/^(\/\/+|\/\*+|\*+\/?|#+|<!--|-->|"""|--)\s?/, ''));
|
|
91
|
+
else break;
|
|
92
|
+
}
|
|
93
|
+
const leading = (lead.length ? lead.join(' ') : '').slice(0, LEAD_CHARS);
|
|
94
|
+
const text = `${filePath}\n${names}\n${leading}`;
|
|
95
|
+
return text.length > DOC_CHARS ? text.slice(0, DOC_CHARS) : text;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function readJson(file) {
|
|
99
|
+
try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function loadStore(indexDir) {
|
|
103
|
+
const manifest = await readJson(path.join(indexDir, SEMANTIC_MANIFEST));
|
|
104
|
+
if (!manifest || manifest.v !== 1 || !Array.isArray(manifest.order)) return null;
|
|
105
|
+
let bin;
|
|
106
|
+
try { bin = await fs.readFile(path.join(indexDir, SEMANTIC_BIN)); } catch { return null; }
|
|
107
|
+
const rec = 4 + manifest.dims;
|
|
108
|
+
if (bin.length !== rec * manifest.order.length) return null;
|
|
109
|
+
return { manifest, bin, rec };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Explicit build step. Embeds only documents whose hash (document text + model) is not in the
|
|
113
|
+
// existing store. Returns stats. Never called from index build/refresh or hooks.
|
|
114
|
+
export async function buildSemanticVectors({
|
|
115
|
+
rootDir = process.cwd(), env = process.env, dims = 0,
|
|
116
|
+
batchSize = 64, onProgress = null,
|
|
117
|
+
} = {}) {
|
|
118
|
+
const cfg = embedConfig(env);
|
|
119
|
+
if (!cfg) throw new Error('UKIT_INDEX_EMBED_URL is not set; semantic vectors are opt-in');
|
|
120
|
+
const root = path.resolve(rootDir);
|
|
121
|
+
const indexDir = path.join(root, '.cache', 'index');
|
|
122
|
+
const filesArtifact = await readJson(path.join(indexDir, 'files.json'));
|
|
123
|
+
const symbolsArtifact = await readJson(path.join(indexDir, 'symbols.json'));
|
|
124
|
+
if (!filesArtifact?.items) throw new Error('index missing; run the index build first');
|
|
125
|
+
const symbolsByFile = new Map();
|
|
126
|
+
for (const s of symbolsArtifact?.items ?? []) {
|
|
127
|
+
if (!symbolsByFile.has(s.filePath)) symbolsByFile.set(s.filePath, []);
|
|
128
|
+
symbolsByFile.get(s.filePath).push(s.name);
|
|
129
|
+
}
|
|
130
|
+
// Reuse vectors from a store built with the same model and truncation.
|
|
131
|
+
const old = await loadStore(indexDir);
|
|
132
|
+
const reusable = old && old.manifest.model === cfg.model
|
|
133
|
+
&& (!dims || old.manifest.dims === dims) ? old : null;
|
|
134
|
+
const oldByHash = new Map();
|
|
135
|
+
if (reusable) reusable.manifest.order.forEach(([, h], i) => oldByHash.set(h, reusable.bin.subarray(i * reusable.rec, (i + 1) * reusable.rec)));
|
|
136
|
+
|
|
137
|
+
const docs = [];
|
|
138
|
+
for (const item of filesArtifact.items) {
|
|
139
|
+
const filePath = item.filePath;
|
|
140
|
+
if (typeof filePath !== 'string' || isSensitive(filePath)) continue;
|
|
141
|
+
let content = '';
|
|
142
|
+
try { content = await fs.readFile(path.join(root, filePath), 'utf8'); } catch { continue; }
|
|
143
|
+
const text = buildDocumentText({ filePath, symbols: symbolsByFile.get(filePath) ?? [], content });
|
|
144
|
+
docs.push({ filePath, text, hash: sha(`${cfg.model}\0${text}`) });
|
|
145
|
+
}
|
|
146
|
+
const missing = [...new Map(docs.filter((d) => !oldByHash.has(d.hash)).map((d) => [d.hash, d])).values()];
|
|
147
|
+
const fresh = new Map();
|
|
148
|
+
let outDims = dims || reusable?.manifest.dims || 0;
|
|
149
|
+
// Requests are strictly sequential: the embedding service is serial, so parallel requests
|
|
150
|
+
// only queue. Partial progress is saved: a failed batch leaves its files without vectors and
|
|
151
|
+
// the error is rethrown after the store is written, so a rerun only embeds what is missing.
|
|
152
|
+
const failures = [];
|
|
153
|
+
try {
|
|
154
|
+
for (let i = 0; i < missing.length; i += batchSize) {
|
|
155
|
+
const batch = missing.slice(i, i + batchSize);
|
|
156
|
+
const rows = await embedTexts(batch.map((d) => d.text), cfg, { timeoutMs: 180000 });
|
|
157
|
+
if (!outDims) outDims = rows[0].length;
|
|
158
|
+
if (rows[0].length < outDims) throw new Error('endpoint returned fewer dimensions than requested');
|
|
159
|
+
batch.forEach((d, k) => fresh.set(d.hash, rows[k]));
|
|
160
|
+
onProgress?.({ done: fresh.size, total: missing.length });
|
|
161
|
+
}
|
|
162
|
+
} catch (e) { failures.push(e); }
|
|
163
|
+
if (!outDims) outDims = old?.manifest.dims ?? 0;
|
|
164
|
+
if (!outDims && docs.length) throw new Error('no vectors produced');
|
|
165
|
+
|
|
166
|
+
const rec = 4 + outDims;
|
|
167
|
+
const order = docs.filter((d) => fresh.has(d.hash) || oldByHash.has(d.hash)).map((d) => [d.filePath, d.hash]);
|
|
168
|
+
const bin = Buffer.alloc(rec * order.length);
|
|
169
|
+
order.forEach(([, hash], i) => {
|
|
170
|
+
const off = i * rec;
|
|
171
|
+
const prior = oldByHash.get(hash);
|
|
172
|
+
if (prior) { prior.copy(bin, off); return; }
|
|
173
|
+
const v = normalizeTo(fresh.get(hash), outDims);
|
|
174
|
+
const maxAbs = Math.max(...v.map(Math.abs)) || 1;
|
|
175
|
+
bin.writeFloatLE(maxAbs / 127, off);
|
|
176
|
+
for (let k = 0; k < outDims; k += 1) bin.writeInt8(Math.max(-127, Math.min(127, Math.round(v[k] / maxAbs * 127))), off + 4 + k);
|
|
177
|
+
});
|
|
178
|
+
const manifest = { v: 1, model: cfg.model, dims: outDims, createdAt: new Date().toISOString(), order };
|
|
179
|
+
const tmp = `.tmp-${process.pid}`;
|
|
180
|
+
await fs.writeFile(path.join(indexDir, SEMANTIC_BIN + tmp), bin);
|
|
181
|
+
await fs.writeFile(path.join(indexDir, SEMANTIC_MANIFEST + tmp), JSON.stringify(manifest));
|
|
182
|
+
await fs.rename(path.join(indexDir, SEMANTIC_BIN + tmp), path.join(indexDir, SEMANTIC_BIN));
|
|
183
|
+
await fs.rename(path.join(indexDir, SEMANTIC_MANIFEST + tmp), path.join(indexDir, SEMANTIC_MANIFEST));
|
|
184
|
+
if (failures.length) throw new Error(`partial store saved (${order.length}/${docs.length} files); rerun to resume: ${failures[0]?.message ?? failures[0]}`);
|
|
185
|
+
return { files: docs.length, embedded: fresh.size, reused: docs.length - missing.length, dims: outDims, bytes: bin.length, model: cfg.model };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Cache-key part: changes when the vector store is built, rebuilt or removed, so a result cached
|
|
189
|
+
// before `embed-index` ran never outlives the vectors. Never throws.
|
|
190
|
+
export async function semanticStoreSignature(rootDir) {
|
|
191
|
+
try {
|
|
192
|
+
const stat = await fs.stat(path.join(path.resolve(rootDir), '.cache', 'index', SEMANTIC_MANIFEST));
|
|
193
|
+
return `${stat.mtimeMs}:${stat.size}`;
|
|
194
|
+
} catch {
|
|
195
|
+
return 'none';
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const STORE_CACHE = new Map();
|
|
200
|
+
|
|
201
|
+
// Query-side ranking. Returns [{ filePath, similarity }] (best first) or null on ANY problem
|
|
202
|
+
// (lane off, no vectors, model/dimension mismatch, timeout, HTTP error). Never throws.
|
|
203
|
+
export async function semanticCandidates({
|
|
204
|
+
rootDir, query, env = process.env, timeoutMs = QUERY_TIMEOUT_MS, limit = 100,
|
|
205
|
+
queryInstruction = QUERY_INSTRUCTION, accept = () => true, weight = () => 1,
|
|
206
|
+
} = {}) {
|
|
207
|
+
try {
|
|
208
|
+
const cfg = embedConfig(env);
|
|
209
|
+
if (!cfg) return null;
|
|
210
|
+
const indexDir = path.join(path.resolve(rootDir), '.cache', 'index');
|
|
211
|
+
const stat = await fs.stat(path.join(indexDir, SEMANTIC_MANIFEST));
|
|
212
|
+
const key = `${indexDir}:${stat.mtimeMs}:${stat.size}`;
|
|
213
|
+
let store = STORE_CACHE.get(key);
|
|
214
|
+
if (!store) { store = await loadStore(indexDir); if (!store) return null; STORE_CACHE.clear(); STORE_CACHE.set(key, store); }
|
|
215
|
+
const { manifest, bin, rec } = store;
|
|
216
|
+
if (manifest.model !== cfg.model) return null;
|
|
217
|
+
const [raw] = await embedTexts([`${queryInstruction}${query}`], cfg, { timeoutMs });
|
|
218
|
+
if (raw.length < manifest.dims) return null;
|
|
219
|
+
const q = normalizeTo(raw, manifest.dims);
|
|
220
|
+
const scored = [];
|
|
221
|
+
for (let i = 0; i < manifest.order.length; i += 1) {
|
|
222
|
+
const filePath = manifest.order[i][0];
|
|
223
|
+
if (!accept(filePath)) continue;
|
|
224
|
+
const off = i * rec;
|
|
225
|
+
let dot = 0;
|
|
226
|
+
for (let k = 0; k < manifest.dims; k += 1) dot += q[k] * bin.readInt8(off + 4 + k);
|
|
227
|
+
scored.push({ filePath, similarity: dot * bin.readFloatLE(off) * weight(filePath) });
|
|
228
|
+
}
|
|
229
|
+
scored.sort((a, b) => b.similarity - a.similarity || a.filePath.localeCompare(b.filePath));
|
|
230
|
+
return scored.slice(0, limit);
|
|
231
|
+
} catch {
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Plain-language gate: >= 4 words, no path separators or code punctuation, no quotes/colons,
|
|
237
|
+
// no trailing period (a finished statement is error-message shaped), no error-leading word, and
|
|
238
|
+
// no identifier-shaped token (camelCase, snake_case, kebab with digits, dotted name, file
|
|
239
|
+
// extension, token containing a digit, ALL-CAPS code such as ENOENT). Non-ASCII text
|
|
240
|
+
// (Vietnamese) and ordinary lowercase sentences pass; everything else keeps the lexical result.
|
|
241
|
+
const ERROR_LEADS = /^(?:error|errors|failed|failure|cannot|can't|unable|exception|invalid|unexpected|uncaught|warning|fatal|missing|undefined|null|typeerror|referenceerror)\b/i;
|
|
242
|
+
function identifierShaped(token) {
|
|
243
|
+
const t = token.replace(/^[^\w]+|[^\w]+$/g, '');
|
|
244
|
+
if (!t) return false;
|
|
245
|
+
return /[a-z][A-Z]/.test(t) || /_/.test(t) || /\d/.test(t) || /\w\.\w/.test(t) || /\w-\w*\d/.test(t)
|
|
246
|
+
|| /^[A-Z]{3,}$/.test(t) || /^[A-Z][a-z]+[A-Z]/.test(t) || /\.(?:js|mjs|cjs|ts|tsx|jsx|json|md|py|go|rs|java|rb|yml|yaml|sh)\b/i.test(token);
|
|
247
|
+
}
|
|
248
|
+
export function looksLikeQuestion(query) {
|
|
249
|
+
const q = String(query ?? '').trim();
|
|
250
|
+
const words = q.split(/\s+/).filter(Boolean);
|
|
251
|
+
if (words.length < 4 || /[\\/]|[(){}\[\]=;<>@#$%^&*|~`'"]|::|=>|:/.test(q.replace(/[^\x00-\x7f]/g, ''))) return false;
|
|
252
|
+
if (/[^\x20-\x7e]/.test(q)) return true;
|
|
253
|
+
if (/\.\s*$/.test(q) || ERROR_LEADS.test(q) || words.some(identifierShaped)) return false;
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Combine: weighted RRF over the lexical ranking and the semantic list. Semantic-only files
|
|
258
|
+
// are injected with a floor score. Returns null (caller keeps lexical) when no candidates.
|
|
259
|
+
export function fuseSemantic({ ranked, candidates, rrfK = 5, weight = 1, depth = 100 }) {
|
|
260
|
+
if (!candidates?.length) return null;
|
|
261
|
+
const rrf = new Map();
|
|
262
|
+
ranked.slice(0, depth).forEach((e, i) => rrf.set(e.filePath, 1 / (rrfK + i + 1)));
|
|
263
|
+
candidates.slice(0, depth).forEach((c, i) => rrf.set(c.filePath, (rrf.get(c.filePath) ?? 0) + weight / (rrfK + i + 1)));
|
|
264
|
+
const byPath = new Map(ranked.map((e) => [e.filePath, e]));
|
|
265
|
+
const floor = ranked.length ? ranked[ranked.length - 1].score : 0.01;
|
|
266
|
+
return [...rrf.entries()]
|
|
267
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
268
|
+
.map(([filePath]) => byPath.get(filePath) ?? { filePath, score: Math.min(0.01, floor), reasons: ['semantic:vector'], tests: [] });
|
|
269
|
+
}
|
|
@@ -3,6 +3,7 @@ import fs from 'node:fs/promises';
|
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import crypto from 'node:crypto';
|
|
5
5
|
import * as indexCore from './lib/index-core.mjs';
|
|
6
|
+
import { embedConfig, semanticStoreSignature } from './lib/semantic-lane.mjs';
|
|
6
7
|
import {
|
|
7
8
|
DEFAULT_RECENT_CACHE_MAX_ENTRIES,
|
|
8
9
|
readRecentCacheEntry,
|
|
@@ -42,6 +43,8 @@ if (!query && !symbolQuery) {
|
|
|
42
43
|
query,
|
|
43
44
|
limit: effectiveLimit,
|
|
44
45
|
showOutline,
|
|
46
|
+
// Lane on: key on the vector store's mtime+size (or "none") so building vectors invalidates.
|
|
47
|
+
semantic: embedConfig() ? `1:${await semanticStoreSignature(rootDir)}` : 0,
|
|
45
48
|
});
|
|
46
49
|
const cached = await readRecentCacheEntry(cachePath, requestKey, {
|
|
47
50
|
maxEntries: DEFAULT_RECENT_CACHE_MAX_ENTRIES,
|
|
@@ -52,11 +55,14 @@ if (!query && !symbolQuery) {
|
|
|
52
55
|
rootDir,
|
|
53
56
|
query,
|
|
54
57
|
limit: effectiveLimit,
|
|
58
|
+
// CLI-only: the opt-in embedding lane (needs UKIT_INDEX_EMBED_URL); never passed from hooks.
|
|
59
|
+
semantic: true,
|
|
55
60
|
});
|
|
56
61
|
// The status rides on the array as a non-enumerable prop, so the cache stores it explicitly.
|
|
57
62
|
const contentLaneTruncated = cached ? (cached.contentLaneTruncated ?? []) : [...(results.contentLaneTruncated ?? [])];
|
|
58
63
|
|
|
59
|
-
|
|
64
|
+
// A lane-on query that fell back to keywords (timeout/HTTP error/no vectors) is never persisted.
|
|
65
|
+
if (!cached && !results.semanticFellBack) {
|
|
60
66
|
await writeRecentCacheEntry(cachePath, {
|
|
61
67
|
requestKey,
|
|
62
68
|
updatedAt: Date.now(),
|
|
@@ -244,12 +250,14 @@ function buildQueryRequestKey({
|
|
|
244
250
|
query = '',
|
|
245
251
|
limit = 5,
|
|
246
252
|
showOutline = true,
|
|
253
|
+
semantic = 0,
|
|
247
254
|
} = {}) {
|
|
248
255
|
return buildCompactMachineKey('query-v2', {
|
|
249
256
|
indexGeneratedAtMs,
|
|
250
257
|
query: String(query || '').trim(),
|
|
251
258
|
limit: Number(limit),
|
|
252
259
|
showOutline: Boolean(showOutline),
|
|
260
|
+
...(semantic ? { semantic } : {}),
|
|
253
261
|
});
|
|
254
262
|
}
|
|
255
263
|
|
|
@@ -74,6 +74,17 @@ if (changedFiles.length > 0 && !force) {
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
// 3.5.5: keep existing topic pages current. Rebuilds only pages that already exist
|
|
78
|
+
// and are stale (agent block and cited notes preserved); never creates pages; never
|
|
79
|
+
// fails the refresh. Runs here (git hook, backgrounded + deadline-armed), not in a hook path.
|
|
80
|
+
try {
|
|
81
|
+
if (process.env.UKIT_MEMORY_TOPICS !== '0') {
|
|
82
|
+
const { refreshTopics } = await import('../runtime/memory-topics.mjs');
|
|
83
|
+
const res = await refreshTopics(rootDir);
|
|
84
|
+
if (res.rebuilt.length > 0) console.log(`topics-rebuilt: ${res.rebuilt.length}`);
|
|
85
|
+
}
|
|
86
|
+
} catch { /* topic pages are optional */ }
|
|
87
|
+
|
|
77
88
|
function getRootDir(argv) {
|
|
78
89
|
const value = readFlagValue(argv, '--root');
|
|
79
90
|
return value ? path.resolve(value) : process.cwd();
|
|
@@ -421,8 +421,8 @@ export const ROUTE_SHADOW_QUESTIONS = Object.freeze([
|
|
|
421
421
|
// capability.impact.v1 — noul kind; no threshold baseline exists on
|
|
422
422
|
// routeSummary (capabilityPolicy.recommended is unpopulated upstream), so
|
|
423
423
|
// the question has no comparison value.
|
|
424
|
-
// learn.candidate-class.v1 —
|
|
425
|
-
// (
|
|
424
|
+
// learn.candidate-class.v1 — the `learn` family classifies mined lesson candidates
|
|
425
|
+
// (gated by decisionPlane.families.learn.stage, off by default); it is not a route-boundary question.
|
|
426
426
|
]);
|
|
427
427
|
|
|
428
428
|
// Only families whose resolved stage is not 'off' get a question.
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
demoteByValidity, dropSuperseded, fuseRanked, loadMarkdownDocs, normalizeMemorySources, rankDocs, STALE_MARKER,
|
|
7
7
|
} from './memory-recall.mjs';
|
|
8
8
|
import { withValidity } from './memory-validity.mjs';
|
|
9
|
+
import { findTopicFor } from './memory-topics.mjs';
|
|
9
10
|
import { eligible, resolveProjectIdentityMirror } from './memory-policy.mjs';
|
|
10
11
|
import { resolveMemoryStage } from './memory-flags.mjs';
|
|
11
12
|
|
|
@@ -113,10 +114,12 @@ export function recallForTarget({ target, neighbours, docs, query = '', limit =
|
|
|
113
114
|
const flat = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
|
|
114
115
|
const KIND_LABEL = { twin: 'identical at last index', declared: 'declared', drifted: 'differs now' };
|
|
115
116
|
|
|
116
|
-
export function renderEditHint({ target, neighbours, memory = [], maxChars = 600 } = {}) {
|
|
117
|
+
export function renderEditHint({ target, neighbours, memory = [], maxChars = 600, topic = null } = {}) {
|
|
117
118
|
try {
|
|
118
119
|
const n = neighbours ?? {};
|
|
119
120
|
const lines = [];
|
|
121
|
+
// 3.5.5: pointer to the module's topic page (path only; the page is read on request).
|
|
122
|
+
if (topic?.rel) lines.push(`module page: ${flat(topic.rel)} (read it for the module overview; \`ukit memory topic list\` shows if it is stale)`);
|
|
120
123
|
const mirrors = (Array.isArray(n.mirrors) ? n.mirrors : []).map((m) => (typeof m === 'string' ? { path: m, kind: 'declared' } : m))
|
|
121
124
|
.filter((m) => m?.path).sort((a, b) => String(a.path).localeCompare(String(b.path))).slice(0, 2);
|
|
122
125
|
for (const m of mirrors) lines.push(`mirror (${KIND_LABEL[m.kind] ?? 'declared'}): ${flat(m.path)}`);
|
|
@@ -250,7 +253,11 @@ export async function editHintFor({
|
|
|
250
253
|
const drop = new Set(Array.isArray(exclude?.memoryIds) ? exclude.memoryIds : []);
|
|
251
254
|
const docs = [...records, ...markdown].filter((d) => !drop.has(d.id));
|
|
252
255
|
const { ranked } = recallForTarget({ target: rel, neighbours, docs, query });
|
|
253
|
-
|
|
256
|
+
// Topic pointer: one header read per ancestor dir, async, never hashes the module;
|
|
257
|
+
// skipped when the memory budget already missed so the graph part stays fast.
|
|
258
|
+
const topic = memoryMissed || cfg?.memory?.topics?.hint === false || signal?.aborted
|
|
259
|
+
? null : await findTopicFor(projectRoot, rel, { checkStale: false });
|
|
260
|
+
const text = renderEditHint({ target: rel, neighbours, memory: ranked, topic });
|
|
254
261
|
if (!text) return { text: '', neighbours, memoryIds: [], memoryMissed };
|
|
255
262
|
return { text, neighbours, memoryIds: ranked.slice(0, 2).map((e) => e.doc.id), memoryMissed };
|
|
256
263
|
} catch {
|
|
@@ -80,6 +80,11 @@ export function eligible(record, context = {}, policy = {}) {
|
|
|
80
80
|
if (NON_AUTHORITY_TRUST_TIERS.includes(record.trust_tier)) {
|
|
81
81
|
return deny('non-authority-trust-tier');
|
|
82
82
|
}
|
|
83
|
+
// Records written before candidates got trust_tier 'candidate' sit as
|
|
84
|
+
// active/verified with meta.legacyStatus 'pending' — still unreviewed.
|
|
85
|
+
if (record.meta?.legacyStatus === 'pending') {
|
|
86
|
+
return deny('pending-approval');
|
|
87
|
+
}
|
|
83
88
|
|
|
84
89
|
if (record.scope === 'user') {
|
|
85
90
|
if (context.includeUser !== true) {
|
|
@@ -155,9 +155,17 @@ function stem(token) {
|
|
|
155
155
|
|
|
156
156
|
// Tokens of `text`: whole tokens plus their separator-split parts
|
|
157
157
|
// (`async-lock` -> async-lock, async, lock), stemmed, stopwords removed.
|
|
158
|
-
|
|
158
|
+
// A camelCase identifier also yields its words (`deriveDeterministicBucket`
|
|
159
|
+
// -> derive, deterministic, bucket); the whole identifier stays a token too.
|
|
160
|
+
const CAMEL_RE = /\b[a-z][a-z0-9]*(?:[A-Z][a-z0-9]+)+\b/g;
|
|
161
|
+
const splitCamel = (word) => `${word} ${word.replace(/([a-z0-9])([A-Z])/g, '$1 $2')}`;
|
|
162
|
+
|
|
163
|
+
// Query side only (`{ camel: true }`): in a long note the split words of its many
|
|
164
|
+
// identifiers (`postEditReviewPolicy`) would add common-word matches.
|
|
165
|
+
export function tokenize(text, { camel = false } = {}) {
|
|
159
166
|
const out = [];
|
|
160
|
-
|
|
167
|
+
const source = camel ? String(text ?? '').replace(CAMEL_RE, splitCamel) : text;
|
|
168
|
+
for (const raw of normalize(source).split(/\s+/)) {
|
|
161
169
|
if (!raw) continue;
|
|
162
170
|
const parts = raw.split(/[./:_-]+/).filter((part) => part.length > 1 && !/^\d+$/.test(part));
|
|
163
171
|
const candidates = parts.length > 1 || (parts.length === 1 && parts[0] !== raw) ? [raw, ...parts] : [raw];
|
|
@@ -169,6 +177,21 @@ export function tokenize(text) {
|
|
|
169
177
|
return out;
|
|
170
178
|
}
|
|
171
179
|
|
|
180
|
+
// File names in a query (`name.ext`): a query that names a file is only
|
|
181
|
+
// answered by a note that names it too and covers the rest of the question.
|
|
182
|
+
const QUERY_FILE_RE = /[\w-]+\.(?:test\.)?(?:m?js|cjs|tsx?|jsx|sh|md|json|py|ya?ml)\b/gi;
|
|
183
|
+
const FILE_EXT_TOKENS = new Set(['js', 'mjs', 'cjs', 'ts', 'tsx', 'jsx', 'sh', 'md', 'json', 'py', 'yml', 'yaml', 'test']);
|
|
184
|
+
const FILE_REST_COVERAGE = 0.3;
|
|
185
|
+
const FILE_REST_MIN_TOKENS = 2;
|
|
186
|
+
function queryFileSpec(query) {
|
|
187
|
+
if (typeof query !== 'string') return null;
|
|
188
|
+
const files = query.match(QUERY_FILE_RE);
|
|
189
|
+
if (!files) return null;
|
|
190
|
+
const fileTokens = new Set(files.flatMap((f) => tokenize(f.toLowerCase())).filter((t) => !FILE_EXT_TOKENS.has(t)));
|
|
191
|
+
const restTokens = [...new Set(tokenize(query.replace(QUERY_FILE_RE, ' '), { camel: true }))];
|
|
192
|
+
return fileTokens.size > 0 ? { fileTokens, restTokens } : null;
|
|
193
|
+
}
|
|
194
|
+
|
|
172
195
|
function episodeBody(text) {
|
|
173
196
|
return String(text ?? '')
|
|
174
197
|
.replace(/^\s*Session\s+\S+?:\s*/i, '')
|
|
@@ -272,7 +295,7 @@ function idf(totalDocs, docFreq) {
|
|
|
272
295
|
// `docs` is the ELIGIBLE set (IDF is computed over exactly this set);
|
|
273
296
|
// `query` is a string or a pre-tokenized array. Never throws.
|
|
274
297
|
export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
275
|
-
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query))];
|
|
298
|
+
const queryTokens = [...new Set(Array.isArray(query) ? query : tokenize(query, { camel: true }))];
|
|
276
299
|
const empty = {
|
|
277
300
|
ranked: [], candidateCount: 0, queryTokenCount: queryTokens.length, topScore: 0, abstained: true,
|
|
278
301
|
};
|
|
@@ -302,6 +325,10 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
|
302
325
|
let queryMass = 0;
|
|
303
326
|
for (const token of queryTokens) queryMass += idf(total, docFreq.get(token) ?? 0);
|
|
304
327
|
|
|
328
|
+
const fileSpec = total >= SMALL_CORPUS_DOCS ? queryFileSpec(query) : null;
|
|
329
|
+
let restMass = 0;
|
|
330
|
+
if (fileSpec) for (const token of fileSpec.restTokens) restMass += idf(total, docFreq.get(token) ?? 0);
|
|
331
|
+
|
|
305
332
|
const scored = [];
|
|
306
333
|
let candidateCount = 0;
|
|
307
334
|
for (const { doc, tf, length } of stats) {
|
|
@@ -324,6 +351,15 @@ export function rankDocs(docs, query, { limit = 10, twins = 'detail' } = {}) {
|
|
|
324
351
|
// clear the absolute floor above while the query's real subject is absent.
|
|
325
352
|
// It must also cover a fraction of the query's own IDF mass.
|
|
326
353
|
if (total >= SMALL_CORPUS_DOCS && doc?.type === 'episode' && idfMass < EPISODE_MIN_COVERAGE * queryMass) continue;
|
|
354
|
+
if (fileSpec) {
|
|
355
|
+
let named = false;
|
|
356
|
+
for (const token of fileSpec.fileTokens) if (tf.has(token)) { named = true; break; }
|
|
357
|
+
let restHit = 0;
|
|
358
|
+
for (const token of fileSpec.restTokens) if (tf.has(token)) restHit += idf(total, docFreq.get(token));
|
|
359
|
+
// One leftover word ("what does X check") is too little to judge coverage by.
|
|
360
|
+
const judgeRest = fileSpec.restTokens.length >= FILE_REST_MIN_TOKENS && restMass > 0;
|
|
361
|
+
if (!named || (judgeRest && restHit < FILE_REST_COVERAGE * restMass)) continue;
|
|
362
|
+
}
|
|
327
363
|
scored.push({ doc, score: score * (TYPE_PRIOR[doc.type] ?? 1), matched });
|
|
328
364
|
}
|
|
329
365
|
// Ties break on the stable logical id, never on a timestamp (CTX-01): a re-saved
|