@ngockhoale/ukit 3.5.4 → 3.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/manifests/engineConformance.yaml +0 -13
  3. package/manifests/platform.full.yaml +25 -0
  4. package/package.json +3 -9
  5. package/src/cli/commands/indexTools.js +1 -0
  6. package/src/cli/commands/memory.js +44 -4
  7. package/src/core/applyPlan.js +25 -72
  8. package/src/core/diffPlan.js +38 -9
  9. package/src/core/managedTemplateMerge.js +146 -0
  10. package/src/core/memory/memoryCheckRun.js +13 -1
  11. package/src/core/memory/memoryFlags.js +5 -6
  12. package/src/core/memory/memoryHit.js +31 -15
  13. package/src/core/memory/policy.js +5 -0
  14. package/src/core/memory/recallEngine.js +39 -3
  15. package/src/core/memory/recallLanes.js +182 -0
  16. package/src/core/memory/retrieval.js +13 -1
  17. package/src/core/memory/store.js +17 -0
  18. package/src/core/memory/topicCommand.js +75 -0
  19. package/src/core/memory/topicPages.js +545 -0
  20. package/src/core/metadata.js +12 -8
  21. package/src/core/runInstallPipeline.js +16 -10
  22. package/src/core/runtimeConfig.js +59 -83
  23. package/src/core/uninstall.js +8 -1
  24. package/src/decision/registry.js +33 -16
  25. package/src/decision/runtimeDecide.js +5 -5
  26. package/src/index/buildIndex.js +287 -6
  27. package/src/index/languageTools.js +100 -0
  28. package/src/index/paths.js +2 -1
  29. package/src/index/queryIndex.js +174 -10
  30. package/src/index/routeResolver.js +2 -2
  31. package/src/index/semanticLane.js +269 -0
  32. package/src/learning/lessonClass.js +110 -0
  33. package/src/learning/patternProposals.js +111 -7
  34. package/src/learning/reeditMiner.js +104 -0
  35. package/src/learning/selfImprove.js +9 -1
  36. package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
  37. package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
  38. package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
  39. package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
  40. package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
  41. package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
  42. package/template_project/.claude/ukit/index/query-index.mjs +9 -1
  43. package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
  44. package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
  45. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
  46. package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
  47. package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
  48. package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
  49. package/template_project/ukit/storage/config.json +19 -34
  50. package/manifests/instructionRules.yaml +0 -442
  51. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
  52. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
  53. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
  54. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
  55. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
  56. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
  57. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
  58. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
  59. package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
  60. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
  61. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
  62. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
  63. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
  64. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
  65. package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
  66. package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
  67. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
  68. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
@@ -1,8 +1,12 @@
1
1
  import fs from 'node:fs/promises';
2
2
  import path from 'node:path';
3
3
  import { execFile } from 'node:child_process';
4
+ import { createHash } from 'node:crypto';
5
+
6
+ import { withFileLock } from '../core/fileOps.js';
4
7
 
5
8
  import { getArtifactPath, getIndexDir, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION, isLikelyTestFilePath } from './paths.js';
9
+ import { embedConfig, looksLikeQuestion, semanticCandidates, fuseSemantic, semanticStoreSignature } from './semanticLane.js';
6
10
  import { importsNeedAliasContext, loadImportAliasContext, resolveImportSpecifier } from './importResolution.js';
7
11
  import {
8
12
  buildSearchDescriptor,
@@ -54,7 +58,7 @@ function safeSchemaArtifact(artifact) {
54
58
  }
55
59
 
56
60
  // contentLane=false skips the error-string content search (context-resolution paths never pay for it).
57
- export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true } = {}) {
61
+ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true, semantic = false } = {}) {
58
62
  const normalizedQuery = String(query ?? '').trim();
59
63
  if (!normalizedQuery) {
60
64
  return [];
@@ -75,11 +79,15 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
75
79
  query: normalizedQuery,
76
80
  limit,
77
81
  contentLane,
82
+ semantic: semantic && embedConfig() ? `1:${await semanticStoreSignature(absoluteRoot)}` : 0,
78
83
  });
79
84
  if (QUERY_RESULT_CACHE.has(queryCacheKey)) {
80
85
  return QUERY_RESULT_CACHE.get(queryCacheKey);
81
86
  }
82
87
  const queryPromise = (async () => {
88
+ // `laneStatus.fellBack`: the lane was asked for (question-shaped, lexical not decisive) and
89
+ // produced nothing (timeout / HTTP error / no vectors). Such a result must not be cached.
90
+ const laneStatus = { fellBack: false };
83
91
  const { files, searchDescriptorsByFile, tokenWeights } = await loadQuerySearchBundle(absoluteRoot);
84
92
  const preferredArchetypes = inferPreferredArchetypes(queryDescriptor.tokens);
85
93
  const includeLikelyTestFiles = shouldIncludeLikelyTestFiles(queryDescriptor);
@@ -153,7 +161,10 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
153
161
  }
154
162
 
155
163
  if (directScores.size === 0) {
156
- return withContentLaneStatus(Object.freeze([]), contentLaneRan, contentCutShort);
164
+ const onlySemantic = semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked: [], includeLikelyTestFiles, files, laneStatus }) : null;
165
+ if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
166
+ if (onlySemantic) return withContentLaneStatus(freezeQueryResults(onlySemantic.slice(0, limit)), contentLaneRan, contentCutShort);
167
+ return withContentLaneStatus(Object.freeze([]), contentLaneRan, contentCutShort, laneStatus.fellBack);
157
168
  }
158
169
 
159
170
  const {
@@ -196,12 +207,15 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
196
207
  }),
197
208
  });
198
209
 
199
- return withContentLaneStatus(freezeQueryResults(
200
- [...finalScores.values()]
201
- .map((entry) => ({ ...entry, score: finalizeScore(entry) }))
202
- .sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath))
203
- .slice(0, limit),
204
- ), contentLaneRan, contentCutShort);
210
+ const ranked = [...finalScores.values()]
211
+ .map((entry) => ({ ...entry, score: finalizeScore(entry) }))
212
+ .sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath));
213
+ const fused = contentLane && ranked.length > 0 && ranked[0].score < FUSION_WEAK_SCORE && queryDescriptor.tokens.size >= FUSION_MIN_TOKENS
214
+ ? await fuseWeakQuery({ absoluteRoot, query: normalizedQuery, ranked, searchDescriptorsByFile, includeLikelyTestFiles })
215
+ : null;
216
+ const semanticMerged = !fused && semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked, includeLikelyTestFiles, files, laneStatus }) : null;
217
+ if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
218
+ return withContentLaneStatus(freezeQueryResults((fused ?? semanticMerged ?? ranked).slice(0, limit)), contentLaneRan, contentCutShort, laneStatus.fellBack);
205
219
  })().catch((error) => {
206
220
  QUERY_RESULT_CACHE.delete(queryCacheKey);
207
221
  throw error;
@@ -211,6 +225,154 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
211
225
  return queryPromise;
212
226
  }
213
227
 
228
+ // Real-embedding lane: only when the CLI passes semantic:true AND UKIT_INDEX_EMBED_URL is set.
229
+ // Applies to question-shaped queries only; any problem returns null (lexical result unchanged).
230
+ const SEMANTIC_CONFIDENT_MARGIN = 2;
231
+ async function applySemanticLane({ absoluteRoot, query, ranked, includeLikelyTestFiles, files, laneStatus = {} }) {
232
+ try {
233
+ if (!embedConfig() || !looksLikeQuestion(query)) return null;
234
+ // Lexical is already decisive (top-1 clear of top-2): keep it exactly, no network call.
235
+ if (ranked.length > 1 && ranked[0].score - ranked[1].score >= SEMANTIC_CONFIDENT_MARGIN) return null;
236
+ const known = new Set(safeItems(files).map((f) => f?.filePath));
237
+ const candidates = await semanticCandidates({
238
+ rootDir: absoluteRoot,
239
+ query,
240
+ // Vectors can outlive a deleted/renamed file: only paths in the current index qualify.
241
+ accept: (filePath) => known.has(filePath) && (includeLikelyTestFiles || !isLikelyTestFilePath(filePath)),
242
+ weight: mirrorPrior,
243
+ });
244
+ const fused = fuseSemantic({ ranked, candidates });
245
+ if (!fused) laneStatus.fellBack = true;
246
+ return fused;
247
+ } catch {
248
+ laneStatus.fellBack = true;
249
+ return null;
250
+ }
251
+ }
252
+
253
+ // Opt-in lane fusion (default OFF): a weak natural-language query (best score below
254
+ // FUSION_WEAK_SCORE, >= FUSION_MIN_TOKENS tokens) is re-ranked by RRF over the scored list
255
+ // and a hashed-embedding lane (the retriever.js vector lane). Enabled by
256
+ // `.ukit/storage/config.json` -> index.fusion === true or UKIT_INDEX_FUSION=1. The flag is
257
+ // read only after the weak gate passes, and contentLane:false callers (hooks/resolveContext)
258
+ // never reach it. Never throws; async fs only.
259
+ const FUSION_WEAK_SCORE = 6;
260
+ const FUSION_MIN_TOKENS = 4;
261
+ const FUSION_RRF_K = 5;
262
+ const FUSION_VECTOR_WEIGHT = 1;
263
+ const FUSION_LANE_LIMIT = 100;
264
+ const FUSION_DOC_VECTORS = new WeakMap();
265
+
266
+ // Inlined copy of src/core/codeintel/vectorProvider.js hashVectorize/cosine (256 dims, FNV-1a
267
+ // word + trigram features, L2-normalized) so the shipped index-core.mjs stays dependency-free.
268
+ function fusionFnv1a(str) {
269
+ let h = 0x811c9dc5;
270
+ for (let i = 0; i < str.length; i += 1) {
271
+ h ^= str.charCodeAt(i);
272
+ h = Math.imul(h, 0x01000193) >>> 0;
273
+ }
274
+ return h >>> 0;
275
+ }
276
+
277
+ function hashVectorize(text) {
278
+ const dims = 256;
279
+ const vec = new Array(dims).fill(0);
280
+ for (const token of String(text ?? '').split(/[^A-Za-z0-9_$]+/).map((t) => t.toLowerCase()).filter(Boolean)) {
281
+ vec[fusionFnv1a(`w:${token}`) % dims] += 1;
282
+ const padded = `#${token}#`;
283
+ for (let i = 0; i + 3 <= padded.length; i += 1) vec[fusionFnv1a(`t:${padded.slice(i, i + 3)}`) % dims] += 0.1;
284
+ }
285
+ const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
286
+ if (norm > 0) for (let i = 0; i < dims; i += 1) vec[i] /= norm;
287
+ return vec;
288
+ }
289
+
290
+ function cosineSimilarity(a, b) {
291
+ let dot = 0;
292
+ for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
293
+ return dot;
294
+ }
295
+
296
+ async function isFusionEnabled(absoluteRoot) {
297
+ if (process.env.UKIT_INDEX_FUSION === '1') return true;
298
+ if (process.env.UKIT_INDEX_FUSION === '0') return false;
299
+ try {
300
+ const config = JSON.parse(await fs.readFile(path.join(absoluteRoot, '.ukit', 'storage', 'config.json'), 'utf8'));
301
+ return config?.index?.fusion === true;
302
+ } catch {
303
+ return false;
304
+ }
305
+ }
306
+
307
+ async function fuseWeakQuery({ absoluteRoot, query, ranked, searchDescriptorsByFile, includeLikelyTestFiles }) {
308
+ try {
309
+ if (!(await isFusionEnabled(absoluteRoot))) return null;
310
+ let docVectors = FUSION_DOC_VECTORS.get(searchDescriptorsByFile);
311
+ if (!docVectors) {
312
+ docVectors = new Map();
313
+ for (const [filePath, descriptor] of searchDescriptorsByFile) {
314
+ docVectors.set(filePath, hashVectorize(`${filePath} ${(descriptor.symbols ?? []).map((s) => s.name).join(' ')}`));
315
+ }
316
+ FUSION_DOC_VECTORS.set(searchDescriptorsByFile, docVectors);
317
+ }
318
+ const queryVector = hashVectorize(query);
319
+ const vectorLane = [];
320
+ for (const [filePath, vector] of docVectors) {
321
+ if (isLikelyTestFilePath(filePath) && !includeLikelyTestFiles) continue;
322
+ const similarity = cosineSimilarity(queryVector, vector);
323
+ if (similarity > 0) vectorLane.push([filePath, similarity]);
324
+ }
325
+ vectorLane.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
326
+ const vectorPaths = vectorLane.slice(0, FUSION_LANE_LIMIT).map(([filePath]) => filePath);
327
+ const indexPaths = ranked.slice(0, FUSION_LANE_LIMIT).map((entry) => entry.filePath);
328
+ const rrf = new Map();
329
+ indexPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + 1 / (FUSION_RRF_K + i + 1)));
330
+ vectorPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + FUSION_VECTOR_WEIGHT / (FUSION_RRF_K + i + 1)));
331
+ const byPath = new Map(ranked.map((entry) => [entry.filePath, entry]));
332
+ const order = [...rrf.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([filePath]) => filePath);
333
+ const floor = ranked[ranked.length - 1].score;
334
+ const merged = order.map((filePath) => byPath.get(filePath)
335
+ ?? { filePath, score: Math.min(0.01, floor), reasons: ['fusion:vector'], tests: [] });
336
+ await emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths: merged.map((e) => e.filePath) });
337
+ return merged;
338
+ } catch {
339
+ return null;
340
+ }
341
+ }
342
+
343
+ // Same event shape and file as retriever.js emitRetrieverLaneTelemetry (query stored as sha256).
344
+ async function emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths }) {
345
+ try {
346
+ const event = {
347
+ ts: Date.now(),
348
+ query: createHash('sha256').update(String(query ?? '')).digest('hex'),
349
+ mode: 'index-query',
350
+ lanes: {
351
+ index: { hits: indexPaths.length, weightUsed: 1, paths: indexPaths },
352
+ vector: { hits: vectorPaths.length, weightUsed: FUSION_VECTOR_WEIGHT, paths: vectorPaths },
353
+ },
354
+ mergedPaths: mergedPaths.slice(0, 100),
355
+ merge: 'rrf',
356
+ };
357
+ const filePath = path.join(absoluteRoot, '.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
358
+ await fs.mkdir(path.dirname(filePath), { recursive: true });
359
+ await withFileLock(filePath, async () => {
360
+ try {
361
+ const st = await fs.stat(filePath);
362
+ if (st.size > 256 * 1024) {
363
+ const kept = (await fs.readFile(filePath, 'utf8')).split('\n').filter(Boolean).slice(-2000);
364
+ await fs.writeFile(filePath, kept.length ? `${kept.join('\n')}\n` : '', 'utf8');
365
+ }
366
+ } catch {
367
+ // Missing file is fine; append below recreates it.
368
+ }
369
+ await fs.appendFile(filePath, `${JSON.stringify(event)}\n`, 'utf8');
370
+ });
371
+ } catch {
372
+ // Telemetry is advisory.
373
+ }
374
+ }
375
+
214
376
  // The `callable` symbol regex matches any `const x = (`, so it also captures plain
215
377
  // parenthesised expressions such as `const rows = (items ?? [])`. Harmless for fuzzy
216
378
  // ranking, misleading in an outline that is read as a map of the file's API. Those rows
@@ -417,7 +579,7 @@ const BOOST_LIFT_RATIO = 0.2; // import/hotspot/archetype boosts may add at most
417
579
  const CONTENT_LANE_CONFIDENT_SCORE = 30; // a best direct score at/above this (exact symbol or path hit) skips the content lane
418
580
  const CONTENT_LANE_MIN_TOKENS = 3; // shorter queries are names, not messages
419
581
  const CONTENT_LANE_LIMIT = 40; // candidates only share tokens somewhere in the file; the same-line check below keeps the real ones
420
- const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|sql|sh)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
582
+ const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|svelte|sql|sh|py|go|java|cs|php|rb|rs)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
421
583
  const CONTENT_LANE_BONUS = 14; // added to a file that already has a direct score
422
584
  const CONTENT_LANE_ONLY_SCORE = 13; // base for a file found only by content; below CONTENT_LANE_CONFIDENT_SCORE by design
423
585
  const CONTENT_LANE_MAX_FILES = 1500; // files read per query, in priority order (primary bound)
@@ -1434,9 +1596,11 @@ export function queryMatchedSymbolNames(result) {
1434
1596
  // Non-enumerable (array equality and JSON stay unchanged), like boundedSourceSearch rows.truncated:
1435
1597
  // `contentLane` is true when the content search ran; `contentLaneTruncated` lists why it was cut
1436
1598
  // short ('verify-cap', 'time-budget', 'max-files', ...) so a partial scan never reads as "no match".
1437
- function withContentLaneStatus(results, ran, truncated) {
1599
+ function withContentLaneStatus(results, ran, truncated, semanticFellBack = false) {
1438
1600
  // results arrive frozen, so copy before attaching the status and freeze again.
1439
1601
  const out = [...results];
1602
+ // true when the embedding lane was asked for and returned nothing: callers must not persist this result.
1603
+ Object.defineProperty(out, 'semanticFellBack', { value: Boolean(semanticFellBack), enumerable: false });
1440
1604
  Object.defineProperty(out, 'contentLane', { value: ran, enumerable: false });
1441
1605
  Object.defineProperty(out, 'contentLaneTruncated', { value: Object.freeze([...truncated]), enumerable: false });
1442
1606
  return Object.freeze(out);
@@ -273,8 +273,8 @@ export const ROUTE_SHADOW_QUESTIONS = Object.freeze([
273
273
  // capability.impact.v1 — noul kind; no threshold baseline exists on
274
274
  // routeSummary (capabilityPolicy.recommended is unpopulated upstream), so
275
275
  // the question has no comparison value.
276
- // learn.candidate-class.v1 — reserved for the memoryV2 `decision` plane
277
- // (memoryFlags.js), not the route boundary.
276
+ // learn.candidate-class.v1 — the `learn` family classifies mined lesson candidates
277
+ // (gated by decisionPlane.families.learn.stage, off by default); it is not a route-boundary question.
278
278
  ]);
279
279
 
280
280
  // Only families whose resolved stage is not 'off' get a question.
@@ -0,0 +1,269 @@
1
+ // Real-embedding lane for `ukit index query` (opt-in; no dependency; global fetch only).
2
+ //
3
+ // PRIVACY: building vectors SENDS FILE TEXT (path, symbol names, leading comment and,
4
+ // at most ~240 characters per file) to the endpoint in
5
+ // UKIT_INDEX_EMBED_URL, and queries send the question text. With no URL configured nothing
6
+ // here runs and nothing leaves the machine. Sensitive paths (.env, keys, credentials ...)
7
+ // are never embedded; only files already present in the index are considered.
8
+ //
9
+ // Byte-identical mirror: template_project/.claude/ukit/index/lib/semantic-lane.mjs
10
+ // (tests/index/semanticLane.test.js enforces it). Keep this file dependency-free.
11
+ //
12
+ // Env: UKIT_INDEX_EMBED_URL (base URL ending in /v1), UKIT_INDEX_EMBED_MODEL,
13
+ // UKIT_INDEX_EMBED_KEY (optional Bearer).
14
+ // Build is an explicit command (embed-index.mjs), never part of index build/refresh, never in
15
+ // a hook. Query use is reachable only through queryCodeIndex({ semantic: true }), which only
16
+ // the CLI query path passes.
17
+ import fs from 'node:fs/promises';
18
+ import path from 'node:path';
19
+ import crypto from 'node:crypto';
20
+
21
+ export const SEMANTIC_MANIFEST = 'semantic.json';
22
+ export const SEMANTIC_BIN = 'semantic.bin';
23
+ // A CLI query waits at most this long for the embedding of the question. Measured on the
24
+ // reference endpoint: warm question embedding ~0.45 s, 64 descriptors ~24 s (serial service).
25
+ export const QUERY_TIMEOUT_MS = 1500;
26
+ // One descriptor per file, ~60 tokens. Endpoint throughput is ~170-200 tokens/s and serial, so
27
+ // file bodies (>1 h for 2.4k files) are not embedded; descriptors build in minutes.
28
+ export const DOC_CHARS = 240;
29
+ const NAMES_CHARS = 90;
30
+ const LEAD_CHARS = 100;
31
+ const QUERY_INSTRUCTION = 'Instruct: Given a question about a code repository, retrieve the source file that answers it\nQuery: ';
32
+
33
+ export function embedConfig(env = process.env) {
34
+ // Accepts the base (.../v1) or the full endpoint (.../v1/embeddings).
35
+ const url = String(env.UKIT_INDEX_EMBED_URL ?? '').trim().replace(/\/+$/, '').replace(/\/embeddings$/i, '');
36
+ if (!url) return null;
37
+ return { url, model: String(env.UKIT_INDEX_EMBED_MODEL ?? '').trim() || 'unic-embedding', key: String(env.UKIT_INDEX_EMBED_KEY ?? '').trim() };
38
+ }
39
+
40
+ function isSensitive(value) {
41
+ const segments = String(value ?? '').replace(/\\/g, '/').split('/');
42
+ const base = segments[segments.length - 1] ?? '';
43
+ return base.toLowerCase().startsWith('.env')
44
+ || /\.(?:pem|key|p12|pfx|jks|keystore)$/i.test(base)
45
+ || /^(?:credentials?|\.npmrc|\.netrc|\.pypirc)(?:\.[^/]*)?$/i.test(base)
46
+ || /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.[^/]*)?$/i.test(base)
47
+ || segments.includes('.ssh') || segments.includes('.aws') || segments.includes('.gnupg');
48
+ }
49
+
50
+ const sha = (text) => crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
51
+
52
+ // POST /embeddings; returns arrays in input order or throws. Validates shape.
53
+ export async function embedTexts(texts, cfg, { timeoutMs = 60000 } = {}) {
54
+ const headers = { 'content-type': 'application/json' };
55
+ if (cfg.key) headers.authorization = `Bearer ${cfg.key}`;
56
+ const response = await fetch(`${cfg.url}/embeddings`, {
57
+ method: 'POST',
58
+ headers,
59
+ body: JSON.stringify({ model: cfg.model, input: texts }),
60
+ signal: AbortSignal.timeout(timeoutMs),
61
+ });
62
+ if (!response.ok) throw new Error(`embedding endpoint HTTP ${response.status}`);
63
+ const data = (await response.json())?.data;
64
+ if (!Array.isArray(data) || data.length !== texts.length) throw new Error('embedding response shape mismatch');
65
+ const rows = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding);
66
+ const dims = rows[0]?.length;
67
+ if (!dims || rows.some((r) => !Array.isArray(r) || r.length !== dims || r.some((x) => !Number.isFinite(x)))) {
68
+ throw new Error('embedding response vectors invalid');
69
+ }
70
+ return rows;
71
+ }
72
+
73
+ function normalizeTo(vec, dims) {
74
+ const out = vec.slice(0, dims);
75
+ let n = 0;
76
+ for (const x of out) n += x * x;
77
+ n = Math.sqrt(n) || 1;
78
+ return out.map((x) => x / n);
79
+ }
80
+
81
+ // Descriptor for one file: path + exported/declared symbol names + leading doc comment,
82
+ // capped at DOC_CHARS. The path is never truncated; names and comment share the remainder.
83
+ export function buildDocumentText({ filePath, symbols = [], content = '' }) {
84
+ const names = [...new Set(symbols)].join(' ').slice(0, NAMES_CHARS);
85
+ const lines = String(content).split('\n');
86
+ const lead = [];
87
+ for (const line of lines.slice(0, 80)) {
88
+ const t = line.trim();
89
+ if (!t || t.startsWith('#!') || t === '---' || t.startsWith("'use ")) { if (lead.length) break; continue; }
90
+ if (/^(\/\/|\/\*|\*|#|<!--|"""|--)/.test(t)) lead.push(t.replace(/^(\/\/+|\/\*+|\*+\/?|#+|<!--|-->|"""|--)\s?/, ''));
91
+ else break;
92
+ }
93
+ const leading = (lead.length ? lead.join(' ') : '').slice(0, LEAD_CHARS);
94
+ const text = `${filePath}\n${names}\n${leading}`;
95
+ return text.length > DOC_CHARS ? text.slice(0, DOC_CHARS) : text;
96
+ }
97
+
98
+ async function readJson(file) {
99
+ try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; }
100
+ }
101
+
102
+ async function loadStore(indexDir) {
103
+ const manifest = await readJson(path.join(indexDir, SEMANTIC_MANIFEST));
104
+ if (!manifest || manifest.v !== 1 || !Array.isArray(manifest.order)) return null;
105
+ let bin;
106
+ try { bin = await fs.readFile(path.join(indexDir, SEMANTIC_BIN)); } catch { return null; }
107
+ const rec = 4 + manifest.dims;
108
+ if (bin.length !== rec * manifest.order.length) return null;
109
+ return { manifest, bin, rec };
110
+ }
111
+
112
+ // Explicit build step. Embeds only documents whose hash (document text + model) is not in the
113
+ // existing store. Returns stats. Never called from index build/refresh or hooks.
114
+ export async function buildSemanticVectors({
115
+ rootDir = process.cwd(), env = process.env, dims = 0,
116
+ batchSize = 64, onProgress = null,
117
+ } = {}) {
118
+ const cfg = embedConfig(env);
119
+ if (!cfg) throw new Error('UKIT_INDEX_EMBED_URL is not set; semantic vectors are opt-in');
120
+ const root = path.resolve(rootDir);
121
+ const indexDir = path.join(root, '.cache', 'index');
122
+ const filesArtifact = await readJson(path.join(indexDir, 'files.json'));
123
+ const symbolsArtifact = await readJson(path.join(indexDir, 'symbols.json'));
124
+ if (!filesArtifact?.items) throw new Error('index missing; run the index build first');
125
+ const symbolsByFile = new Map();
126
+ for (const s of symbolsArtifact?.items ?? []) {
127
+ if (!symbolsByFile.has(s.filePath)) symbolsByFile.set(s.filePath, []);
128
+ symbolsByFile.get(s.filePath).push(s.name);
129
+ }
130
+ // Reuse vectors from a store built with the same model and truncation.
131
+ const old = await loadStore(indexDir);
132
+ const reusable = old && old.manifest.model === cfg.model
133
+ && (!dims || old.manifest.dims === dims) ? old : null;
134
+ const oldByHash = new Map();
135
+ if (reusable) reusable.manifest.order.forEach(([, h], i) => oldByHash.set(h, reusable.bin.subarray(i * reusable.rec, (i + 1) * reusable.rec)));
136
+
137
+ const docs = [];
138
+ for (const item of filesArtifact.items) {
139
+ const filePath = item.filePath;
140
+ if (typeof filePath !== 'string' || isSensitive(filePath)) continue;
141
+ let content = '';
142
+ try { content = await fs.readFile(path.join(root, filePath), 'utf8'); } catch { continue; }
143
+ const text = buildDocumentText({ filePath, symbols: symbolsByFile.get(filePath) ?? [], content });
144
+ docs.push({ filePath, text, hash: sha(`${cfg.model}\0${text}`) });
145
+ }
146
+ const missing = [...new Map(docs.filter((d) => !oldByHash.has(d.hash)).map((d) => [d.hash, d])).values()];
147
+ const fresh = new Map();
148
+ let outDims = dims || reusable?.manifest.dims || 0;
149
+ // Requests are strictly sequential: the embedding service is serial, so parallel requests
150
+ // only queue. Partial progress is saved: a failed batch leaves its files without vectors and
151
+ // the error is rethrown after the store is written, so a rerun only embeds what is missing.
152
+ const failures = [];
153
+ try {
154
+ for (let i = 0; i < missing.length; i += batchSize) {
155
+ const batch = missing.slice(i, i + batchSize);
156
+ const rows = await embedTexts(batch.map((d) => d.text), cfg, { timeoutMs: 180000 });
157
+ if (!outDims) outDims = rows[0].length;
158
+ if (rows[0].length < outDims) throw new Error('endpoint returned fewer dimensions than requested');
159
+ batch.forEach((d, k) => fresh.set(d.hash, rows[k]));
160
+ onProgress?.({ done: fresh.size, total: missing.length });
161
+ }
162
+ } catch (e) { failures.push(e); }
163
+ if (!outDims) outDims = old?.manifest.dims ?? 0;
164
+ if (!outDims && docs.length) throw new Error('no vectors produced');
165
+
166
+ const rec = 4 + outDims;
167
+ const order = docs.filter((d) => fresh.has(d.hash) || oldByHash.has(d.hash)).map((d) => [d.filePath, d.hash]);
168
+ const bin = Buffer.alloc(rec * order.length);
169
+ order.forEach(([, hash], i) => {
170
+ const off = i * rec;
171
+ const prior = oldByHash.get(hash);
172
+ if (prior) { prior.copy(bin, off); return; }
173
+ const v = normalizeTo(fresh.get(hash), outDims);
174
+ const maxAbs = Math.max(...v.map(Math.abs)) || 1;
175
+ bin.writeFloatLE(maxAbs / 127, off);
176
+ for (let k = 0; k < outDims; k += 1) bin.writeInt8(Math.max(-127, Math.min(127, Math.round(v[k] / maxAbs * 127))), off + 4 + k);
177
+ });
178
+ const manifest = { v: 1, model: cfg.model, dims: outDims, createdAt: new Date().toISOString(), order };
179
+ const tmp = `.tmp-${process.pid}`;
180
+ await fs.writeFile(path.join(indexDir, SEMANTIC_BIN + tmp), bin);
181
+ await fs.writeFile(path.join(indexDir, SEMANTIC_MANIFEST + tmp), JSON.stringify(manifest));
182
+ await fs.rename(path.join(indexDir, SEMANTIC_BIN + tmp), path.join(indexDir, SEMANTIC_BIN));
183
+ await fs.rename(path.join(indexDir, SEMANTIC_MANIFEST + tmp), path.join(indexDir, SEMANTIC_MANIFEST));
184
+ if (failures.length) throw new Error(`partial store saved (${order.length}/${docs.length} files); rerun to resume: ${failures[0]?.message ?? failures[0]}`);
185
+ return { files: docs.length, embedded: fresh.size, reused: docs.length - missing.length, dims: outDims, bytes: bin.length, model: cfg.model };
186
+ }
187
+
188
+ // Cache-key part: changes when the vector store is built, rebuilt or removed, so a result cached
189
+ // before `embed-index` ran never outlives the vectors. Never throws.
190
+ export async function semanticStoreSignature(rootDir) {
191
+ try {
192
+ const stat = await fs.stat(path.join(path.resolve(rootDir), '.cache', 'index', SEMANTIC_MANIFEST));
193
+ return `${stat.mtimeMs}:${stat.size}`;
194
+ } catch {
195
+ return 'none';
196
+ }
197
+ }
198
+
199
+ const STORE_CACHE = new Map();
200
+
201
+ // Query-side ranking. Returns [{ filePath, similarity }] (best first) or null on ANY problem
202
+ // (lane off, no vectors, model/dimension mismatch, timeout, HTTP error). Never throws.
203
+ export async function semanticCandidates({
204
+ rootDir, query, env = process.env, timeoutMs = QUERY_TIMEOUT_MS, limit = 100,
205
+ queryInstruction = QUERY_INSTRUCTION, accept = () => true, weight = () => 1,
206
+ } = {}) {
207
+ try {
208
+ const cfg = embedConfig(env);
209
+ if (!cfg) return null;
210
+ const indexDir = path.join(path.resolve(rootDir), '.cache', 'index');
211
+ const stat = await fs.stat(path.join(indexDir, SEMANTIC_MANIFEST));
212
+ const key = `${indexDir}:${stat.mtimeMs}:${stat.size}`;
213
+ let store = STORE_CACHE.get(key);
214
+ if (!store) { store = await loadStore(indexDir); if (!store) return null; STORE_CACHE.clear(); STORE_CACHE.set(key, store); }
215
+ const { manifest, bin, rec } = store;
216
+ if (manifest.model !== cfg.model) return null;
217
+ const [raw] = await embedTexts([`${queryInstruction}${query}`], cfg, { timeoutMs });
218
+ if (raw.length < manifest.dims) return null;
219
+ const q = normalizeTo(raw, manifest.dims);
220
+ const scored = [];
221
+ for (let i = 0; i < manifest.order.length; i += 1) {
222
+ const filePath = manifest.order[i][0];
223
+ if (!accept(filePath)) continue;
224
+ const off = i * rec;
225
+ let dot = 0;
226
+ for (let k = 0; k < manifest.dims; k += 1) dot += q[k] * bin.readInt8(off + 4 + k);
227
+ scored.push({ filePath, similarity: dot * bin.readFloatLE(off) * weight(filePath) });
228
+ }
229
+ scored.sort((a, b) => b.similarity - a.similarity || a.filePath.localeCompare(b.filePath));
230
+ return scored.slice(0, limit);
231
+ } catch {
232
+ return null;
233
+ }
234
+ }
235
+
236
+ // Plain-language gate: >= 4 words, no path separators or code punctuation, no quotes/colons,
237
+ // no trailing period (a finished statement is error-message shaped), no error-leading word, and
238
+ // no identifier-shaped token (camelCase, snake_case, kebab with digits, dotted name, file
239
+ // extension, token containing a digit, ALL-CAPS code such as ENOENT). Non-ASCII text
240
+ // (Vietnamese) and ordinary lowercase sentences pass; everything else keeps the lexical result.
241
+ const ERROR_LEADS = /^(?:error|errors|failed|failure|cannot|can't|unable|exception|invalid|unexpected|uncaught|warning|fatal|missing|undefined|null|typeerror|referenceerror)\b/i;
242
+ function identifierShaped(token) {
243
+ const t = token.replace(/^[^\w]+|[^\w]+$/g, '');
244
+ if (!t) return false;
245
+ return /[a-z][A-Z]/.test(t) || /_/.test(t) || /\d/.test(t) || /\w\.\w/.test(t) || /\w-\w*\d/.test(t)
246
+ || /^[A-Z]{3,}$/.test(t) || /^[A-Z][a-z]+[A-Z]/.test(t) || /\.(?:js|mjs|cjs|ts|tsx|jsx|json|md|py|go|rs|java|rb|yml|yaml|sh)\b/i.test(token);
247
+ }
248
+ export function looksLikeQuestion(query) {
249
+ const q = String(query ?? '').trim();
250
+ const words = q.split(/\s+/).filter(Boolean);
251
+ if (words.length < 4 || /[\\/]|[(){}\[\]=;<>@#$%^&*|~`'"]|::|=>|:/.test(q.replace(/[^\x00-\x7f]/g, ''))) return false;
252
+ if (/[^\x20-\x7e]/.test(q)) return true;
253
+ if (/\.\s*$/.test(q) || ERROR_LEADS.test(q) || words.some(identifierShaped)) return false;
254
+ return true;
255
+ }
256
+
257
+ // Combine: weighted RRF over the lexical ranking and the semantic list. Semantic-only files
258
+ // are injected with a floor score. Returns null (caller keeps lexical) when no candidates.
259
+ export function fuseSemantic({ ranked, candidates, rrfK = 5, weight = 1, depth = 100 }) {
260
+ if (!candidates?.length) return null;
261
+ const rrf = new Map();
262
+ ranked.slice(0, depth).forEach((e, i) => rrf.set(e.filePath, 1 / (rrfK + i + 1)));
263
+ candidates.slice(0, depth).forEach((c, i) => rrf.set(c.filePath, (rrf.get(c.filePath) ?? 0) + weight / (rrfK + i + 1)));
264
+ const byPath = new Map(ranked.map((e) => [e.filePath, e]));
265
+ const floor = ranked.length ? ranked[ranked.length - 1].score : 0.01;
266
+ return [...rrf.entries()]
267
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
268
+ .map(([filePath]) => byPath.get(filePath) ?? { filePath, score: Math.min(0.01, floor), reasons: ['semantic:vector'], tests: [] });
269
+ }
@@ -0,0 +1,110 @@
1
+ // lessonClass.js — is a mined pattern a real recurring lesson, or noise?
2
+ //
3
+ // Deterministic rules are the floor and the fallback. For the candidates the
4
+ // rules leave 'ambiguous', the decision plane (`unic-decision`, the owner's
5
+ // local decision model, reached through src/decision/client.js) is asked one
6
+ // yes/no question each. Its answer can only DOWNGRADE a candidate to 'noise';
7
+ // it never approves or promotes anything. Stage off, provider unreachable,
8
+ // malformed answer, timeout or throw -> the floor result stands.
9
+ //
10
+ // Call sites: `ukit memory learn` and the session-end self-improve pass only.
11
+ // Never from a deadline-armed hook.
12
+
13
+ import { resolveConfigStage } from '../core/runtimeConfig.js';
14
+ import { createDecisionClient } from '../decision/client.js';
15
+
16
+ export const LESSON_DECISION_KEY = 'learn.candidate-class.v1';
17
+ export const LESSON_TIMEOUT_MS = 2000;
18
+ const MAX_ASKED = 8;
19
+ const TEXT_CHARS = 200;
20
+
21
+ export function resolveLessonStage(config) {
22
+ return resolveConfigStage(config, 'decisionPlane.families.learn.stage');
23
+ }
24
+
25
+ // Failure patterns: runners that verify work are lessons; shell navigation is noise.
26
+ const RUNNER_RE = /^(yarn|npm|pnpm|npx|vitest|jest|pytest|cargo|go|make|tsc|eslint)\b/;
27
+ const TRIVIA_RE = /^(cd|ls|pwd|echo|cat|head|tail|grep|find|which|true|false)\b/;
28
+
29
+ export function classifyPattern(pattern) {
30
+ const sig = String(pattern?.signature ?? '').trim().toLowerCase();
31
+ if (!sig || TRIVIA_RE.test(sig)) return 'noise';
32
+ return RUNNER_RE.test(sig) ? 'lesson' : 'ambiguous';
33
+ }
34
+
35
+ // Re-edited files: product source is a lesson, data files are noise, the rest is unclear.
36
+ // Current layout: src/, bin/, scripts/ and the shipped template runtime. (The old
37
+ // templates/.claude/** layout is gone and is dropped by dropMissingFiles first.)
38
+ const SOURCE_RE = /^(src|lib|bin|scripts|template_project\/\.claude\/(hooks|ukit))\/.*\.(js|mjs|cjs|ts|tsx|sh|py)$/;
39
+ const DATA_RE = /\.(json|ya?ml|lock|txt|csv|svg|png)$/;
40
+
41
+ export function classifyReedit(reedit) {
42
+ const file = String(reedit?.file ?? '');
43
+ if (!file || DATA_RE.test(file)) return 'noise';
44
+ return SOURCE_RE.test(file) ? 'lesson' : 'ambiguous';
45
+ }
46
+
47
+ const oneLine = (text) => String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, TEXT_CHARS);
48
+
49
+ function withTimeout(promise, ms) {
50
+ let timer;
51
+ const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ status: 'timeout', answers: [] }), ms); });
52
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
53
+ }
54
+
55
+ /**
56
+ * @param {Array<{id:string, text:string, cls:'lesson'|'noise'|'ambiguous'}>} items
57
+ * @returns {Promise<{items: Array, meta: {stage, asked, answered, changed, applied, status}}>}
58
+ * `answered` counts real valid provider responses, so callers can tell a run
59
+ * that used the decision plane from one that fell back.
60
+ */
61
+ export async function refineWithDecision(items, {
62
+ client, stage = 'off', timeoutMs = LESSON_TIMEOUT_MS,
63
+ } = {}) {
64
+ const meta = { stage, asked: 0, answered: 0, changed: 0, applied: false, status: 'skipped' };
65
+ const list = Array.isArray(items) ? items : [];
66
+ try {
67
+ if (stage === 'off' || !client) return { items: list, meta };
68
+ const ambiguous = list.filter((item) => item?.cls === 'ambiguous').slice(0, MAX_ASKED);
69
+ if (ambiguous.length === 0) return { items: list, meta };
70
+ meta.asked = ambiguous.length;
71
+ const result = await withTimeout(client.requestBatch({
72
+ statePacket: { task: 'learn-candidate-class', candidateCount: ambiguous.length },
73
+ questions: ambiguous.map((item, i) => ({
74
+ decisionKey: `${LESSON_DECISION_KEY}#c${i + 1}`,
75
+ schemaVersion: 1,
76
+ family: 'learn',
77
+ owner: 'lessonClass',
78
+ kind: 'noul',
79
+ instruction: `Is this a real recurring problem worth remembering for future edits? ${oneLine(item.text)}`,
80
+ candidates: [],
81
+ })),
82
+ deadlineMs: timeoutMs,
83
+ }), timeoutMs);
84
+ meta.status = result?.status ?? 'invalid';
85
+ const downgrade = new Set();
86
+ for (const answer of result?.answers ?? []) {
87
+ if (answer?.validationStatus !== 'valid' || !Number.isFinite(answer.value)) continue;
88
+ const idx = Number(String(answer.decisionKey).split('#c')[1]);
89
+ if (!Number.isInteger(idx) || idx < 1 || idx > ambiguous.length) continue;
90
+ meta.answered += 1;
91
+ if (answer.value < 0.5) downgrade.add(ambiguous[idx - 1]);
92
+ }
93
+ if (stage === 'shadow' || downgrade.size === 0) return { items: list, meta };
94
+ meta.changed = downgrade.size;
95
+ meta.applied = true;
96
+ return { items: list.map((item) => (downgrade.has(item) ? { ...item, cls: 'noise' } : item)), meta };
97
+ } catch {
98
+ meta.status = 'error';
99
+ return { items: list, meta };
100
+ }
101
+ }
102
+
103
+ export function createLessonClient({ config, projectRoot, homeDir, transport } = {}) {
104
+ if (resolveLessonStage(config) === 'off') return null;
105
+ try {
106
+ return createDecisionClient({ config, projectRoot, homeDir, transport });
107
+ } catch {
108
+ return null;
109
+ }
110
+ }