@ngockhoale/ukit 3.5.4 → 3.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/manifests/engineConformance.yaml +0 -13
- package/manifests/platform.full.yaml +25 -0
- package/package.json +3 -9
- package/src/cli/commands/indexTools.js +1 -0
- package/src/cli/commands/memory.js +44 -4
- package/src/core/applyPlan.js +25 -72
- package/src/core/diffPlan.js +38 -9
- package/src/core/managedTemplateMerge.js +146 -0
- package/src/core/memory/memoryCheckRun.js +13 -1
- package/src/core/memory/memoryFlags.js +5 -6
- package/src/core/memory/memoryHit.js +31 -15
- package/src/core/memory/policy.js +5 -0
- package/src/core/memory/recallEngine.js +39 -3
- package/src/core/memory/recallLanes.js +182 -0
- package/src/core/memory/retrieval.js +13 -1
- package/src/core/memory/store.js +17 -0
- package/src/core/memory/topicCommand.js +75 -0
- package/src/core/memory/topicPages.js +545 -0
- package/src/core/metadata.js +12 -8
- package/src/core/runInstallPipeline.js +16 -10
- package/src/core/runtimeConfig.js +59 -83
- package/src/core/uninstall.js +8 -1
- package/src/decision/registry.js +33 -16
- package/src/decision/runtimeDecide.js +5 -5
- package/src/index/buildIndex.js +287 -6
- package/src/index/languageTools.js +100 -0
- package/src/index/paths.js +2 -1
- package/src/index/queryIndex.js +174 -10
- package/src/index/routeResolver.js +2 -2
- package/src/index/semanticLane.js +269 -0
- package/src/learning/lessonClass.js +110 -0
- package/src/learning/patternProposals.js +111 -7
- package/src/learning/reeditMiner.js +104 -0
- package/src/learning/selfImprove.js +9 -1
- package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
- package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
- package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
- package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
- package/template_project/.claude/ukit/index/lib/index-core.mjs +561 -14
- package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
- package/template_project/.claude/ukit/index/query-index.mjs +9 -1
- package/template_project/.claude/ukit/index/refresh-index.mjs +11 -0
- package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
- package/template_project/.claude/ukit/runtime/memory-lanes.mjs +9 -2
- package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
- package/template_project/.claude/ukit/runtime/memory-recall.mjs +39 -3
- package/template_project/.claude/ukit/runtime/memory-topics.mjs +545 -0
- package/template_project/ukit/storage/config.json +19 -34
- package/manifests/instructionRules.yaml +0 -442
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
package/src/index/queryIndex.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import fs from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { execFile } from 'node:child_process';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
5
|
+
|
|
6
|
+
import { withFileLock } from '../core/fileOps.js';
|
|
4
7
|
|
|
5
8
|
import { getArtifactPath, getIndexDir, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION, isLikelyTestFilePath } from './paths.js';
|
|
9
|
+
import { embedConfig, looksLikeQuestion, semanticCandidates, fuseSemantic, semanticStoreSignature } from './semanticLane.js';
|
|
6
10
|
import { importsNeedAliasContext, loadImportAliasContext, resolveImportSpecifier } from './importResolution.js';
|
|
7
11
|
import {
|
|
8
12
|
buildSearchDescriptor,
|
|
@@ -54,7 +58,7 @@ function safeSchemaArtifact(artifact) {
|
|
|
54
58
|
}
|
|
55
59
|
|
|
56
60
|
// contentLane=false skips the error-string content search (context-resolution paths never pay for it).
|
|
57
|
-
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true } = {}) {
|
|
61
|
+
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true, semantic = false } = {}) {
|
|
58
62
|
const normalizedQuery = String(query ?? '').trim();
|
|
59
63
|
if (!normalizedQuery) {
|
|
60
64
|
return [];
|
|
@@ -75,11 +79,15 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
75
79
|
query: normalizedQuery,
|
|
76
80
|
limit,
|
|
77
81
|
contentLane,
|
|
82
|
+
semantic: semantic && embedConfig() ? `1:${await semanticStoreSignature(absoluteRoot)}` : 0,
|
|
78
83
|
});
|
|
79
84
|
if (QUERY_RESULT_CACHE.has(queryCacheKey)) {
|
|
80
85
|
return QUERY_RESULT_CACHE.get(queryCacheKey);
|
|
81
86
|
}
|
|
82
87
|
const queryPromise = (async () => {
|
|
88
|
+
// `laneStatus.fellBack`: the lane was asked for (question-shaped, lexical not decisive) and
|
|
89
|
+
// produced nothing (timeout / HTTP error / no vectors). Such a result must not be cached.
|
|
90
|
+
const laneStatus = { fellBack: false };
|
|
83
91
|
const { files, searchDescriptorsByFile, tokenWeights } = await loadQuerySearchBundle(absoluteRoot);
|
|
84
92
|
const preferredArchetypes = inferPreferredArchetypes(queryDescriptor.tokens);
|
|
85
93
|
const includeLikelyTestFiles = shouldIncludeLikelyTestFiles(queryDescriptor);
|
|
@@ -153,7 +161,10 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
153
161
|
}
|
|
154
162
|
|
|
155
163
|
if (directScores.size === 0) {
|
|
156
|
-
|
|
164
|
+
const onlySemantic = semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked: [], includeLikelyTestFiles, files, laneStatus }) : null;
|
|
165
|
+
if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
166
|
+
if (onlySemantic) return withContentLaneStatus(freezeQueryResults(onlySemantic.slice(0, limit)), contentLaneRan, contentCutShort);
|
|
167
|
+
return withContentLaneStatus(Object.freeze([]), contentLaneRan, contentCutShort, laneStatus.fellBack);
|
|
157
168
|
}
|
|
158
169
|
|
|
159
170
|
const {
|
|
@@ -196,12 +207,15 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
196
207
|
}),
|
|
197
208
|
});
|
|
198
209
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
210
|
+
const ranked = [...finalScores.values()]
|
|
211
|
+
.map((entry) => ({ ...entry, score: finalizeScore(entry) }))
|
|
212
|
+
.sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath));
|
|
213
|
+
const fused = contentLane && ranked.length > 0 && ranked[0].score < FUSION_WEAK_SCORE && queryDescriptor.tokens.size >= FUSION_MIN_TOKENS
|
|
214
|
+
? await fuseWeakQuery({ absoluteRoot, query: normalizedQuery, ranked, searchDescriptorsByFile, includeLikelyTestFiles })
|
|
215
|
+
: null;
|
|
216
|
+
const semanticMerged = !fused && semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked, includeLikelyTestFiles, files, laneStatus }) : null;
|
|
217
|
+
if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
218
|
+
return withContentLaneStatus(freezeQueryResults((fused ?? semanticMerged ?? ranked).slice(0, limit)), contentLaneRan, contentCutShort, laneStatus.fellBack);
|
|
205
219
|
})().catch((error) => {
|
|
206
220
|
QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
207
221
|
throw error;
|
|
@@ -211,6 +225,154 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
211
225
|
return queryPromise;
|
|
212
226
|
}
|
|
213
227
|
|
|
228
|
+
// Real-embedding lane: only when the CLI passes semantic:true AND UKIT_INDEX_EMBED_URL is set.
|
|
229
|
+
// Applies to question-shaped queries only; any problem returns null (lexical result unchanged).
|
|
230
|
+
const SEMANTIC_CONFIDENT_MARGIN = 2;
|
|
231
|
+
async function applySemanticLane({ absoluteRoot, query, ranked, includeLikelyTestFiles, files, laneStatus = {} }) {
|
|
232
|
+
try {
|
|
233
|
+
if (!embedConfig() || !looksLikeQuestion(query)) return null;
|
|
234
|
+
// Lexical is already decisive (top-1 clear of top-2): keep it exactly, no network call.
|
|
235
|
+
if (ranked.length > 1 && ranked[0].score - ranked[1].score >= SEMANTIC_CONFIDENT_MARGIN) return null;
|
|
236
|
+
const known = new Set(safeItems(files).map((f) => f?.filePath));
|
|
237
|
+
const candidates = await semanticCandidates({
|
|
238
|
+
rootDir: absoluteRoot,
|
|
239
|
+
query,
|
|
240
|
+
// Vectors can outlive a deleted/renamed file: only paths in the current index qualify.
|
|
241
|
+
accept: (filePath) => known.has(filePath) && (includeLikelyTestFiles || !isLikelyTestFilePath(filePath)),
|
|
242
|
+
weight: mirrorPrior,
|
|
243
|
+
});
|
|
244
|
+
const fused = fuseSemantic({ ranked, candidates });
|
|
245
|
+
if (!fused) laneStatus.fellBack = true;
|
|
246
|
+
return fused;
|
|
247
|
+
} catch {
|
|
248
|
+
laneStatus.fellBack = true;
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Opt-in lane fusion (default OFF): a weak natural-language query (best score below
|
|
254
|
+
// FUSION_WEAK_SCORE, >= FUSION_MIN_TOKENS tokens) is re-ranked by RRF over the scored list
|
|
255
|
+
// and a hashed-embedding lane (the retriever.js vector lane). Enabled by
|
|
256
|
+
// `.ukit/storage/config.json` -> index.fusion === true or UKIT_INDEX_FUSION=1. The flag is
|
|
257
|
+
// read only after the weak gate passes, and contentLane:false callers (hooks/resolveContext)
|
|
258
|
+
// never reach it. Never throws; async fs only.
|
|
259
|
+
const FUSION_WEAK_SCORE = 6;
|
|
260
|
+
const FUSION_MIN_TOKENS = 4;
|
|
261
|
+
const FUSION_RRF_K = 5;
|
|
262
|
+
const FUSION_VECTOR_WEIGHT = 1;
|
|
263
|
+
const FUSION_LANE_LIMIT = 100;
|
|
264
|
+
const FUSION_DOC_VECTORS = new WeakMap();
|
|
265
|
+
|
|
266
|
+
// Inlined copy of src/core/codeintel/vectorProvider.js hashVectorize/cosine (256 dims, FNV-1a
|
|
267
|
+
// word + trigram features, L2-normalized) so the shipped index-core.mjs stays dependency-free.
|
|
268
|
+
function fusionFnv1a(str) {
|
|
269
|
+
let h = 0x811c9dc5;
|
|
270
|
+
for (let i = 0; i < str.length; i += 1) {
|
|
271
|
+
h ^= str.charCodeAt(i);
|
|
272
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
273
|
+
}
|
|
274
|
+
return h >>> 0;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function hashVectorize(text) {
|
|
278
|
+
const dims = 256;
|
|
279
|
+
const vec = new Array(dims).fill(0);
|
|
280
|
+
for (const token of String(text ?? '').split(/[^A-Za-z0-9_$]+/).map((t) => t.toLowerCase()).filter(Boolean)) {
|
|
281
|
+
vec[fusionFnv1a(`w:${token}`) % dims] += 1;
|
|
282
|
+
const padded = `#${token}#`;
|
|
283
|
+
for (let i = 0; i + 3 <= padded.length; i += 1) vec[fusionFnv1a(`t:${padded.slice(i, i + 3)}`) % dims] += 0.1;
|
|
284
|
+
}
|
|
285
|
+
const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
|
|
286
|
+
if (norm > 0) for (let i = 0; i < dims; i += 1) vec[i] /= norm;
|
|
287
|
+
return vec;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function cosineSimilarity(a, b) {
|
|
291
|
+
let dot = 0;
|
|
292
|
+
for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
|
|
293
|
+
return dot;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
async function isFusionEnabled(absoluteRoot) {
|
|
297
|
+
if (process.env.UKIT_INDEX_FUSION === '1') return true;
|
|
298
|
+
if (process.env.UKIT_INDEX_FUSION === '0') return false;
|
|
299
|
+
try {
|
|
300
|
+
const config = JSON.parse(await fs.readFile(path.join(absoluteRoot, '.ukit', 'storage', 'config.json'), 'utf8'));
|
|
301
|
+
return config?.index?.fusion === true;
|
|
302
|
+
} catch {
|
|
303
|
+
return false;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
async function fuseWeakQuery({ absoluteRoot, query, ranked, searchDescriptorsByFile, includeLikelyTestFiles }) {
|
|
308
|
+
try {
|
|
309
|
+
if (!(await isFusionEnabled(absoluteRoot))) return null;
|
|
310
|
+
let docVectors = FUSION_DOC_VECTORS.get(searchDescriptorsByFile);
|
|
311
|
+
if (!docVectors) {
|
|
312
|
+
docVectors = new Map();
|
|
313
|
+
for (const [filePath, descriptor] of searchDescriptorsByFile) {
|
|
314
|
+
docVectors.set(filePath, hashVectorize(`${filePath} ${(descriptor.symbols ?? []).map((s) => s.name).join(' ')}`));
|
|
315
|
+
}
|
|
316
|
+
FUSION_DOC_VECTORS.set(searchDescriptorsByFile, docVectors);
|
|
317
|
+
}
|
|
318
|
+
const queryVector = hashVectorize(query);
|
|
319
|
+
const vectorLane = [];
|
|
320
|
+
for (const [filePath, vector] of docVectors) {
|
|
321
|
+
if (isLikelyTestFilePath(filePath) && !includeLikelyTestFiles) continue;
|
|
322
|
+
const similarity = cosineSimilarity(queryVector, vector);
|
|
323
|
+
if (similarity > 0) vectorLane.push([filePath, similarity]);
|
|
324
|
+
}
|
|
325
|
+
vectorLane.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
|
|
326
|
+
const vectorPaths = vectorLane.slice(0, FUSION_LANE_LIMIT).map(([filePath]) => filePath);
|
|
327
|
+
const indexPaths = ranked.slice(0, FUSION_LANE_LIMIT).map((entry) => entry.filePath);
|
|
328
|
+
const rrf = new Map();
|
|
329
|
+
indexPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + 1 / (FUSION_RRF_K + i + 1)));
|
|
330
|
+
vectorPaths.forEach((filePath, i) => rrf.set(filePath, (rrf.get(filePath) ?? 0) + FUSION_VECTOR_WEIGHT / (FUSION_RRF_K + i + 1)));
|
|
331
|
+
const byPath = new Map(ranked.map((entry) => [entry.filePath, entry]));
|
|
332
|
+
const order = [...rrf.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([filePath]) => filePath);
|
|
333
|
+
const floor = ranked[ranked.length - 1].score;
|
|
334
|
+
const merged = order.map((filePath) => byPath.get(filePath)
|
|
335
|
+
?? { filePath, score: Math.min(0.01, floor), reasons: ['fusion:vector'], tests: [] });
|
|
336
|
+
await emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths: merged.map((e) => e.filePath) });
|
|
337
|
+
return merged;
|
|
338
|
+
} catch {
|
|
339
|
+
return null;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// Same event shape and file as retriever.js emitRetrieverLaneTelemetry (query stored as sha256).
|
|
344
|
+
async function emitFusionTelemetry(absoluteRoot, { query, indexPaths, vectorPaths, mergedPaths }) {
|
|
345
|
+
try {
|
|
346
|
+
const event = {
|
|
347
|
+
ts: Date.now(),
|
|
348
|
+
query: createHash('sha256').update(String(query ?? '')).digest('hex'),
|
|
349
|
+
mode: 'index-query',
|
|
350
|
+
lanes: {
|
|
351
|
+
index: { hits: indexPaths.length, weightUsed: 1, paths: indexPaths },
|
|
352
|
+
vector: { hits: vectorPaths.length, weightUsed: FUSION_VECTOR_WEIGHT, paths: vectorPaths },
|
|
353
|
+
},
|
|
354
|
+
mergedPaths: mergedPaths.slice(0, 100),
|
|
355
|
+
merge: 'rrf',
|
|
356
|
+
};
|
|
357
|
+
const filePath = path.join(absoluteRoot, '.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
|
|
358
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
359
|
+
await withFileLock(filePath, async () => {
|
|
360
|
+
try {
|
|
361
|
+
const st = await fs.stat(filePath);
|
|
362
|
+
if (st.size > 256 * 1024) {
|
|
363
|
+
const kept = (await fs.readFile(filePath, 'utf8')).split('\n').filter(Boolean).slice(-2000);
|
|
364
|
+
await fs.writeFile(filePath, kept.length ? `${kept.join('\n')}\n` : '', 'utf8');
|
|
365
|
+
}
|
|
366
|
+
} catch {
|
|
367
|
+
// Missing file is fine; append below recreates it.
|
|
368
|
+
}
|
|
369
|
+
await fs.appendFile(filePath, `${JSON.stringify(event)}\n`, 'utf8');
|
|
370
|
+
});
|
|
371
|
+
} catch {
|
|
372
|
+
// Telemetry is advisory.
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
214
376
|
// The `callable` symbol regex matches any `const x = (`, so it also captures plain
|
|
215
377
|
// parenthesised expressions such as `const rows = (items ?? [])`. Harmless for fuzzy
|
|
216
378
|
// ranking, misleading in an outline that is read as a map of the file's API. Those rows
|
|
@@ -417,7 +579,7 @@ const BOOST_LIFT_RATIO = 0.2; // import/hotspot/archetype boosts may add at most
|
|
|
417
579
|
const CONTENT_LANE_CONFIDENT_SCORE = 30; // a best direct score at/above this (exact symbol or path hit) skips the content lane
|
|
418
580
|
const CONTENT_LANE_MIN_TOKENS = 3; // shorter queries are names, not messages
|
|
419
581
|
const CONTENT_LANE_LIMIT = 40; // candidates only share tokens somewhere in the file; the same-line check below keeps the real ones
|
|
420
|
-
const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|sql|sh)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
|
|
582
|
+
const CONTENT_LANE_CODE_EXT = /\.(?:m?js|cjs|jsx?|tsx?|vue|svelte|sql|sh|py|go|java|cs|php|rb|rs)$/i; // docs/data files quote queries verbatim (baselines, logs); only source carries error strings
|
|
421
583
|
const CONTENT_LANE_BONUS = 14; // added to a file that already has a direct score
|
|
422
584
|
const CONTENT_LANE_ONLY_SCORE = 13; // base for a file found only by content; below CONTENT_LANE_CONFIDENT_SCORE by design
|
|
423
585
|
const CONTENT_LANE_MAX_FILES = 1500; // files read per query, in priority order (primary bound)
|
|
@@ -1434,9 +1596,11 @@ export function queryMatchedSymbolNames(result) {
|
|
|
1434
1596
|
// Non-enumerable (array equality and JSON stay unchanged), like boundedSourceSearch rows.truncated:
|
|
1435
1597
|
// `contentLane` is true when the content search ran; `contentLaneTruncated` lists why it was cut
|
|
1436
1598
|
// short ('verify-cap', 'time-budget', 'max-files', ...) so a partial scan never reads as "no match".
|
|
1437
|
-
function withContentLaneStatus(results, ran, truncated) {
|
|
1599
|
+
function withContentLaneStatus(results, ran, truncated, semanticFellBack = false) {
|
|
1438
1600
|
// results arrive frozen, so copy before attaching the status and freeze again.
|
|
1439
1601
|
const out = [...results];
|
|
1602
|
+
// true when the embedding lane was asked for and returned nothing: callers must not persist this result.
|
|
1603
|
+
Object.defineProperty(out, 'semanticFellBack', { value: Boolean(semanticFellBack), enumerable: false });
|
|
1440
1604
|
Object.defineProperty(out, 'contentLane', { value: ran, enumerable: false });
|
|
1441
1605
|
Object.defineProperty(out, 'contentLaneTruncated', { value: Object.freeze([...truncated]), enumerable: false });
|
|
1442
1606
|
return Object.freeze(out);
|
|
@@ -273,8 +273,8 @@ export const ROUTE_SHADOW_QUESTIONS = Object.freeze([
|
|
|
273
273
|
// capability.impact.v1 — noul kind; no threshold baseline exists on
|
|
274
274
|
// routeSummary (capabilityPolicy.recommended is unpopulated upstream), so
|
|
275
275
|
// the question has no comparison value.
|
|
276
|
-
// learn.candidate-class.v1 —
|
|
277
|
-
// (
|
|
276
|
+
// learn.candidate-class.v1 — the `learn` family classifies mined lesson candidates
|
|
277
|
+
// (gated by decisionPlane.families.learn.stage, off by default); it is not a route-boundary question.
|
|
278
278
|
]);
|
|
279
279
|
|
|
280
280
|
// Only families whose resolved stage is not 'off' get a question.
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// Real-embedding lane for `ukit index query` (opt-in; no dependency; global fetch only).
|
|
2
|
+
//
|
|
3
|
+
// PRIVACY: building vectors SENDS FILE TEXT (path, symbol names, leading comment and,
|
|
4
|
+
// at most ~240 characters per file) to the endpoint in
|
|
5
|
+
// UKIT_INDEX_EMBED_URL, and queries send the question text. With no URL configured nothing
|
|
6
|
+
// here runs and nothing leaves the machine. Sensitive paths (.env, keys, credentials ...)
|
|
7
|
+
// are never embedded; only files already present in the index are considered.
|
|
8
|
+
//
|
|
9
|
+
// Byte-identical mirror: template_project/.claude/ukit/index/lib/semantic-lane.mjs
|
|
10
|
+
// (tests/index/semanticLane.test.js enforces it). Keep this file dependency-free.
|
|
11
|
+
//
|
|
12
|
+
// Env: UKIT_INDEX_EMBED_URL (base URL ending in /v1), UKIT_INDEX_EMBED_MODEL,
|
|
13
|
+
// UKIT_INDEX_EMBED_KEY (optional Bearer).
|
|
14
|
+
// Build is an explicit command (embed-index.mjs), never part of index build/refresh, never in
|
|
15
|
+
// a hook. Query use is reachable only through queryCodeIndex({ semantic: true }), which only
|
|
16
|
+
// the CLI query path passes.
|
|
17
|
+
import fs from 'node:fs/promises';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import crypto from 'node:crypto';
|
|
20
|
+
|
|
21
|
+
export const SEMANTIC_MANIFEST = 'semantic.json';
|
|
22
|
+
export const SEMANTIC_BIN = 'semantic.bin';
|
|
23
|
+
// A CLI query waits at most this long for the embedding of the question. Measured on the
|
|
24
|
+
// reference endpoint: warm question embedding ~0.45 s, 64 descriptors ~24 s (serial service).
|
|
25
|
+
export const QUERY_TIMEOUT_MS = 1500;
|
|
26
|
+
// One descriptor per file, ~60 tokens. Endpoint throughput is ~170-200 tokens/s and serial, so
|
|
27
|
+
// file bodies (>1 h for 2.4k files) are not embedded; descriptors build in minutes.
|
|
28
|
+
export const DOC_CHARS = 240;
|
|
29
|
+
const NAMES_CHARS = 90;
|
|
30
|
+
const LEAD_CHARS = 100;
|
|
31
|
+
const QUERY_INSTRUCTION = 'Instruct: Given a question about a code repository, retrieve the source file that answers it\nQuery: ';
|
|
32
|
+
|
|
33
|
+
export function embedConfig(env = process.env) {
|
|
34
|
+
// Accepts the base (.../v1) or the full endpoint (.../v1/embeddings).
|
|
35
|
+
const url = String(env.UKIT_INDEX_EMBED_URL ?? '').trim().replace(/\/+$/, '').replace(/\/embeddings$/i, '');
|
|
36
|
+
if (!url) return null;
|
|
37
|
+
return { url, model: String(env.UKIT_INDEX_EMBED_MODEL ?? '').trim() || 'unic-embedding', key: String(env.UKIT_INDEX_EMBED_KEY ?? '').trim() };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function isSensitive(value) {
|
|
41
|
+
const segments = String(value ?? '').replace(/\\/g, '/').split('/');
|
|
42
|
+
const base = segments[segments.length - 1] ?? '';
|
|
43
|
+
return base.toLowerCase().startsWith('.env')
|
|
44
|
+
|| /\.(?:pem|key|p12|pfx|jks|keystore)$/i.test(base)
|
|
45
|
+
|| /^(?:credentials?|\.npmrc|\.netrc|\.pypirc)(?:\.[^/]*)?$/i.test(base)
|
|
46
|
+
|| /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.[^/]*)?$/i.test(base)
|
|
47
|
+
|| segments.includes('.ssh') || segments.includes('.aws') || segments.includes('.gnupg');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const sha = (text) => crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
|
|
51
|
+
|
|
52
|
+
// POST /embeddings; returns arrays in input order or throws. Validates shape.
|
|
53
|
+
export async function embedTexts(texts, cfg, { timeoutMs = 60000 } = {}) {
|
|
54
|
+
const headers = { 'content-type': 'application/json' };
|
|
55
|
+
if (cfg.key) headers.authorization = `Bearer ${cfg.key}`;
|
|
56
|
+
const response = await fetch(`${cfg.url}/embeddings`, {
|
|
57
|
+
method: 'POST',
|
|
58
|
+
headers,
|
|
59
|
+
body: JSON.stringify({ model: cfg.model, input: texts }),
|
|
60
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
61
|
+
});
|
|
62
|
+
if (!response.ok) throw new Error(`embedding endpoint HTTP ${response.status}`);
|
|
63
|
+
const data = (await response.json())?.data;
|
|
64
|
+
if (!Array.isArray(data) || data.length !== texts.length) throw new Error('embedding response shape mismatch');
|
|
65
|
+
const rows = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding);
|
|
66
|
+
const dims = rows[0]?.length;
|
|
67
|
+
if (!dims || rows.some((r) => !Array.isArray(r) || r.length !== dims || r.some((x) => !Number.isFinite(x)))) {
|
|
68
|
+
throw new Error('embedding response vectors invalid');
|
|
69
|
+
}
|
|
70
|
+
return rows;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function normalizeTo(vec, dims) {
|
|
74
|
+
const out = vec.slice(0, dims);
|
|
75
|
+
let n = 0;
|
|
76
|
+
for (const x of out) n += x * x;
|
|
77
|
+
n = Math.sqrt(n) || 1;
|
|
78
|
+
return out.map((x) => x / n);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Descriptor for one file: path + exported/declared symbol names + leading doc comment,
|
|
82
|
+
// capped at DOC_CHARS. The path is never truncated; names and comment share the remainder.
|
|
83
|
+
export function buildDocumentText({ filePath, symbols = [], content = '' }) {
|
|
84
|
+
const names = [...new Set(symbols)].join(' ').slice(0, NAMES_CHARS);
|
|
85
|
+
const lines = String(content).split('\n');
|
|
86
|
+
const lead = [];
|
|
87
|
+
for (const line of lines.slice(0, 80)) {
|
|
88
|
+
const t = line.trim();
|
|
89
|
+
if (!t || t.startsWith('#!') || t === '---' || t.startsWith("'use ")) { if (lead.length) break; continue; }
|
|
90
|
+
if (/^(\/\/|\/\*|\*|#|<!--|"""|--)/.test(t)) lead.push(t.replace(/^(\/\/+|\/\*+|\*+\/?|#+|<!--|-->|"""|--)\s?/, ''));
|
|
91
|
+
else break;
|
|
92
|
+
}
|
|
93
|
+
const leading = (lead.length ? lead.join(' ') : '').slice(0, LEAD_CHARS);
|
|
94
|
+
const text = `${filePath}\n${names}\n${leading}`;
|
|
95
|
+
return text.length > DOC_CHARS ? text.slice(0, DOC_CHARS) : text;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function readJson(file) {
|
|
99
|
+
try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function loadStore(indexDir) {
|
|
103
|
+
const manifest = await readJson(path.join(indexDir, SEMANTIC_MANIFEST));
|
|
104
|
+
if (!manifest || manifest.v !== 1 || !Array.isArray(manifest.order)) return null;
|
|
105
|
+
let bin;
|
|
106
|
+
try { bin = await fs.readFile(path.join(indexDir, SEMANTIC_BIN)); } catch { return null; }
|
|
107
|
+
const rec = 4 + manifest.dims;
|
|
108
|
+
if (bin.length !== rec * manifest.order.length) return null;
|
|
109
|
+
return { manifest, bin, rec };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Explicit build step. Embeds only documents whose hash (document text + model) is not in the
|
|
113
|
+
// existing store. Returns stats. Never called from index build/refresh or hooks.
|
|
114
|
+
export async function buildSemanticVectors({
|
|
115
|
+
rootDir = process.cwd(), env = process.env, dims = 0,
|
|
116
|
+
batchSize = 64, onProgress = null,
|
|
117
|
+
} = {}) {
|
|
118
|
+
const cfg = embedConfig(env);
|
|
119
|
+
if (!cfg) throw new Error('UKIT_INDEX_EMBED_URL is not set; semantic vectors are opt-in');
|
|
120
|
+
const root = path.resolve(rootDir);
|
|
121
|
+
const indexDir = path.join(root, '.cache', 'index');
|
|
122
|
+
const filesArtifact = await readJson(path.join(indexDir, 'files.json'));
|
|
123
|
+
const symbolsArtifact = await readJson(path.join(indexDir, 'symbols.json'));
|
|
124
|
+
if (!filesArtifact?.items) throw new Error('index missing; run the index build first');
|
|
125
|
+
const symbolsByFile = new Map();
|
|
126
|
+
for (const s of symbolsArtifact?.items ?? []) {
|
|
127
|
+
if (!symbolsByFile.has(s.filePath)) symbolsByFile.set(s.filePath, []);
|
|
128
|
+
symbolsByFile.get(s.filePath).push(s.name);
|
|
129
|
+
}
|
|
130
|
+
// Reuse vectors from a store built with the same model and truncation.
|
|
131
|
+
const old = await loadStore(indexDir);
|
|
132
|
+
const reusable = old && old.manifest.model === cfg.model
|
|
133
|
+
&& (!dims || old.manifest.dims === dims) ? old : null;
|
|
134
|
+
const oldByHash = new Map();
|
|
135
|
+
if (reusable) reusable.manifest.order.forEach(([, h], i) => oldByHash.set(h, reusable.bin.subarray(i * reusable.rec, (i + 1) * reusable.rec)));
|
|
136
|
+
|
|
137
|
+
const docs = [];
|
|
138
|
+
for (const item of filesArtifact.items) {
|
|
139
|
+
const filePath = item.filePath;
|
|
140
|
+
if (typeof filePath !== 'string' || isSensitive(filePath)) continue;
|
|
141
|
+
let content = '';
|
|
142
|
+
try { content = await fs.readFile(path.join(root, filePath), 'utf8'); } catch { continue; }
|
|
143
|
+
const text = buildDocumentText({ filePath, symbols: symbolsByFile.get(filePath) ?? [], content });
|
|
144
|
+
docs.push({ filePath, text, hash: sha(`${cfg.model}\0${text}`) });
|
|
145
|
+
}
|
|
146
|
+
const missing = [...new Map(docs.filter((d) => !oldByHash.has(d.hash)).map((d) => [d.hash, d])).values()];
|
|
147
|
+
const fresh = new Map();
|
|
148
|
+
let outDims = dims || reusable?.manifest.dims || 0;
|
|
149
|
+
// Requests are strictly sequential: the embedding service is serial, so parallel requests
|
|
150
|
+
// only queue. Partial progress is saved: a failed batch leaves its files without vectors and
|
|
151
|
+
// the error is rethrown after the store is written, so a rerun only embeds what is missing.
|
|
152
|
+
const failures = [];
|
|
153
|
+
try {
|
|
154
|
+
for (let i = 0; i < missing.length; i += batchSize) {
|
|
155
|
+
const batch = missing.slice(i, i + batchSize);
|
|
156
|
+
const rows = await embedTexts(batch.map((d) => d.text), cfg, { timeoutMs: 180000 });
|
|
157
|
+
if (!outDims) outDims = rows[0].length;
|
|
158
|
+
if (rows[0].length < outDims) throw new Error('endpoint returned fewer dimensions than requested');
|
|
159
|
+
batch.forEach((d, k) => fresh.set(d.hash, rows[k]));
|
|
160
|
+
onProgress?.({ done: fresh.size, total: missing.length });
|
|
161
|
+
}
|
|
162
|
+
} catch (e) { failures.push(e); }
|
|
163
|
+
if (!outDims) outDims = old?.manifest.dims ?? 0;
|
|
164
|
+
if (!outDims && docs.length) throw new Error('no vectors produced');
|
|
165
|
+
|
|
166
|
+
const rec = 4 + outDims;
|
|
167
|
+
const order = docs.filter((d) => fresh.has(d.hash) || oldByHash.has(d.hash)).map((d) => [d.filePath, d.hash]);
|
|
168
|
+
const bin = Buffer.alloc(rec * order.length);
|
|
169
|
+
order.forEach(([, hash], i) => {
|
|
170
|
+
const off = i * rec;
|
|
171
|
+
const prior = oldByHash.get(hash);
|
|
172
|
+
if (prior) { prior.copy(bin, off); return; }
|
|
173
|
+
const v = normalizeTo(fresh.get(hash), outDims);
|
|
174
|
+
const maxAbs = Math.max(...v.map(Math.abs)) || 1;
|
|
175
|
+
bin.writeFloatLE(maxAbs / 127, off);
|
|
176
|
+
for (let k = 0; k < outDims; k += 1) bin.writeInt8(Math.max(-127, Math.min(127, Math.round(v[k] / maxAbs * 127))), off + 4 + k);
|
|
177
|
+
});
|
|
178
|
+
const manifest = { v: 1, model: cfg.model, dims: outDims, createdAt: new Date().toISOString(), order };
|
|
179
|
+
const tmp = `.tmp-${process.pid}`;
|
|
180
|
+
await fs.writeFile(path.join(indexDir, SEMANTIC_BIN + tmp), bin);
|
|
181
|
+
await fs.writeFile(path.join(indexDir, SEMANTIC_MANIFEST + tmp), JSON.stringify(manifest));
|
|
182
|
+
await fs.rename(path.join(indexDir, SEMANTIC_BIN + tmp), path.join(indexDir, SEMANTIC_BIN));
|
|
183
|
+
await fs.rename(path.join(indexDir, SEMANTIC_MANIFEST + tmp), path.join(indexDir, SEMANTIC_MANIFEST));
|
|
184
|
+
if (failures.length) throw new Error(`partial store saved (${order.length}/${docs.length} files); rerun to resume: ${failures[0]?.message ?? failures[0]}`);
|
|
185
|
+
return { files: docs.length, embedded: fresh.size, reused: docs.length - missing.length, dims: outDims, bytes: bin.length, model: cfg.model };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Cache-key part: changes when the vector store is built, rebuilt or removed, so a result cached
|
|
189
|
+
// before `embed-index` ran never outlives the vectors. Never throws.
|
|
190
|
+
export async function semanticStoreSignature(rootDir) {
|
|
191
|
+
try {
|
|
192
|
+
const stat = await fs.stat(path.join(path.resolve(rootDir), '.cache', 'index', SEMANTIC_MANIFEST));
|
|
193
|
+
return `${stat.mtimeMs}:${stat.size}`;
|
|
194
|
+
} catch {
|
|
195
|
+
return 'none';
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const STORE_CACHE = new Map();
|
|
200
|
+
|
|
201
|
+
// Query-side ranking. Returns [{ filePath, similarity }] (best first) or null on ANY problem
|
|
202
|
+
// (lane off, no vectors, model/dimension mismatch, timeout, HTTP error). Never throws.
|
|
203
|
+
export async function semanticCandidates({
|
|
204
|
+
rootDir, query, env = process.env, timeoutMs = QUERY_TIMEOUT_MS, limit = 100,
|
|
205
|
+
queryInstruction = QUERY_INSTRUCTION, accept = () => true, weight = () => 1,
|
|
206
|
+
} = {}) {
|
|
207
|
+
try {
|
|
208
|
+
const cfg = embedConfig(env);
|
|
209
|
+
if (!cfg) return null;
|
|
210
|
+
const indexDir = path.join(path.resolve(rootDir), '.cache', 'index');
|
|
211
|
+
const stat = await fs.stat(path.join(indexDir, SEMANTIC_MANIFEST));
|
|
212
|
+
const key = `${indexDir}:${stat.mtimeMs}:${stat.size}`;
|
|
213
|
+
let store = STORE_CACHE.get(key);
|
|
214
|
+
if (!store) { store = await loadStore(indexDir); if (!store) return null; STORE_CACHE.clear(); STORE_CACHE.set(key, store); }
|
|
215
|
+
const { manifest, bin, rec } = store;
|
|
216
|
+
if (manifest.model !== cfg.model) return null;
|
|
217
|
+
const [raw] = await embedTexts([`${queryInstruction}${query}`], cfg, { timeoutMs });
|
|
218
|
+
if (raw.length < manifest.dims) return null;
|
|
219
|
+
const q = normalizeTo(raw, manifest.dims);
|
|
220
|
+
const scored = [];
|
|
221
|
+
for (let i = 0; i < manifest.order.length; i += 1) {
|
|
222
|
+
const filePath = manifest.order[i][0];
|
|
223
|
+
if (!accept(filePath)) continue;
|
|
224
|
+
const off = i * rec;
|
|
225
|
+
let dot = 0;
|
|
226
|
+
for (let k = 0; k < manifest.dims; k += 1) dot += q[k] * bin.readInt8(off + 4 + k);
|
|
227
|
+
scored.push({ filePath, similarity: dot * bin.readFloatLE(off) * weight(filePath) });
|
|
228
|
+
}
|
|
229
|
+
scored.sort((a, b) => b.similarity - a.similarity || a.filePath.localeCompare(b.filePath));
|
|
230
|
+
return scored.slice(0, limit);
|
|
231
|
+
} catch {
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Plain-language gate: >= 4 words, no path separators or code punctuation, no quotes/colons,
|
|
237
|
+
// no trailing period (a finished statement is error-message shaped), no error-leading word, and
|
|
238
|
+
// no identifier-shaped token (camelCase, snake_case, kebab with digits, dotted name, file
|
|
239
|
+
// extension, token containing a digit, ALL-CAPS code such as ENOENT). Non-ASCII text
|
|
240
|
+
// (Vietnamese) and ordinary lowercase sentences pass; everything else keeps the lexical result.
|
|
241
|
+
const ERROR_LEADS = /^(?:error|errors|failed|failure|cannot|can't|unable|exception|invalid|unexpected|uncaught|warning|fatal|missing|undefined|null|typeerror|referenceerror)\b/i;
|
|
242
|
+
function identifierShaped(token) {
|
|
243
|
+
const t = token.replace(/^[^\w]+|[^\w]+$/g, '');
|
|
244
|
+
if (!t) return false;
|
|
245
|
+
return /[a-z][A-Z]/.test(t) || /_/.test(t) || /\d/.test(t) || /\w\.\w/.test(t) || /\w-\w*\d/.test(t)
|
|
246
|
+
|| /^[A-Z]{3,}$/.test(t) || /^[A-Z][a-z]+[A-Z]/.test(t) || /\.(?:js|mjs|cjs|ts|tsx|jsx|json|md|py|go|rs|java|rb|yml|yaml|sh)\b/i.test(token);
|
|
247
|
+
}
|
|
248
|
+
export function looksLikeQuestion(query) {
|
|
249
|
+
const q = String(query ?? '').trim();
|
|
250
|
+
const words = q.split(/\s+/).filter(Boolean);
|
|
251
|
+
if (words.length < 4 || /[\\/]|[(){}\[\]=;<>@#$%^&*|~`'"]|::|=>|:/.test(q.replace(/[^\x00-\x7f]/g, ''))) return false;
|
|
252
|
+
if (/[^\x20-\x7e]/.test(q)) return true;
|
|
253
|
+
if (/\.\s*$/.test(q) || ERROR_LEADS.test(q) || words.some(identifierShaped)) return false;
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Combine: weighted RRF over the lexical ranking and the semantic list. Semantic-only files
|
|
258
|
+
// are injected with a floor score. Returns null (caller keeps lexical) when no candidates.
|
|
259
|
+
export function fuseSemantic({ ranked, candidates, rrfK = 5, weight = 1, depth = 100 }) {
|
|
260
|
+
if (!candidates?.length) return null;
|
|
261
|
+
const rrf = new Map();
|
|
262
|
+
ranked.slice(0, depth).forEach((e, i) => rrf.set(e.filePath, 1 / (rrfK + i + 1)));
|
|
263
|
+
candidates.slice(0, depth).forEach((c, i) => rrf.set(c.filePath, (rrf.get(c.filePath) ?? 0) + weight / (rrfK + i + 1)));
|
|
264
|
+
const byPath = new Map(ranked.map((e) => [e.filePath, e]));
|
|
265
|
+
const floor = ranked.length ? ranked[ranked.length - 1].score : 0.01;
|
|
266
|
+
return [...rrf.entries()]
|
|
267
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
268
|
+
.map(([filePath]) => byPath.get(filePath) ?? { filePath, score: Math.min(0.01, floor), reasons: ['semantic:vector'], tests: [] });
|
|
269
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// lessonClass.js — is a mined pattern a real recurring lesson, or noise?
|
|
2
|
+
//
|
|
3
|
+
// Deterministic rules are the floor and the fallback. For the candidates the
|
|
4
|
+
// rules leave 'ambiguous', the decision plane (`unic-decision`, the owner's
|
|
5
|
+
// local decision model, reached through src/decision/client.js) is asked one
|
|
6
|
+
// yes/no question each. Its answer can only DOWNGRADE a candidate to 'noise';
|
|
7
|
+
// it never approves or promotes anything. Stage off, provider unreachable,
|
|
8
|
+
// malformed answer, timeout or throw -> the floor result stands.
|
|
9
|
+
//
|
|
10
|
+
// Call sites: `ukit memory learn` and the session-end self-improve pass only.
|
|
11
|
+
// Never from a deadline-armed hook.
|
|
12
|
+
|
|
13
|
+
import { resolveConfigStage } from '../core/runtimeConfig.js';
|
|
14
|
+
import { createDecisionClient } from '../decision/client.js';
|
|
15
|
+
|
|
16
|
+
export const LESSON_DECISION_KEY = 'learn.candidate-class.v1';
|
|
17
|
+
export const LESSON_TIMEOUT_MS = 2000;
|
|
18
|
+
const MAX_ASKED = 8;
|
|
19
|
+
const TEXT_CHARS = 200;
|
|
20
|
+
|
|
21
|
+
export function resolveLessonStage(config) {
|
|
22
|
+
return resolveConfigStage(config, 'decisionPlane.families.learn.stage');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Failure patterns: runners that verify work are lessons; shell navigation is noise.
|
|
26
|
+
const RUNNER_RE = /^(yarn|npm|pnpm|npx|vitest|jest|pytest|cargo|go|make|tsc|eslint)\b/;
|
|
27
|
+
const TRIVIA_RE = /^(cd|ls|pwd|echo|cat|head|tail|grep|find|which|true|false)\b/;
|
|
28
|
+
|
|
29
|
+
export function classifyPattern(pattern) {
|
|
30
|
+
const sig = String(pattern?.signature ?? '').trim().toLowerCase();
|
|
31
|
+
if (!sig || TRIVIA_RE.test(sig)) return 'noise';
|
|
32
|
+
return RUNNER_RE.test(sig) ? 'lesson' : 'ambiguous';
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Re-edited files: product source is a lesson, data files are noise, the rest is unclear.
|
|
36
|
+
// Current layout: src/, bin/, scripts/ and the shipped template runtime. (The old
|
|
37
|
+
// templates/.claude/** layout is gone and is dropped by dropMissingFiles first.)
|
|
38
|
+
const SOURCE_RE = /^(src|lib|bin|scripts|template_project\/\.claude\/(hooks|ukit))\/.*\.(js|mjs|cjs|ts|tsx|sh|py)$/;
|
|
39
|
+
const DATA_RE = /\.(json|ya?ml|lock|txt|csv|svg|png)$/;
|
|
40
|
+
|
|
41
|
+
export function classifyReedit(reedit) {
|
|
42
|
+
const file = String(reedit?.file ?? '');
|
|
43
|
+
if (!file || DATA_RE.test(file)) return 'noise';
|
|
44
|
+
return SOURCE_RE.test(file) ? 'lesson' : 'ambiguous';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const oneLine = (text) => String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, TEXT_CHARS);
|
|
48
|
+
|
|
49
|
+
function withTimeout(promise, ms) {
|
|
50
|
+
let timer;
|
|
51
|
+
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ status: 'timeout', answers: [] }), ms); });
|
|
52
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* @param {Array<{id:string, text:string, cls:'lesson'|'noise'|'ambiguous'}>} items
|
|
57
|
+
* @returns {Promise<{items: Array, meta: {stage, asked, answered, changed, applied, status}}>}
|
|
58
|
+
* `answered` counts real valid provider responses, so callers can tell a run
|
|
59
|
+
* that used the decision plane from one that fell back.
|
|
60
|
+
*/
|
|
61
|
+
export async function refineWithDecision(items, {
|
|
62
|
+
client, stage = 'off', timeoutMs = LESSON_TIMEOUT_MS,
|
|
63
|
+
} = {}) {
|
|
64
|
+
const meta = { stage, asked: 0, answered: 0, changed: 0, applied: false, status: 'skipped' };
|
|
65
|
+
const list = Array.isArray(items) ? items : [];
|
|
66
|
+
try {
|
|
67
|
+
if (stage === 'off' || !client) return { items: list, meta };
|
|
68
|
+
const ambiguous = list.filter((item) => item?.cls === 'ambiguous').slice(0, MAX_ASKED);
|
|
69
|
+
if (ambiguous.length === 0) return { items: list, meta };
|
|
70
|
+
meta.asked = ambiguous.length;
|
|
71
|
+
const result = await withTimeout(client.requestBatch({
|
|
72
|
+
statePacket: { task: 'learn-candidate-class', candidateCount: ambiguous.length },
|
|
73
|
+
questions: ambiguous.map((item, i) => ({
|
|
74
|
+
decisionKey: `${LESSON_DECISION_KEY}#c${i + 1}`,
|
|
75
|
+
schemaVersion: 1,
|
|
76
|
+
family: 'learn',
|
|
77
|
+
owner: 'lessonClass',
|
|
78
|
+
kind: 'noul',
|
|
79
|
+
instruction: `Is this a real recurring problem worth remembering for future edits? ${oneLine(item.text)}`,
|
|
80
|
+
candidates: [],
|
|
81
|
+
})),
|
|
82
|
+
deadlineMs: timeoutMs,
|
|
83
|
+
}), timeoutMs);
|
|
84
|
+
meta.status = result?.status ?? 'invalid';
|
|
85
|
+
const downgrade = new Set();
|
|
86
|
+
for (const answer of result?.answers ?? []) {
|
|
87
|
+
if (answer?.validationStatus !== 'valid' || !Number.isFinite(answer.value)) continue;
|
|
88
|
+
const idx = Number(String(answer.decisionKey).split('#c')[1]);
|
|
89
|
+
if (!Number.isInteger(idx) || idx < 1 || idx > ambiguous.length) continue;
|
|
90
|
+
meta.answered += 1;
|
|
91
|
+
if (answer.value < 0.5) downgrade.add(ambiguous[idx - 1]);
|
|
92
|
+
}
|
|
93
|
+
if (stage === 'shadow' || downgrade.size === 0) return { items: list, meta };
|
|
94
|
+
meta.changed = downgrade.size;
|
|
95
|
+
meta.applied = true;
|
|
96
|
+
return { items: list.map((item) => (downgrade.has(item) ? { ...item, cls: 'noise' } : item)), meta };
|
|
97
|
+
} catch {
|
|
98
|
+
meta.status = 'error';
|
|
99
|
+
return { items: list, meta };
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function createLessonClient({ config, projectRoot, homeDir, transport } = {}) {
|
|
104
|
+
if (resolveLessonStage(config) === 'off') return null;
|
|
105
|
+
try {
|
|
106
|
+
return createDecisionClient({ config, projectRoot, homeDir, transport });
|
|
107
|
+
} catch {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
}
|