@ngockhoale/ukit 3.5.5 → 3.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/manifests/engineConformance.yaml +0 -13
- package/manifests/platform.full.yaml +25 -0
- package/package.json +1 -8
- package/src/cli/commands/indexTools.js +1 -0
- package/src/cli/commands/memory.js +33 -4
- package/src/core/applyPlan.js +25 -72
- package/src/core/diffPlan.js +38 -9
- package/src/core/managedTemplateMerge.js +146 -0
- package/src/core/memory/memoryFlags.js +5 -6
- package/src/core/memory/policy.js +5 -0
- package/src/core/memory/store.js +17 -0
- package/src/core/metadata.js +12 -8
- package/src/core/runInstallPipeline.js +16 -10
- package/src/core/runtimeConfig.js +14 -83
- package/src/core/uninstall.js +8 -1
- package/src/decision/registry.js +17 -17
- package/src/decision/runtimeDecide.js +5 -5
- package/src/index/queryIndex.js +41 -4
- package/src/index/routeResolver.js +2 -2
- package/src/index/semanticLane.js +269 -0
- package/src/learning/lessonClass.js +110 -0
- package/src/learning/patternProposals.js +111 -7
- package/src/learning/reeditMiner.js +104 -0
- package/src/learning/selfImprove.js +9 -1
- package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
- package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
- package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
- package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
- package/template_project/.claude/ukit/index/lib/index-core.mjs +43 -4
- package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
- package/template_project/.claude/ukit/index/query-index.mjs +9 -1
- package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
- package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
- package/template_project/ukit/storage/config.json +0 -32
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
- package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
|
@@ -44,6 +44,7 @@ import { collectFeedbackEvents } from '../diagnostics/feedbackEvents.js';
|
|
|
44
44
|
import { collectSkillAccuracy } from '../diagnostics/skillAccuracy.js';
|
|
45
45
|
import { computeTuningSuggestions } from './tuning.js';
|
|
46
46
|
import { proposeFromPatterns } from './patternProposals.js';
|
|
47
|
+
import { createLessonClient, resolveLessonStage } from './lessonClass.js';
|
|
47
48
|
import { proposeCodeFixTasks } from './codeProposals.js';
|
|
48
49
|
import {
|
|
49
50
|
TUNABLE_KEYS,
|
|
@@ -253,10 +254,17 @@ export async function runSelfImprove(projectRoot, {
|
|
|
253
254
|
steps.push(await step('proposals', async () => {
|
|
254
255
|
const projectId = (await detectProjectContext(root, { homeDir })).project.name;
|
|
255
256
|
const minCount = Number(config?.learning?.proposals?.minCount) || undefined;
|
|
256
|
-
|
|
257
|
+
// Session end is an allowed call site for the decision plane; stage off
|
|
258
|
+
// (the default) or an unreachable provider leaves the rule result.
|
|
259
|
+
const stage = resolveLessonStage(config);
|
|
260
|
+
const client = createLessonClient({ config, projectRoot: root, homeDir });
|
|
261
|
+
const res = await proposeFromPatterns(root, projectId, { minCount, dryRun: false, client, stage });
|
|
257
262
|
return {
|
|
258
263
|
status: res.error ? 'failed' : 'ok',
|
|
259
264
|
proposed: res.proposed.filter((p) => p.status === 'proposed').length,
|
|
265
|
+
...(res.decision && res.decision.stage !== 'off'
|
|
266
|
+
? { decision: { asked: res.decision.asked, answered: res.decision.answered, changed: res.decision.changed } }
|
|
267
|
+
: {}),
|
|
260
268
|
reason: res.error,
|
|
261
269
|
};
|
|
262
270
|
}));
|
|
@@ -87,9 +87,9 @@ With both the philosophy and the conceptual framework established, express it on
|
|
|
87
87
|
|
|
88
88
|
To create museum or magazine quality work, use the design philosophy as the foundation. Create one single page, highly visual, design-forward PDF or PNG output (unless asked for more pages). Generally use repeating patterns and perfect shapes. Treat the abstract philosophical design as if it were a scientific bible, borrowing the visual language of systematic observation—dense accumulation of marks, repeated elements, or layered patterns that build meaning through patient repetition and reward sustained viewing. Add sparse, clinical typography and systematic reference markers that suggest this could be a diagram from an imaginary discipline, treating the invisible subject with the same reverence typically reserved for documenting observable phenomena. Anchor the piece with simple phrase(s) or details positioned subtly, using a limited color palette that feels intentional and cohesive. Embrace the paradox of using analytical visual language to express ideas about human experience: the result should feel like an artifact that proves something ephemeral can be studied, mapped, and understood through careful attention. This is true art.
|
|
89
89
|
|
|
90
|
-
**Text as a contextual element**: Text is always minimal and visual-first, but let context guide whether that means whisper-quiet labels or bold typographic gestures. A punk venue poster might have larger, more aggressive type than a minimalist ceramics studio identity. Most of the time, font should be thin. All use of fonts must be design-forward and prioritize visual communication. Regardless of text scale, nothing falls off the page and nothing overlaps. Every element must be contained within the canvas boundaries with proper margins. Check carefully that all text, graphics, and visual elements have breathing room and clear separation. This is non-negotiable for professional execution. **IMPORTANT: Use different fonts if writing text. Start with the curated built-in font set in `./canvas-fonts
|
|
90
|
+
**Text as a contextual element**: Text is always minimal and visual-first, but let context guide whether that means whisper-quiet labels or bold typographic gestures. A punk venue poster might have larger, more aggressive type than a minimalist ceramics studio identity. Most of the time, font should be thin. All use of fonts must be design-forward and prioritize visual communication. Regardless of text scale, nothing falls off the page and nothing overlaps. Every element must be contained within the canvas boundaries with proper margins. Check carefully that all text, graphics, and visual elements have breathing room and clear separation. This is non-negotiable for professional execution. **IMPORTANT: Use different fonts if writing text. Start with the curated built-in font set in `./canvas-fonts`: Outfit (sans), Instrument Serif (serif), JetBrains Mono (mono), Tektur (display), Nothing You Could Do (script). Only widen beyond that when the task truly needs it: more faces (Bricolage Grotesque, Instrument Sans, Lora, Young Serif, plus bold/italic variable files) can be downloaded with `node ./scripts/fetch-fonts.mjs --list` / `node ./scripts/fetch-fonts.mjs <Family>` when the network is available. Regardless of approach, sophistication is non-negotiable.**
|
|
91
91
|
|
|
92
|
-
Prefer the bundled curated font set first so the skill stays fast, portable, and production-friendly after a simple install. Get creative by making the typography actually part of the art itself -- if the art is abstract, bring the font onto the canvas, not typeset digitally.
|
|
92
|
+
Prefer the bundled curated font set first so the skill stays fast, portable, and production-friendly after a simple install; if the network is unavailable, stay with the bundled faces. Get creative by making the typography actually part of the art itself -- if the art is abstract, bring the font onto the canvas, not typeset digitally.
|
|
93
93
|
|
|
94
94
|
To push boundaries, follow design instinct/intuition while using the philosophy as a guiding principle. Embrace ultimate design freedom and choice. Push aesthetics and design to the frontier.
|
|
95
95
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
Copyright 2022 The Instrument
|
|
1
|
+
Copyright 2022 The Instrument Serif Project Authors (https://github.com/Instrument/instrument-serif)
|
|
2
2
|
|
|
3
3
|
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
|
4
4
|
This license is copied below, and is also available with a FAQ at:
|
|
5
|
-
https://
|
|
5
|
+
https://scripts.sil.org/OFL
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
-----------------------------------------------------------
|
|
@@ -18,7 +18,7 @@ with others.
|
|
|
18
18
|
|
|
19
19
|
The OFL allows the licensed fonts to be used, studied, modified and
|
|
20
20
|
redistributed freely as long as they are not sold by themselves. The
|
|
21
|
-
fonts, including any derivative works, can be bundled, embedded,
|
|
21
|
+
fonts, including any derivative works, can be bundled, embedded,
|
|
22
22
|
redistributed and/or sold with any software provided that any reserved
|
|
23
23
|
names are not used by derivative works. The fonts and derivatives,
|
|
24
24
|
however, cannot be released under any other type of license. The
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Fetch optional canvas-design fonts on demand (dependency-free, OFL fonts only).
|
|
3
|
+
// Usage: node fetch-fonts.mjs --list | --all | <Family> [<Family> ...]
|
|
4
|
+
// Source: google/fonts at a pinned commit (immutable raw URLs). Every file is
|
|
5
|
+
// SHA-256 verified before it is written into ../canvas-fonts/.
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import { mkdir, rename, writeFile, access, unlink } from 'node:fs/promises';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import { fileURLToPath } from 'node:url';
|
|
10
|
+
|
|
11
|
+
export const PINNED_SHA = 'cc39c4c2c304bc368c9b7c311ea2bcc6013299fe';
|
|
12
|
+
export const REQUEST_TIMEOUT_MS = 30_000;
|
|
13
|
+
const RAW = `https://raw.githubusercontent.com/google/fonts/${PINNED_SHA}/ofl`;
|
|
14
|
+
|
|
15
|
+
const url = (dir, file) =>
|
|
16
|
+
`${RAW}/${dir}/${file.replace(/\[/g, '%5B').replace(/\]/g, '%5D').replace(/,/g, '%2C')}`;
|
|
17
|
+
|
|
18
|
+
// local = file name written into canvas-fonts/. Variable fonts cover all weights.
|
|
19
|
+
const FAMILIES = {
|
|
20
|
+
BricolageGrotesque: {
|
|
21
|
+
dir: 'bricolagegrotesque',
|
|
22
|
+
licenseSha256: '4b5a7d8f37f5602621c8a8d7358a6a2e71317e6c231c661e15aef0275d3e07ba',
|
|
23
|
+
files: [{ local: 'BricolageGrotesque-Variable.ttf', remote: 'BricolageGrotesque[opsz,wdth,wght].ttf', sha256: '413e7357809ddd12fd80a96a8a396de0e401638d4acd3cb3e37532f0472ac682' }],
|
|
24
|
+
},
|
|
25
|
+
InstrumentSans: {
|
|
26
|
+
dir: 'instrumentsans',
|
|
27
|
+
licenseSha256: '9e27a72ed30eb49a08678f6a5d6ed98ec7ba5368f541637ee0683ec9134ef966',
|
|
28
|
+
files: [
|
|
29
|
+
{ local: 'InstrumentSans-Variable.ttf', remote: 'InstrumentSans[wdth,wght].ttf', sha256: 'b24f1812584816958afcf22e22d08e44318c5e51651e25d2438efdde389b33b1' },
|
|
30
|
+
{ local: 'InstrumentSans-Italic-Variable.ttf', remote: 'InstrumentSans-Italic[wdth,wght].ttf', sha256: 'a74203cc5066a3b2f8de1a7b0887ef897773c2319dc86c911f8e85350cde0d07' },
|
|
31
|
+
],
|
|
32
|
+
},
|
|
33
|
+
InstrumentSerif: {
|
|
34
|
+
dir: 'instrumentserif',
|
|
35
|
+
licenseSha256: '129ed7618959716959f2941fdd5b49e0ad6e6c1d78726761786a00253d865521',
|
|
36
|
+
files: [{ local: 'InstrumentSerif-Italic.ttf', remote: 'InstrumentSerif-Italic.ttf', sha256: '08939b8bdf534afec24ae0ef5e03f948940cd9a8fe08e7fecbad040e62327385' }],
|
|
37
|
+
},
|
|
38
|
+
JetBrainsMono: {
|
|
39
|
+
dir: 'jetbrainsmono',
|
|
40
|
+
licenseSha256: 'b2fe5e8987594e9ffd1d2ca52a2f5d73eb8335243893c5d6254b5ad69269591d',
|
|
41
|
+
files: [
|
|
42
|
+
{ local: 'JetBrainsMono-Variable.ttf', remote: 'JetBrainsMono[wght].ttf', sha256: '48715a42ec242c21e9f02692891e147d022299a52e48d5e413e1a942193ffeda' },
|
|
43
|
+
{ local: 'JetBrainsMono-Italic-Variable.ttf', remote: 'JetBrainsMono-Italic[wght].ttf', sha256: '85ae2a5cd3f56baf1ce1c21a851322c58e3d8fbe8e8ad4a4d090a820dd7fe558' },
|
|
44
|
+
],
|
|
45
|
+
},
|
|
46
|
+
Lora: {
|
|
47
|
+
dir: 'lora',
|
|
48
|
+
licenseSha256: '1d9a970809ac804b582a6ce7f0ebc4e7fefcbfd7ff6299cad35ee656a21be716',
|
|
49
|
+
files: [
|
|
50
|
+
{ local: 'Lora-Variable.ttf', remote: 'Lora[wght].ttf', sha256: '822a6621ccbe8d97d20ac88c1c41f5615c9c2c202eaa75f272cd452aac6475a7' },
|
|
51
|
+
{ local: 'Lora-Italic-Variable.ttf', remote: 'Lora-Italic[wght].ttf', sha256: '22d8d8854b53807aa664ca34f2031a9ed57a1d0dea296b8b96cdd3aad937a2b3' },
|
|
52
|
+
],
|
|
53
|
+
},
|
|
54
|
+
Outfit: {
|
|
55
|
+
dir: 'outfit',
|
|
56
|
+
licenseSha256: 'c676351bf8576b9aba743cd5eaa8c0e7ee0d51f805d720447b4df4ddb6a2e416',
|
|
57
|
+
files: [{ local: 'Outfit-Variable.ttf', remote: 'Outfit[wght].ttf', sha256: 'fc7287273e66929776e2ba54f144fe699080bec29f61bf649d70d871468aeade' }],
|
|
58
|
+
},
|
|
59
|
+
Tektur: {
|
|
60
|
+
dir: 'tektur',
|
|
61
|
+
licenseSha256: 'ff4bb37f716836feb5038e1bf5c47e5a1946717fcfabb175086ed49f85ea47fd',
|
|
62
|
+
files: [{ local: 'Tektur-Variable.ttf', remote: 'Tektur[wdth,wght].ttf', sha256: 'a4c451e5336874811267c2a514efa09bca7bc8cabf32a310f0b44e111eacb933' }],
|
|
63
|
+
},
|
|
64
|
+
YoungSerif: {
|
|
65
|
+
dir: 'youngserif',
|
|
66
|
+
licenseSha256: '737d8d343b674c6894f373de71a044b6782c130f2aebf180933d63ddf7a3dc19',
|
|
67
|
+
files: [{ local: 'YoungSerif-Regular.ttf', remote: 'YoungSerif-Regular.ttf', sha256: '42bb2602364b28569fdce5eeb6ab3d81cf8090ba81b211f1229ed69f33155523' }],
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
/** Flat manifest: [{ family, local, url, sha256, license }] */
|
|
72
|
+
export function buildManifest() {
|
|
73
|
+
const out = [];
|
|
74
|
+
for (const [family, def] of Object.entries(FAMILIES)) {
|
|
75
|
+
for (const f of def.files) {
|
|
76
|
+
out.push({ family, local: f.local, url: url(def.dir, f.remote), sha256: f.sha256, license: false });
|
|
77
|
+
}
|
|
78
|
+
out.push({ family, local: `${family}-OFL.txt`, url: url(def.dir, 'OFL.txt'), sha256: def.licenseSha256, license: true });
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function familyNames() {
|
|
84
|
+
return Object.keys(FAMILIES);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Resolve a local name inside fontsDir; throws if it would escape. */
|
|
88
|
+
export function safeTarget(fontsDir, local) {
|
|
89
|
+
const root = path.resolve(fontsDir);
|
|
90
|
+
const target = path.resolve(root, local);
|
|
91
|
+
if (path.dirname(target) !== root || local !== path.basename(local)) {
|
|
92
|
+
throw new Error(`refusing to write outside canvas-fonts: ${local}`);
|
|
93
|
+
}
|
|
94
|
+
return target;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function download(u) {
|
|
98
|
+
const res = await fetch(u, { signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), redirect: 'follow' });
|
|
99
|
+
if (!res.ok) throw new Error(`HTTP ${res.status} for ${u}`);
|
|
100
|
+
return Buffer.from(await res.arrayBuffer());
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function exists(p) {
|
|
104
|
+
try { await access(p); return true; } catch { return false; }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function main(argv) {
|
|
108
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
109
|
+
const fontsDir = path.resolve(here, '..', 'canvas-fonts');
|
|
110
|
+
const names = familyNames();
|
|
111
|
+
if (argv.includes('--list') || argv.length === 0) {
|
|
112
|
+
console.log('Available families (OFL, google/fonts @ ' + PINNED_SHA.slice(0, 12) + '):\n ' + names.join('\n '));
|
|
113
|
+
console.log('\nUsage: node fetch-fonts.mjs <Family...> | --all');
|
|
114
|
+
return 0;
|
|
115
|
+
}
|
|
116
|
+
const wanted = argv.includes('--all') ? names : argv;
|
|
117
|
+
const unknown = wanted.filter((n) => !names.includes(n));
|
|
118
|
+
if (unknown.length) {
|
|
119
|
+
console.error(`Unknown family: ${unknown.join(', ')}. Use --list.`);
|
|
120
|
+
return 1;
|
|
121
|
+
}
|
|
122
|
+
await mkdir(fontsDir, { recursive: true });
|
|
123
|
+
for (const entry of buildManifest().filter((e) => wanted.includes(e.family))) {
|
|
124
|
+
const target = safeTarget(fontsDir, entry.local);
|
|
125
|
+
// Keep an already bundled licence file; it is the same OFL family text.
|
|
126
|
+
if (entry.license && (await exists(target))) continue;
|
|
127
|
+
const data = await download(entry.url);
|
|
128
|
+
const got = createHash('sha256').update(data).digest('hex');
|
|
129
|
+
if (got !== entry.sha256) throw new Error(`SHA-256 mismatch for ${entry.local}: expected ${entry.sha256}, got ${got}`);
|
|
130
|
+
const tmp = `${target}.${process.pid}.tmp`;
|
|
131
|
+
try {
|
|
132
|
+
await writeFile(tmp, data);
|
|
133
|
+
await rename(tmp, target);
|
|
134
|
+
} catch (err) {
|
|
135
|
+
await unlink(tmp).catch(() => {});
|
|
136
|
+
throw err;
|
|
137
|
+
}
|
|
138
|
+
console.log(`fetched ${entry.local}`);
|
|
139
|
+
}
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
144
|
+
main(process.argv.slice(2)).then(
|
|
145
|
+
(code) => process.exit(code),
|
|
146
|
+
(err) => { console.error(`fetch-fonts failed: ${err.message}`); process.exit(1); },
|
|
147
|
+
);
|
|
148
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Explicit, opt-in step: embed one short descriptor per indexed file through UKIT_INDEX_EMBED_URL
|
|
3
|
+
// (OpenAI-compatible /v1/embeddings). PRIVACY: this sends file text (path, symbol names, leading
|
|
4
|
+
// comment; at most ~240 characters per file) to that endpoint. Never part of index build/refresh
|
|
5
|
+
// and never run from a hook. Usage: node .claude/ukit/index/embed-index.mjs [--dims N] [--root DIR]
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { buildSemanticVectors, embedConfig } from './lib/semantic-lane.mjs';
|
|
8
|
+
|
|
9
|
+
const args = process.argv.slice(2);
|
|
10
|
+
const flag = (name, fallback) => {
|
|
11
|
+
const i = args.indexOf(name);
|
|
12
|
+
return i >= 0 && args[i + 1] ? args[i + 1] : fallback;
|
|
13
|
+
};
|
|
14
|
+
try {
|
|
15
|
+
const cfg = embedConfig();
|
|
16
|
+
if (cfg) console.log(`[ukit:index:embed] sending file descriptors (path, symbols, leading comment) to ${new URL(cfg.url).host}`);
|
|
17
|
+
const started = Date.now();
|
|
18
|
+
const result = await buildSemanticVectors({
|
|
19
|
+
rootDir: path.resolve(flag('--root', process.cwd())),
|
|
20
|
+
dims: Number.parseInt(flag('--dims', '0'), 10) || 0,
|
|
21
|
+
});
|
|
22
|
+
console.log(`[ukit:index:embed] ${result.embedded} embedded, ${result.reused} reused, ${result.files} files, dims=${result.dims}, ${result.bytes} bytes, ${Date.now() - started} ms`);
|
|
23
|
+
} catch (error) {
|
|
24
|
+
console.error(`[ukit:index:embed] ${error && error.message ? error.message : error}`);
|
|
25
|
+
process.exitCode = 1;
|
|
26
|
+
}
|
|
@@ -4,6 +4,7 @@ import crypto from 'node:crypto';
|
|
|
4
4
|
import { execFile, spawnSync } from 'node:child_process';
|
|
5
5
|
import { withFileLock } from '../../runtime/token-utils.mjs';
|
|
6
6
|
import { withAsyncLock } from '../../runtime/async-lock.mjs';
|
|
7
|
+
import { embedConfig, looksLikeQuestion, semanticCandidates, fuseSemantic, semanticStoreSignature } from './semantic-lane.mjs';
|
|
7
8
|
// edit-graph.mjs is optional: a partial install must not lose the whole index core.
|
|
8
9
|
let editGraphModule;
|
|
9
10
|
function loadEditGraphModule() {
|
|
@@ -1398,7 +1399,7 @@ export async function getFileOutline({ rootDir = process.cwd(), filePath, limit
|
|
|
1398
1399
|
}
|
|
1399
1400
|
|
|
1400
1401
|
// contentLane=false skips the error-string content search (context-resolution paths never pay for it).
|
|
1401
|
-
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true } = {}) {
|
|
1402
|
+
export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5, contentLane = true, semantic = false } = {}) {
|
|
1402
1403
|
const normalizedQuery = String(query ?? '').trim();
|
|
1403
1404
|
if (!normalizedQuery) return [];
|
|
1404
1405
|
if (limit <= 0) return Object.freeze([]);
|
|
@@ -1411,11 +1412,15 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1411
1412
|
query: normalizedQuery,
|
|
1412
1413
|
limit,
|
|
1413
1414
|
contentLane,
|
|
1415
|
+
semantic: semantic && embedConfig() ? `1:${await semanticStoreSignature(absoluteRoot)}` : 0,
|
|
1414
1416
|
});
|
|
1415
1417
|
if (QUERY_RESULT_CACHE.has(queryCacheKey)) {
|
|
1416
1418
|
return QUERY_RESULT_CACHE.get(queryCacheKey);
|
|
1417
1419
|
}
|
|
1418
1420
|
const queryPromise = (async () => {
|
|
1421
|
+
// `laneStatus.fellBack`: the lane was asked for (question-shaped, lexical not decisive) and
|
|
1422
|
+
// produced nothing (timeout / HTTP error / no vectors). Such a result must not be cached.
|
|
1423
|
+
const laneStatus = { fellBack: false };
|
|
1419
1424
|
const { files, searchDescriptorsByFile, tokenWeights } = await loadQuerySearchBundle(absoluteRoot);
|
|
1420
1425
|
const preferredArchetypes = inferPreferredArchetypes(queryDescriptor.tokens);
|
|
1421
1426
|
const includeLikelyTestFiles = shouldIncludeLikelyTestFiles(queryDescriptor);
|
|
@@ -1481,7 +1486,12 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1481
1486
|
});
|
|
1482
1487
|
}
|
|
1483
1488
|
|
|
1484
|
-
if (directScores.size === 0)
|
|
1489
|
+
if (directScores.size === 0) {
|
|
1490
|
+
const onlySemantic = semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked: [], includeLikelyTestFiles, files, laneStatus }) : null;
|
|
1491
|
+
if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1492
|
+
if (onlySemantic) return withContentLaneStatus(freezeQueryResults(onlySemantic.slice(0, limit)), contentLaneRan, contentCutShort);
|
|
1493
|
+
return withContentLaneStatus(Object.freeze([]), contentLaneRan, contentCutShort, laneStatus.fellBack);
|
|
1494
|
+
}
|
|
1485
1495
|
|
|
1486
1496
|
const {
|
|
1487
1497
|
hotspotScores,
|
|
@@ -1523,7 +1533,9 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1523
1533
|
const fused = contentLane && ranked.length > 0 && ranked[0].score < FUSION_WEAK_SCORE && queryDescriptor.tokens.size >= FUSION_MIN_TOKENS
|
|
1524
1534
|
? await fuseWeakQuery({ absoluteRoot, query: normalizedQuery, ranked, searchDescriptorsByFile, includeLikelyTestFiles })
|
|
1525
1535
|
: null;
|
|
1526
|
-
|
|
1536
|
+
const semanticMerged = !fused && semantic ? await applySemanticLane({ absoluteRoot, query: normalizedQuery, ranked, includeLikelyTestFiles, files, laneStatus }) : null;
|
|
1537
|
+
if (laneStatus.fellBack) QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1538
|
+
return withContentLaneStatus(freezeQueryResults((fused ?? semanticMerged ?? ranked).slice(0, limit)), contentLaneRan, contentCutShort, laneStatus.fellBack);
|
|
1527
1539
|
})().catch((error) => {
|
|
1528
1540
|
QUERY_RESULT_CACHE.delete(queryCacheKey);
|
|
1529
1541
|
throw error;
|
|
@@ -1533,6 +1545,31 @@ export async function queryCodeIndex({ rootDir = process.cwd(), query, limit = 5
|
|
|
1533
1545
|
return queryPromise;
|
|
1534
1546
|
}
|
|
1535
1547
|
|
|
1548
|
+
// Real-embedding lane: only when the CLI passes semantic:true AND UKIT_INDEX_EMBED_URL is set.
|
|
1549
|
+
// Applies to question-shaped queries only; any problem returns null (lexical result unchanged).
|
|
1550
|
+
const SEMANTIC_CONFIDENT_MARGIN = 2;
|
|
1551
|
+
async function applySemanticLane({ absoluteRoot, query, ranked, includeLikelyTestFiles, files, laneStatus = {} }) {
|
|
1552
|
+
try {
|
|
1553
|
+
if (!embedConfig() || !looksLikeQuestion(query)) return null;
|
|
1554
|
+
// Lexical is already decisive (top-1 clear of top-2): keep it exactly, no network call.
|
|
1555
|
+
if (ranked.length > 1 && ranked[0].score - ranked[1].score >= SEMANTIC_CONFIDENT_MARGIN) return null;
|
|
1556
|
+
const known = new Set(safeItems(files).map((f) => f?.filePath));
|
|
1557
|
+
const candidates = await semanticCandidates({
|
|
1558
|
+
rootDir: absoluteRoot,
|
|
1559
|
+
query,
|
|
1560
|
+
// Vectors can outlive a deleted/renamed file: only paths in the current index qualify.
|
|
1561
|
+
accept: (filePath) => known.has(filePath) && (includeLikelyTestFiles || !isLikelyTestFilePath(filePath)),
|
|
1562
|
+
weight: mirrorPrior,
|
|
1563
|
+
});
|
|
1564
|
+
const fused = fuseSemantic({ ranked, candidates });
|
|
1565
|
+
if (!fused) laneStatus.fellBack = true;
|
|
1566
|
+
return fused;
|
|
1567
|
+
} catch {
|
|
1568
|
+
laneStatus.fellBack = true;
|
|
1569
|
+
return null;
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1536
1573
|
// Opt-in lane fusion (default OFF): a weak natural-language query (best score below
|
|
1537
1574
|
// FUSION_WEAK_SCORE, >= FUSION_MIN_TOKENS tokens) is re-ranked by RRF over the scored list
|
|
1538
1575
|
// and a hashed-embedding lane (the retriever.js vector lane). Enabled by
|
|
@@ -1741,9 +1778,11 @@ export function queryMatchedSymbolNames(result) {
|
|
|
1741
1778
|
// Non-enumerable (array equality and JSON stay unchanged), like boundedSourceSearch rows.truncated:
|
|
1742
1779
|
// `contentLane` is true when the content search ran; `contentLaneTruncated` lists why it was cut
|
|
1743
1780
|
// short ('verify-cap', 'time-budget', 'max-files', ...) so a partial scan never reads as "no match".
|
|
1744
|
-
function withContentLaneStatus(results, ran, truncated) {
|
|
1781
|
+
function withContentLaneStatus(results, ran, truncated, semanticFellBack = false) {
|
|
1745
1782
|
// results arrive frozen, so copy before attaching the status and freeze again.
|
|
1746
1783
|
const out = [...results];
|
|
1784
|
+
// true when the embedding lane was asked for and returned nothing: callers must not persist this result.
|
|
1785
|
+
Object.defineProperty(out, 'semanticFellBack', { value: Boolean(semanticFellBack), enumerable: false });
|
|
1747
1786
|
Object.defineProperty(out, 'contentLane', { value: ran, enumerable: false });
|
|
1748
1787
|
Object.defineProperty(out, 'contentLaneTruncated', { value: Object.freeze([...truncated]), enumerable: false });
|
|
1749
1788
|
return Object.freeze(out);
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// Real-embedding lane for `ukit index query` (opt-in; no dependency; global fetch only).
|
|
2
|
+
//
|
|
3
|
+
// PRIVACY: building vectors SENDS FILE TEXT (path, symbol names, leading comment and,
|
|
4
|
+
// at most ~240 characters per file) to the endpoint in
|
|
5
|
+
// UKIT_INDEX_EMBED_URL, and queries send the question text. With no URL configured nothing
|
|
6
|
+
// here runs and nothing leaves the machine. Sensitive paths (.env, keys, credentials ...)
|
|
7
|
+
// are never embedded; only files already present in the index are considered.
|
|
8
|
+
//
|
|
9
|
+
// Byte-identical mirror: template_project/.claude/ukit/index/lib/semantic-lane.mjs
|
|
10
|
+
// (tests/index/semanticLane.test.js enforces it). Keep this file dependency-free.
|
|
11
|
+
//
|
|
12
|
+
// Env: UKIT_INDEX_EMBED_URL (base URL ending in /v1), UKIT_INDEX_EMBED_MODEL,
|
|
13
|
+
// UKIT_INDEX_EMBED_KEY (optional Bearer).
|
|
14
|
+
// Build is an explicit command (embed-index.mjs), never part of index build/refresh, never in
|
|
15
|
+
// a hook. Query use is reachable only through queryCodeIndex({ semantic: true }), which only
|
|
16
|
+
// the CLI query path passes.
|
|
17
|
+
import fs from 'node:fs/promises';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import crypto from 'node:crypto';
|
|
20
|
+
|
|
21
|
+
export const SEMANTIC_MANIFEST = 'semantic.json';
|
|
22
|
+
export const SEMANTIC_BIN = 'semantic.bin';
|
|
23
|
+
// A CLI query waits at most this long for the embedding of the question. Measured on the
|
|
24
|
+
// reference endpoint: warm question embedding ~0.45 s, 64 descriptors ~24 s (serial service).
|
|
25
|
+
export const QUERY_TIMEOUT_MS = 1500;
|
|
26
|
+
// One descriptor per file, ~60 tokens. Endpoint throughput is ~170-200 tokens/s and serial, so
|
|
27
|
+
// file bodies (>1 h for 2.4k files) are not embedded; descriptors build in minutes.
|
|
28
|
+
export const DOC_CHARS = 240;
|
|
29
|
+
const NAMES_CHARS = 90;
|
|
30
|
+
const LEAD_CHARS = 100;
|
|
31
|
+
const QUERY_INSTRUCTION = 'Instruct: Given a question about a code repository, retrieve the source file that answers it\nQuery: ';
|
|
32
|
+
|
|
33
|
+
export function embedConfig(env = process.env) {
|
|
34
|
+
// Accepts the base (.../v1) or the full endpoint (.../v1/embeddings).
|
|
35
|
+
const url = String(env.UKIT_INDEX_EMBED_URL ?? '').trim().replace(/\/+$/, '').replace(/\/embeddings$/i, '');
|
|
36
|
+
if (!url) return null;
|
|
37
|
+
return { url, model: String(env.UKIT_INDEX_EMBED_MODEL ?? '').trim() || 'unic-embedding', key: String(env.UKIT_INDEX_EMBED_KEY ?? '').trim() };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function isSensitive(value) {
|
|
41
|
+
const segments = String(value ?? '').replace(/\\/g, '/').split('/');
|
|
42
|
+
const base = segments[segments.length - 1] ?? '';
|
|
43
|
+
return base.toLowerCase().startsWith('.env')
|
|
44
|
+
|| /\.(?:pem|key|p12|pfx|jks|keystore)$/i.test(base)
|
|
45
|
+
|| /^(?:credentials?|\.npmrc|\.netrc|\.pypirc)(?:\.[^/]*)?$/i.test(base)
|
|
46
|
+
|| /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.[^/]*)?$/i.test(base)
|
|
47
|
+
|| segments.includes('.ssh') || segments.includes('.aws') || segments.includes('.gnupg');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const sha = (text) => crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
|
|
51
|
+
|
|
52
|
+
// POST /embeddings; returns arrays in input order or throws. Validates shape.
|
|
53
|
+
export async function embedTexts(texts, cfg, { timeoutMs = 60000 } = {}) {
|
|
54
|
+
const headers = { 'content-type': 'application/json' };
|
|
55
|
+
if (cfg.key) headers.authorization = `Bearer ${cfg.key}`;
|
|
56
|
+
const response = await fetch(`${cfg.url}/embeddings`, {
|
|
57
|
+
method: 'POST',
|
|
58
|
+
headers,
|
|
59
|
+
body: JSON.stringify({ model: cfg.model, input: texts }),
|
|
60
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
61
|
+
});
|
|
62
|
+
if (!response.ok) throw new Error(`embedding endpoint HTTP ${response.status}`);
|
|
63
|
+
const data = (await response.json())?.data;
|
|
64
|
+
if (!Array.isArray(data) || data.length !== texts.length) throw new Error('embedding response shape mismatch');
|
|
65
|
+
const rows = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding);
|
|
66
|
+
const dims = rows[0]?.length;
|
|
67
|
+
if (!dims || rows.some((r) => !Array.isArray(r) || r.length !== dims || r.some((x) => !Number.isFinite(x)))) {
|
|
68
|
+
throw new Error('embedding response vectors invalid');
|
|
69
|
+
}
|
|
70
|
+
return rows;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function normalizeTo(vec, dims) {
|
|
74
|
+
const out = vec.slice(0, dims);
|
|
75
|
+
let n = 0;
|
|
76
|
+
for (const x of out) n += x * x;
|
|
77
|
+
n = Math.sqrt(n) || 1;
|
|
78
|
+
return out.map((x) => x / n);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Descriptor for one file: path + exported/declared symbol names + leading doc comment,
|
|
82
|
+
// capped at DOC_CHARS. The path is never truncated; names and comment share the remainder.
|
|
83
|
+
export function buildDocumentText({ filePath, symbols = [], content = '' }) {
|
|
84
|
+
const names = [...new Set(symbols)].join(' ').slice(0, NAMES_CHARS);
|
|
85
|
+
const lines = String(content).split('\n');
|
|
86
|
+
const lead = [];
|
|
87
|
+
for (const line of lines.slice(0, 80)) {
|
|
88
|
+
const t = line.trim();
|
|
89
|
+
if (!t || t.startsWith('#!') || t === '---' || t.startsWith("'use ")) { if (lead.length) break; continue; }
|
|
90
|
+
if (/^(\/\/|\/\*|\*|#|<!--|"""|--)/.test(t)) lead.push(t.replace(/^(\/\/+|\/\*+|\*+\/?|#+|<!--|-->|"""|--)\s?/, ''));
|
|
91
|
+
else break;
|
|
92
|
+
}
|
|
93
|
+
const leading = (lead.length ? lead.join(' ') : '').slice(0, LEAD_CHARS);
|
|
94
|
+
const text = `${filePath}\n${names}\n${leading}`;
|
|
95
|
+
return text.length > DOC_CHARS ? text.slice(0, DOC_CHARS) : text;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function readJson(file) {
|
|
99
|
+
try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function loadStore(indexDir) {
|
|
103
|
+
const manifest = await readJson(path.join(indexDir, SEMANTIC_MANIFEST));
|
|
104
|
+
if (!manifest || manifest.v !== 1 || !Array.isArray(manifest.order)) return null;
|
|
105
|
+
let bin;
|
|
106
|
+
try { bin = await fs.readFile(path.join(indexDir, SEMANTIC_BIN)); } catch { return null; }
|
|
107
|
+
const rec = 4 + manifest.dims;
|
|
108
|
+
if (bin.length !== rec * manifest.order.length) return null;
|
|
109
|
+
return { manifest, bin, rec };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Explicit build step. Embeds only documents whose hash (document text + model) is not in the
|
|
113
|
+
// existing store. Returns stats. Never called from index build/refresh or hooks.
|
|
114
|
+
export async function buildSemanticVectors({
|
|
115
|
+
rootDir = process.cwd(), env = process.env, dims = 0,
|
|
116
|
+
batchSize = 64, onProgress = null,
|
|
117
|
+
} = {}) {
|
|
118
|
+
const cfg = embedConfig(env);
|
|
119
|
+
if (!cfg) throw new Error('UKIT_INDEX_EMBED_URL is not set; semantic vectors are opt-in');
|
|
120
|
+
const root = path.resolve(rootDir);
|
|
121
|
+
const indexDir = path.join(root, '.cache', 'index');
|
|
122
|
+
const filesArtifact = await readJson(path.join(indexDir, 'files.json'));
|
|
123
|
+
const symbolsArtifact = await readJson(path.join(indexDir, 'symbols.json'));
|
|
124
|
+
if (!filesArtifact?.items) throw new Error('index missing; run the index build first');
|
|
125
|
+
const symbolsByFile = new Map();
|
|
126
|
+
for (const s of symbolsArtifact?.items ?? []) {
|
|
127
|
+
if (!symbolsByFile.has(s.filePath)) symbolsByFile.set(s.filePath, []);
|
|
128
|
+
symbolsByFile.get(s.filePath).push(s.name);
|
|
129
|
+
}
|
|
130
|
+
// Reuse vectors from a store built with the same model and truncation.
|
|
131
|
+
const old = await loadStore(indexDir);
|
|
132
|
+
const reusable = old && old.manifest.model === cfg.model
|
|
133
|
+
&& (!dims || old.manifest.dims === dims) ? old : null;
|
|
134
|
+
const oldByHash = new Map();
|
|
135
|
+
if (reusable) reusable.manifest.order.forEach(([, h], i) => oldByHash.set(h, reusable.bin.subarray(i * reusable.rec, (i + 1) * reusable.rec)));
|
|
136
|
+
|
|
137
|
+
const docs = [];
|
|
138
|
+
for (const item of filesArtifact.items) {
|
|
139
|
+
const filePath = item.filePath;
|
|
140
|
+
if (typeof filePath !== 'string' || isSensitive(filePath)) continue;
|
|
141
|
+
let content = '';
|
|
142
|
+
try { content = await fs.readFile(path.join(root, filePath), 'utf8'); } catch { continue; }
|
|
143
|
+
const text = buildDocumentText({ filePath, symbols: symbolsByFile.get(filePath) ?? [], content });
|
|
144
|
+
docs.push({ filePath, text, hash: sha(`${cfg.model}\0${text}`) });
|
|
145
|
+
}
|
|
146
|
+
const missing = [...new Map(docs.filter((d) => !oldByHash.has(d.hash)).map((d) => [d.hash, d])).values()];
|
|
147
|
+
const fresh = new Map();
|
|
148
|
+
let outDims = dims || reusable?.manifest.dims || 0;
|
|
149
|
+
// Requests are strictly sequential: the embedding service is serial, so parallel requests
|
|
150
|
+
// only queue. Partial progress is saved: a failed batch leaves its files without vectors and
|
|
151
|
+
// the error is rethrown after the store is written, so a rerun only embeds what is missing.
|
|
152
|
+
const failures = [];
|
|
153
|
+
try {
|
|
154
|
+
for (let i = 0; i < missing.length; i += batchSize) {
|
|
155
|
+
const batch = missing.slice(i, i + batchSize);
|
|
156
|
+
const rows = await embedTexts(batch.map((d) => d.text), cfg, { timeoutMs: 180000 });
|
|
157
|
+
if (!outDims) outDims = rows[0].length;
|
|
158
|
+
if (rows[0].length < outDims) throw new Error('endpoint returned fewer dimensions than requested');
|
|
159
|
+
batch.forEach((d, k) => fresh.set(d.hash, rows[k]));
|
|
160
|
+
onProgress?.({ done: fresh.size, total: missing.length });
|
|
161
|
+
}
|
|
162
|
+
} catch (e) { failures.push(e); }
|
|
163
|
+
if (!outDims) outDims = old?.manifest.dims ?? 0;
|
|
164
|
+
if (!outDims && docs.length) throw new Error('no vectors produced');
|
|
165
|
+
|
|
166
|
+
const rec = 4 + outDims;
|
|
167
|
+
const order = docs.filter((d) => fresh.has(d.hash) || oldByHash.has(d.hash)).map((d) => [d.filePath, d.hash]);
|
|
168
|
+
const bin = Buffer.alloc(rec * order.length);
|
|
169
|
+
order.forEach(([, hash], i) => {
|
|
170
|
+
const off = i * rec;
|
|
171
|
+
const prior = oldByHash.get(hash);
|
|
172
|
+
if (prior) { prior.copy(bin, off); return; }
|
|
173
|
+
const v = normalizeTo(fresh.get(hash), outDims);
|
|
174
|
+
const maxAbs = Math.max(...v.map(Math.abs)) || 1;
|
|
175
|
+
bin.writeFloatLE(maxAbs / 127, off);
|
|
176
|
+
for (let k = 0; k < outDims; k += 1) bin.writeInt8(Math.max(-127, Math.min(127, Math.round(v[k] / maxAbs * 127))), off + 4 + k);
|
|
177
|
+
});
|
|
178
|
+
const manifest = { v: 1, model: cfg.model, dims: outDims, createdAt: new Date().toISOString(), order };
|
|
179
|
+
const tmp = `.tmp-${process.pid}`;
|
|
180
|
+
await fs.writeFile(path.join(indexDir, SEMANTIC_BIN + tmp), bin);
|
|
181
|
+
await fs.writeFile(path.join(indexDir, SEMANTIC_MANIFEST + tmp), JSON.stringify(manifest));
|
|
182
|
+
await fs.rename(path.join(indexDir, SEMANTIC_BIN + tmp), path.join(indexDir, SEMANTIC_BIN));
|
|
183
|
+
await fs.rename(path.join(indexDir, SEMANTIC_MANIFEST + tmp), path.join(indexDir, SEMANTIC_MANIFEST));
|
|
184
|
+
if (failures.length) throw new Error(`partial store saved (${order.length}/${docs.length} files); rerun to resume: ${failures[0]?.message ?? failures[0]}`);
|
|
185
|
+
return { files: docs.length, embedded: fresh.size, reused: docs.length - missing.length, dims: outDims, bytes: bin.length, model: cfg.model };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Cache-key part: changes when the vector store is built, rebuilt or removed, so a result cached
|
|
189
|
+
// before `embed-index` ran never outlives the vectors. Never throws.
|
|
190
|
+
export async function semanticStoreSignature(rootDir) {
|
|
191
|
+
try {
|
|
192
|
+
const stat = await fs.stat(path.join(path.resolve(rootDir), '.cache', 'index', SEMANTIC_MANIFEST));
|
|
193
|
+
return `${stat.mtimeMs}:${stat.size}`;
|
|
194
|
+
} catch {
|
|
195
|
+
return 'none';
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const STORE_CACHE = new Map();
|
|
200
|
+
|
|
201
|
+
// Query-side ranking. Returns [{ filePath, similarity }] (best first) or null on ANY problem
|
|
202
|
+
// (lane off, no vectors, model/dimension mismatch, timeout, HTTP error). Never throws.
|
|
203
|
+
export async function semanticCandidates({
|
|
204
|
+
rootDir, query, env = process.env, timeoutMs = QUERY_TIMEOUT_MS, limit = 100,
|
|
205
|
+
queryInstruction = QUERY_INSTRUCTION, accept = () => true, weight = () => 1,
|
|
206
|
+
} = {}) {
|
|
207
|
+
try {
|
|
208
|
+
const cfg = embedConfig(env);
|
|
209
|
+
if (!cfg) return null;
|
|
210
|
+
const indexDir = path.join(path.resolve(rootDir), '.cache', 'index');
|
|
211
|
+
const stat = await fs.stat(path.join(indexDir, SEMANTIC_MANIFEST));
|
|
212
|
+
const key = `${indexDir}:${stat.mtimeMs}:${stat.size}`;
|
|
213
|
+
let store = STORE_CACHE.get(key);
|
|
214
|
+
if (!store) { store = await loadStore(indexDir); if (!store) return null; STORE_CACHE.clear(); STORE_CACHE.set(key, store); }
|
|
215
|
+
const { manifest, bin, rec } = store;
|
|
216
|
+
if (manifest.model !== cfg.model) return null;
|
|
217
|
+
const [raw] = await embedTexts([`${queryInstruction}${query}`], cfg, { timeoutMs });
|
|
218
|
+
if (raw.length < manifest.dims) return null;
|
|
219
|
+
const q = normalizeTo(raw, manifest.dims);
|
|
220
|
+
const scored = [];
|
|
221
|
+
for (let i = 0; i < manifest.order.length; i += 1) {
|
|
222
|
+
const filePath = manifest.order[i][0];
|
|
223
|
+
if (!accept(filePath)) continue;
|
|
224
|
+
const off = i * rec;
|
|
225
|
+
let dot = 0;
|
|
226
|
+
for (let k = 0; k < manifest.dims; k += 1) dot += q[k] * bin.readInt8(off + 4 + k);
|
|
227
|
+
scored.push({ filePath, similarity: dot * bin.readFloatLE(off) * weight(filePath) });
|
|
228
|
+
}
|
|
229
|
+
scored.sort((a, b) => b.similarity - a.similarity || a.filePath.localeCompare(b.filePath));
|
|
230
|
+
return scored.slice(0, limit);
|
|
231
|
+
} catch {
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// Plain-language gate: >= 4 words, no path separators or code punctuation, no quotes/colons,
|
|
237
|
+
// no trailing period (a finished statement is error-message shaped), no error-leading word, and
|
|
238
|
+
// no identifier-shaped token (camelCase, snake_case, kebab with digits, dotted name, file
|
|
239
|
+
// extension, token containing a digit, ALL-CAPS code such as ENOENT). Non-ASCII text
|
|
240
|
+
// (Vietnamese) and ordinary lowercase sentences pass; everything else keeps the lexical result.
|
|
241
|
+
const ERROR_LEADS = /^(?:error|errors|failed|failure|cannot|can't|unable|exception|invalid|unexpected|uncaught|warning|fatal|missing|undefined|null|typeerror|referenceerror)\b/i;
|
|
242
|
+
function identifierShaped(token) {
|
|
243
|
+
const t = token.replace(/^[^\w]+|[^\w]+$/g, '');
|
|
244
|
+
if (!t) return false;
|
|
245
|
+
return /[a-z][A-Z]/.test(t) || /_/.test(t) || /\d/.test(t) || /\w\.\w/.test(t) || /\w-\w*\d/.test(t)
|
|
246
|
+
|| /^[A-Z]{3,}$/.test(t) || /^[A-Z][a-z]+[A-Z]/.test(t) || /\.(?:js|mjs|cjs|ts|tsx|jsx|json|md|py|go|rs|java|rb|yml|yaml|sh)\b/i.test(token);
|
|
247
|
+
}
|
|
248
|
+
export function looksLikeQuestion(query) {
|
|
249
|
+
const q = String(query ?? '').trim();
|
|
250
|
+
const words = q.split(/\s+/).filter(Boolean);
|
|
251
|
+
if (words.length < 4 || /[\\/]|[(){}\[\]=;<>@#$%^&*|~`'"]|::|=>|:/.test(q.replace(/[^\x00-\x7f]/g, ''))) return false;
|
|
252
|
+
if (/[^\x20-\x7e]/.test(q)) return true;
|
|
253
|
+
if (/\.\s*$/.test(q) || ERROR_LEADS.test(q) || words.some(identifierShaped)) return false;
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Combine: weighted RRF over the lexical ranking and the semantic list. Semantic-only files
|
|
258
|
+
// are injected with a floor score. Returns null (caller keeps lexical) when no candidates.
|
|
259
|
+
export function fuseSemantic({ ranked, candidates, rrfK = 5, weight = 1, depth = 100 }) {
|
|
260
|
+
if (!candidates?.length) return null;
|
|
261
|
+
const rrf = new Map();
|
|
262
|
+
ranked.slice(0, depth).forEach((e, i) => rrf.set(e.filePath, 1 / (rrfK + i + 1)));
|
|
263
|
+
candidates.slice(0, depth).forEach((c, i) => rrf.set(c.filePath, (rrf.get(c.filePath) ?? 0) + weight / (rrfK + i + 1)));
|
|
264
|
+
const byPath = new Map(ranked.map((e) => [e.filePath, e]));
|
|
265
|
+
const floor = ranked.length ? ranked[ranked.length - 1].score : 0.01;
|
|
266
|
+
return [...rrf.entries()]
|
|
267
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
268
|
+
.map(([filePath]) => byPath.get(filePath) ?? { filePath, score: Math.min(0.01, floor), reasons: ['semantic:vector'], tests: [] });
|
|
269
|
+
}
|