@alucard_24/pi-codeindex 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +21 -0
- package/README.md +442 -2
- package/config.schema.json +19 -0
- package/eval/queries.jsonl +8 -0
- package/package.json +33 -3
- package/scripts/setup-gpu.sh +65 -0
- package/scripts/setup-vulkan.mjs +140 -0
- package/scripts/vulkan-sources.json +29 -0
- package/src/backend.js +120 -0
- package/src/chunker.js +276 -0
- package/src/cli.js +204 -0
- package/src/config-ui.js +37 -0
- package/src/config.js +131 -0
- package/src/doctor.js +101 -0
- package/src/embed.js +57 -0
- package/src/eval.js +72 -0
- package/src/extension.js +522 -0
- package/src/graph.js +306 -0
- package/src/indexer.js +302 -0
- package/src/local-embed.js +51 -0
- package/src/model.js +71 -0
- package/src/present.js +177 -0
- package/src/probe.js +23 -0
- package/src/search.js +189 -0
- package/src/service-client.js +126 -0
- package/src/service-policy.js +33 -0
- package/src/service-protocol.js +122 -0
- package/src/service-worker.js +3 -0
- package/src/service.js +172 -0
- package/src/store.js +142 -0
- package/src/sync.js +110 -0
- package/src/util.js +108 -0
- package/src/vulkan.js +151 -0
- package/src/workspace.js +18 -0
package/src/model.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// Download selettivo del solo encoder testo. Nessuna modifica globale a env
|
|
2
|
+
// di Transformers.js; le pipeline caricano un percorso locale assoluto.
|
|
3
|
+
import fs from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { createWriteStream } from 'node:fs';
|
|
6
|
+
import { randomUUID } from 'node:crypto';
|
|
7
|
+
import { Readable } from 'node:stream';
|
|
8
|
+
import { pipeline } from 'node:stream/promises';
|
|
9
|
+
import { MODEL_ID } from './backend.js';
|
|
10
|
+
|
|
11
|
+
export const WEIGHT_FILES = {
|
|
12
|
+
q8: ['onnx/model_quantized.onnx', 'onnx/model_quantized.onnx_data'],
|
|
13
|
+
q4: ['onnx/model_q4.onnx', 'onnx/model_q4.onnx_data'],
|
|
14
|
+
fp16: ['onnx/model_fp16.onnx', 'onnx/model_fp16.onnx_data'],
|
|
15
|
+
};
|
|
16
|
+
const MULTIMODAL_KEYS = ['audio_config', 'audio_token_id', 'boa_token_id', 'boi_token_id',
|
|
17
|
+
'eoa_token_index', 'eoi_token_id', 'image_token_id', 'video_token_id', 'vision_config', 'vision_soft_tokens_per_image'];
|
|
18
|
+
|
|
19
|
+
export function modelPath(config, dtype) {
|
|
20
|
+
return path.join(config.cacheDir, dtype, MODEL_ID);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function exists(file) {
|
|
24
|
+
try { const st = await fs.stat(file); return st.isFile() && st.size > 0; }
|
|
25
|
+
catch (e) { if (e.code === 'ENOENT') return false; throw e; }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function atomicWrite(dest, write) {
|
|
29
|
+
await fs.mkdir(path.dirname(dest), { recursive: true });
|
|
30
|
+
const tmp = `${dest}.${randomUUID()}.part`;
|
|
31
|
+
try {
|
|
32
|
+
await write(tmp);
|
|
33
|
+
await fs.rename(tmp, dest);
|
|
34
|
+
} finally { await fs.rm(tmp, { force: true }); }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function ensureTextModel(profile, config, { fetchImpl = fetch, log = (s) => process.stderr.write(`${s}\n`) } = {}) {
|
|
38
|
+
if (!Object.hasOwn(WEIGHT_FILES, profile.dtype)) throw new Error(`dtype non supportato: ${profile.dtype}`);
|
|
39
|
+
const dir = modelPath(config, profile.dtype);
|
|
40
|
+
const base = `https://huggingface.co/${MODEL_ID}/resolve/main`;
|
|
41
|
+
const files = ['tokenizer.json', 'tokenizer_config.json', ...WEIGHT_FILES[profile.dtype]];
|
|
42
|
+
for (const rel of files) {
|
|
43
|
+
const dest = path.join(dir, rel);
|
|
44
|
+
if (await exists(dest)) continue;
|
|
45
|
+
if (config.offline) throw new Error(`modalità offline: manca ${dest}; disattiva offline per scaricare i pesi`);
|
|
46
|
+
log(`[codeindex] scarico ${profile.dtype}/${rel}…`);
|
|
47
|
+
await atomicWrite(dest, async (tmp) => {
|
|
48
|
+
const response = await fetchImpl(`${base}/${rel}`, { redirect: 'follow', signal: AbortSignal.timeout(300_000) });
|
|
49
|
+
if (!response.ok || !response.body) throw new Error(`download ${rel}: HTTP ${response.status}`);
|
|
50
|
+
await pipeline(Readable.fromWeb(response.body), createWriteStream(tmp, { flags: 'wx' }));
|
|
51
|
+
const size = (await fs.stat(tmp)).size;
|
|
52
|
+
const expected = Number(response.headers.get('content-length') ?? 0);
|
|
53
|
+
if (!size || (expected && expected !== size)) throw new Error(`download incompleto ${rel}: ${size}/${expected} byte`);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
const dest = path.join(dir, 'config.json');
|
|
57
|
+
let current = null;
|
|
58
|
+
try { current = JSON.parse(await fs.readFile(dest, 'utf8')); }
|
|
59
|
+
catch (e) { if (e.code !== 'ENOENT' && !(e instanceof SyntaxError)) throw e; }
|
|
60
|
+
if (!current) {
|
|
61
|
+
if (config.offline) throw new Error(`modalità offline: manca una config valida in ${dest}`);
|
|
62
|
+
const response = await fetchImpl(`${base}/config.json`, { signal: AbortSignal.timeout(30_000) });
|
|
63
|
+
if (!response.ok) throw new Error(`config.json: HTTP ${response.status}`);
|
|
64
|
+
current = await response.json();
|
|
65
|
+
} else if (!MULTIMODAL_KEYS.some((key) => key in current)) {
|
|
66
|
+
return dir;
|
|
67
|
+
}
|
|
68
|
+
for (const key of MULTIMODAL_KEYS) delete current[key];
|
|
69
|
+
await atomicWrite(dest, (tmp) => fs.writeFile(tmp, JSON.stringify(current, null, 2), { flag: 'wx' }));
|
|
70
|
+
return dir;
|
|
71
|
+
}
|
package/src/present.js
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// Formattazione per l'agente: funzioni pure, nessun accesso a pi o al disco.
|
|
2
|
+
// Stanno qui, e non in extension.js, perché sono l'unica parte dell'integrazione
|
|
3
|
+
// che vale la pena testare con `node --test` (l'adapter resta sottile).
|
|
4
|
+
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
|
|
7
|
+
export const OUTLINE_MIN_LINES = 300;
|
|
8
|
+
export const OUTLINE_MIN_SYMBOLS = 5;
|
|
9
|
+
export const OUTLINE_MAX_SYMBOLS = 40;
|
|
10
|
+
// Limite di default di `read` (DEFAULT_MAX_LINES): un `limit` a questo valore è
|
|
11
|
+
// una read completa esplicita, non una lettura mirata.
|
|
12
|
+
const FULL_READ_LIMIT = 2000;
|
|
13
|
+
const MAX_PREVIEW_LINES = 2;
|
|
14
|
+
const MAX_PREVIEW_CHARS = 160;
|
|
15
|
+
|
|
16
|
+
/** 1234 -> "1.2k"; 5230 -> "5.2k". Sotto il migliaio resta il numero esatto. */
|
|
17
|
+
export function fmtK(n) {
|
|
18
|
+
if (!Number.isFinite(n) || n < 0) return '?';
|
|
19
|
+
if (n < 1000) return String(Math.round(n));
|
|
20
|
+
const k = n / 1000;
|
|
21
|
+
return `${k < 10 ? k.toFixed(1) : Math.round(k)}k`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Riga di stato per il footer. `undefined` = nessuna riga (nessun repo git).
|
|
26
|
+
* @param {{repo?: string|null, enabled?: boolean, indexing?: boolean,
|
|
27
|
+
* progress?: {done: number, total: number}|null, chunks?: number}} s
|
|
28
|
+
*/
|
|
29
|
+
export function statusLine(s) {
|
|
30
|
+
if (!s?.repo) return undefined;
|
|
31
|
+
if (s.enabled === false) return 'codeindex: off';
|
|
32
|
+
if (s.indexing) {
|
|
33
|
+
const p = s.progress;
|
|
34
|
+
return p && p.total > 0
|
|
35
|
+
? `codeindex: ${fmtK(p.done)}/${fmtK(p.total)}`
|
|
36
|
+
: 'codeindex: indicizzazione…';
|
|
37
|
+
}
|
|
38
|
+
if (s.chunks > 0) return `codeindex: ${fmtK(s.chunks)} chunk`;
|
|
39
|
+
return 'codeindex: nessun indice';
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Bundle (Fase 4.5): hit + import + importatori + chiamanti euristici dei primi
|
|
44
|
+
* risultati. Formato compatto: ogni fatto è una riga, il resto è contesto.
|
|
45
|
+
*/
|
|
46
|
+
export function formatBundle(query, res, { contexts = 3 } = {}) {
|
|
47
|
+
const lines = [formatHitsHead(res), ''];
|
|
48
|
+
const results = res?.results ?? [];
|
|
49
|
+
results.forEach((r, i) => {
|
|
50
|
+
lines.push(`${(r.score ?? 0).toFixed(3)} ${r.path}:${r.start}-${r.end} [${r.kind}${r.symbols?.length ? ` · ${r.symbols[0]}` : ''}]${r.stale ? ' STALE' : ''}`);
|
|
51
|
+
const p = String(r.preview ?? '').split('\n').slice(0, MAX_PREVIEW_LINES);
|
|
52
|
+
for (const l of p) if (l.trim()) lines.push(` ${oneLine(l)}`);
|
|
53
|
+
if (i >= contexts) return;
|
|
54
|
+
const c = r.context;
|
|
55
|
+
if (c?.imports?.length) lines.push(` imports: ${c.imports.join(', ')}`);
|
|
56
|
+
if (c?.importers?.length) lines.push(` imported by: ${c.importers.join(', ')}`);
|
|
57
|
+
for (const caller of c?.callers ?? []) {
|
|
58
|
+
lines.push(` caller (heuristic): ${caller.path}:${caller.line} in ${caller.container ?? 'top level'}`);
|
|
59
|
+
}
|
|
60
|
+
if (c?.callersTruncated) lines.push(' … more callers not shown');
|
|
61
|
+
});
|
|
62
|
+
if (!results.length) {
|
|
63
|
+
lines.push('No semantic match. Try different wording, or grep for an exact identifier.');
|
|
64
|
+
} else {
|
|
65
|
+
if (results.some((r) => r.stale)) {
|
|
66
|
+
lines.push('STALE = file changed after indexing: re-read it, line numbers may have moved.');
|
|
67
|
+
}
|
|
68
|
+
lines.push('Callers are heuristic (identifier scan), not a call graph. Verify before editing.');
|
|
69
|
+
}
|
|
70
|
+
return lines.join('\n');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function formatHitsHead(res) {
|
|
74
|
+
const n = res?.results?.length ?? 0;
|
|
75
|
+
return `${n} result${n === 1 ? '' : 's'} in ${Math.round(res?.tookMs ?? 0)} ms` +
|
|
76
|
+
`${res?.chunks ? ` · ${fmtK(res.chunks)} chunks indexed` : ''}`;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Riga iniettata nel system prompt: rende `search_code` scopribile senza
|
|
81
|
+
* contare le query già presenti nelle dichiarazioni dei tool.
|
|
82
|
+
* Il conteggio è arrotondato perché il testo resti stabile (prompt cache).
|
|
83
|
+
*/
|
|
84
|
+
export function promptLine(chunks) {
|
|
85
|
+
if (!chunks || chunks < 100) return null;
|
|
86
|
+
return (
|
|
87
|
+
`Code index: ${fmtK(chunks)} chunks of this repository are embedded locally ` +
|
|
88
|
+
'(search_code). Prefer it to localize code before grep/read; grep stays best for exact strings.'
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const oneLine = (s, n = MAX_PREVIEW_CHARS) => String(s ?? '').replace(/\s+/g, ' ').trim().slice(0, n);
|
|
93
|
+
|
|
94
|
+
/** Risultati di ricerca in forma compatta: una riga per hit + anteprima breve. */
|
|
95
|
+
export function formatHits(query, res) {
|
|
96
|
+
const n = res?.results?.length ?? 0;
|
|
97
|
+
const lines = [formatHitsHead(res)];
|
|
98
|
+
for (const r of res?.results ?? []) {
|
|
99
|
+
const sym = r.symbols?.length ? ` · ${r.symbols.slice(0, 3).join(', ')}` : '';
|
|
100
|
+
const stale = r.stale ? ' STALE' : '';
|
|
101
|
+
lines.push(
|
|
102
|
+
`${(r.score ?? 0).toFixed(3)} ${r.path}:${r.start}-${r.end} [${r.kind}${sym}]` +
|
|
103
|
+
` cos ${(r.cos ?? 0).toFixed(3)} lex ${(r.lex ?? 0).toFixed(2)}${stale}`,
|
|
104
|
+
);
|
|
105
|
+
const preview = String(r.preview ?? '').split('\n').slice(0, MAX_PREVIEW_LINES);
|
|
106
|
+
for (const l of preview) if (l.trim()) lines.push(` ${oneLine(l)}`);
|
|
107
|
+
}
|
|
108
|
+
if (n === 0) {
|
|
109
|
+
lines.push(
|
|
110
|
+
'No semantic match. Try a different wording, or grep for an exact identifier — ' +
|
|
111
|
+
'the index finds concepts, not literal strings.',
|
|
112
|
+
);
|
|
113
|
+
} else {
|
|
114
|
+
if (res?.results?.some((r) => r.stale)) {
|
|
115
|
+
lines.push('STALE = file changed after indexing: re-read it, line numbers may have moved.');
|
|
116
|
+
}
|
|
117
|
+
lines.push('Read the hits you need (read with offset/limit for large files); grep for exact strings.');
|
|
118
|
+
}
|
|
119
|
+
return lines.join('\n');
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Outline di un file dall'indice: solo simboli, nessun contenuto.
|
|
124
|
+
* @returns {string|null} null se il file non ha abbastanza simboli indicizzati.
|
|
125
|
+
*/
|
|
126
|
+
export function outlineText(relPath, lineCount, chunks, {
|
|
127
|
+
minLines = OUTLINE_MIN_LINES,
|
|
128
|
+
minSymbols = OUTLINE_MIN_SYMBOLS,
|
|
129
|
+
maxSymbols = OUTLINE_MAX_SYMBOLS,
|
|
130
|
+
} = {}) {
|
|
131
|
+
if (!(lineCount > minLines)) return null;
|
|
132
|
+
const mine = (chunks ?? [])
|
|
133
|
+
.filter((c) => c.path === relPath && c.symbol && c.kind !== 'preamble')
|
|
134
|
+
.sort((a, b) => a.start - b.start);
|
|
135
|
+
const seen = new Set();
|
|
136
|
+
const picked = [];
|
|
137
|
+
for (const c of mine) {
|
|
138
|
+
if (seen.has(c.symbol)) continue;
|
|
139
|
+
seen.add(c.symbol);
|
|
140
|
+
picked.push(c);
|
|
141
|
+
}
|
|
142
|
+
if (picked.length < minSymbols) return null;
|
|
143
|
+
const shown = picked.slice(0, maxSymbols);
|
|
144
|
+
const lines = [
|
|
145
|
+
`[outline from code index · ${relPath} · ${lineCount} lines · ${picked.length} symbols]`,
|
|
146
|
+
];
|
|
147
|
+
for (const c of shown) {
|
|
148
|
+
const range = `${c.start}-${c.end}`.padEnd(12);
|
|
149
|
+
lines.push(`${range}${c.kind.padEnd(10)}${c.symbol}`);
|
|
150
|
+
}
|
|
151
|
+
if (picked.length > shown.length) lines.push(`… ${picked.length - shown.length} more (search_code for details)`);
|
|
152
|
+
lines.push('[full file content follows]');
|
|
153
|
+
return lines.join('\n');
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Decide se annotare una `read`. L'outline serve a chi legge un file intero:
|
|
158
|
+
* una read mirata (offset/limit espliciti) non lo riceve.
|
|
159
|
+
* `offset: 1`/`limit: 2000` sono la read completa scritta in modo esplicito.
|
|
160
|
+
*/
|
|
161
|
+
export function shouldAnnotateRead(input, lineCount, { minLines = OUTLINE_MIN_LINES } = {}) {
|
|
162
|
+
if (!(lineCount > minLines)) return false;
|
|
163
|
+
const offset = input?.offset;
|
|
164
|
+
if (offset !== undefined && offset !== null && offset > 1) return false;
|
|
165
|
+
const limit = input?.limit;
|
|
166
|
+
if (limit !== undefined && limit !== null && limit < FULL_READ_LIMIT) return false;
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Path assoluto -> relativo al repo, o null se esce dal repo. */
|
|
171
|
+
export function toRepoRel(repo, absPath) {
|
|
172
|
+
if (!repo || !absPath) return null;
|
|
173
|
+
const abs = path.resolve(absPath);
|
|
174
|
+
const rel = path.relative(repo, abs);
|
|
175
|
+
if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return null;
|
|
176
|
+
return rel.split(path.sep).join('/');
|
|
177
|
+
}
|
package/src/probe.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Entry point interno, eseguito solo da doctor --probe in un processo isolato.
|
|
2
|
+
import { cosine } from './embed.js';
|
|
3
|
+
import { loadConfig } from './config.js';
|
|
4
|
+
import { createLocalEmbedder } from './local-embed.js';
|
|
5
|
+
|
|
6
|
+
let result, runtime;
|
|
7
|
+
try {
|
|
8
|
+
const t0 = performance.now();
|
|
9
|
+
runtime = createLocalEmbedder((await loadConfig()).config);
|
|
10
|
+
const same = 'title: args.ts | text: export function parseArgs(argv) { return argv.slice(2); }';
|
|
11
|
+
const other = 'title: math.py | text: def factorial(n): return 1 if n < 2 else n * factorial(n - 1)';
|
|
12
|
+
const [a, b, c] = await runtime.embed([same, same, other]);
|
|
13
|
+
const cosSame = cosine(a, b);
|
|
14
|
+
const cosOther = cosine(a, c);
|
|
15
|
+
if (cosSame < 0.999 || cosOther > 0.9999) throw new Error(`smoke test discriminazione fallito: stesso=${cosSame}, diverso=${cosOther}`);
|
|
16
|
+
result = { ok: true, profile: await runtime.getProfile(), elapsedMs: Math.round(performance.now() - t0),
|
|
17
|
+
cosSame, cosOther, attempts: runtime.info().attempts };
|
|
18
|
+
await runtime.dispose();
|
|
19
|
+
} catch (e) {
|
|
20
|
+
result = { ok: false, error: String(e.message ?? e), attempts: runtime?.info().attempts ?? [] };
|
|
21
|
+
}
|
|
22
|
+
if (process.send) process.send(result, () => process.exit(result.ok ? 0 : 1));
|
|
23
|
+
else { console.log(JSON.stringify(result)); process.exitCode = result.ok ? 0 : 1; }
|
package/src/search.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// Ricerca ibrida: coseno sui vettori + overlap lessicale su path/simboli/preview.
|
|
2
|
+
// Due stadi: il lessico viene valutato solo sui migliori candidati del coseno
|
|
3
|
+
// (200 su 30k), così una query resta < 300 ms senza un indice invertito.
|
|
4
|
+
|
|
5
|
+
import fs from 'node:fs/promises';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { embedQuery, getEmbeddingProfile } from './embed.js';
|
|
8
|
+
import { assertCompatibleIndex } from './backend.js';
|
|
9
|
+
import { indexDir, indexPaths, loadIndex } from './store.js';
|
|
10
|
+
import { repoRoot, sha1 } from './util.js';
|
|
11
|
+
|
|
12
|
+
export const WEIGHT_COS = 0.75;
|
|
13
|
+
export const WEIGHT_LEX = 0.25;
|
|
14
|
+
const CANDIDATES = 200;
|
|
15
|
+
const PREVIEW_LINES = 3;
|
|
16
|
+
|
|
17
|
+
const STOPWORDS = new Set([
|
|
18
|
+
'the', 'and', 'for', 'with', 'from', 'into', 'this', 'that', 'are', 'was',
|
|
19
|
+
'how', 'what', 'where', 'when', 'which', 'does', 'used', 'use', 'using',
|
|
20
|
+
'di', 'del', 'della', 'dei', 'delle', 'che', 'chi', 'cui', 'per', 'con',
|
|
21
|
+
'una', 'uno', 'gli', 'nel', 'nella', 'nei', 'come', 'dove', 'quando',
|
|
22
|
+
'viene', 'vengono', 'sono', 'essere', 'alla', 'allo', 'agli', 'sul', 'sui',
|
|
23
|
+
'la', 'le', 'lo', 'il', 'un', 'uno', 'si', 'non', 'più', 'dei', 'delle',
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
/** camelCase/snake_case/kebab -> token minuscoli, stopword rimosse. */
|
|
27
|
+
export function tokenize(text) {
|
|
28
|
+
return String(text ?? '')
|
|
29
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
30
|
+
.replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
|
|
31
|
+
.toLowerCase()
|
|
32
|
+
.split(/[^a-z0-9]+/)
|
|
33
|
+
.filter((t) => t.length >= 2 && !STOPWORDS.has(t));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function lexicalOverlap(queryTokens, chunk) {
|
|
37
|
+
if (queryTokens.size === 0) return 0;
|
|
38
|
+
// Path e simboli sono segnali forti; la preview (200 char) è un segnale debole:
|
|
39
|
+
// un documento lungo che parla dell'argomento non deve battere il file che lo implementa.
|
|
40
|
+
const strong = new Set(tokenize([chunk.path, chunk.symbol ?? '', ...(chunk.symbols ?? [])].join(' ')));
|
|
41
|
+
const weak = new Set(tokenize(chunk.preview ?? ''));
|
|
42
|
+
let score = 0;
|
|
43
|
+
for (const t of queryTokens) score += strong.has(t) ? 1 : weak.has(t) ? 0.5 : 0;
|
|
44
|
+
return score / queryTokens.size;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function dot(a, aOff, b) {
|
|
48
|
+
let s = 0;
|
|
49
|
+
for (let i = 0; i < b.length; i++) s += a[aOff + i] * b[i];
|
|
50
|
+
return s;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Cache in-process: un indice da 30k chunk pesa ~100 MB di JSON + 92 MB di vettori.
|
|
54
|
+
// Rileggerlo a ogni query costerebbe centinaia di ms; lo stat di due file costa ~0,1 ms.
|
|
55
|
+
// La chiave d'invalidazione è la coppia (size, mtime) dei due file dell'indice, quindi
|
|
56
|
+
// anche un reindex fatto da un altro processo viene visto.
|
|
57
|
+
const _cache = new Map();
|
|
58
|
+
const MAX_CACHED = 2;
|
|
59
|
+
|
|
60
|
+
async function stampOf(repo) {
|
|
61
|
+
const { json, bin } = indexPaths(indexDir(repo));
|
|
62
|
+
try {
|
|
63
|
+
const [a, b] = await Promise.all([fs.stat(json), fs.stat(bin)]);
|
|
64
|
+
return `${a.size}:${a.mtimeMs}:${b.size}:${b.mtimeMs}`;
|
|
65
|
+
} catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Butta via l'indice in memoria (dopo un reindex, o alla chiusura della sessione). */
|
|
71
|
+
export function invalidateSearcher(root) {
|
|
72
|
+
if (root) _cache.delete(path.resolve(root));
|
|
73
|
+
else _cache.clear();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export async function loadSearcher(root, { refresh = false } = {}) {
|
|
77
|
+
const repo = path.resolve(root ?? repoRoot(process.cwd()) ?? process.cwd());
|
|
78
|
+
const stamp = await stampOf(repo);
|
|
79
|
+
const hit = _cache.get(repo);
|
|
80
|
+
if (!refresh && hit && stamp && hit.stamp === stamp) return hit.value;
|
|
81
|
+
|
|
82
|
+
const { index, vectors } = await loadIndex(repo);
|
|
83
|
+
if (!index || !vectors || vectors.length === 0) {
|
|
84
|
+
throw new Error(`nessun indice per ${repo}: esegui "index" prima`);
|
|
85
|
+
}
|
|
86
|
+
const value = { repo, index, vectors };
|
|
87
|
+
if (stamp) {
|
|
88
|
+
_cache.delete(repo);
|
|
89
|
+
_cache.set(repo, { stamp, value });
|
|
90
|
+
while (_cache.size > MAX_CACHED) _cache.delete(_cache.keys().next().value);
|
|
91
|
+
}
|
|
92
|
+
return value;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* L1 — provenienza: confronta l'hash su disco con quello indicizzato.
|
|
97
|
+
* Costa una lettura per file distinto (≤ k file, cap 1 MB): non dipende dal
|
|
98
|
+
* watcher, quindi è vero anche se la sessione non ha visto la modifica.
|
|
99
|
+
*/
|
|
100
|
+
async function markStale(repo, results) {
|
|
101
|
+
const seen = new Map();
|
|
102
|
+
for (const r of results) {
|
|
103
|
+
if (seen.has(r.path)) {
|
|
104
|
+
r.stale = seen.get(r.path);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
let stale = true;
|
|
108
|
+
try {
|
|
109
|
+
const buf = await fs.readFile(path.resolve(repo, r.path));
|
|
110
|
+
stale = sha1(buf) !== r.fileHash;
|
|
111
|
+
} catch {
|
|
112
|
+
stale = true; // file sparito: non fidarsi del risultato
|
|
113
|
+
}
|
|
114
|
+
seen.set(r.path, stale);
|
|
115
|
+
r.stale = stale;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export async function previewOf(repo, chunk) {
|
|
120
|
+
try {
|
|
121
|
+
const abs = path.resolve(repo, chunk.path);
|
|
122
|
+
if (!abs.startsWith(repo + path.sep)) return null; // niente path fuori dal repo
|
|
123
|
+
const text = await fs.readFile(abs, 'utf8');
|
|
124
|
+
const lines = text.split('\n').slice(chunk.start - 1, Math.min(chunk.end, chunk.start - 1 + PREVIEW_LINES));
|
|
125
|
+
return lines.map((l) => l.trimEnd()).filter((l, i, arr) => l !== '' || i < arr.length - 1).join('\n');
|
|
126
|
+
} catch {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* @returns {Promise<{results:Array, tookMs:number, candidates:number}>}
|
|
133
|
+
*/
|
|
134
|
+
export async function search(query, { root, k = 8, pathPrefix = null, maxPerFile = 2, checkStale = false } = {}) {
|
|
135
|
+
const t0 = performance.now();
|
|
136
|
+
const { repo, index, vectors } = await loadSearcher(root);
|
|
137
|
+
assertCompatibleIndex(index, await getEmbeddingProfile());
|
|
138
|
+
const qVec = await embedQuery(query);
|
|
139
|
+
const qTokens = new Set(tokenize(query));
|
|
140
|
+
const dims = index.dims;
|
|
141
|
+
|
|
142
|
+
const scored = [];
|
|
143
|
+
for (let i = 0; i < index.chunks.length; i++) {
|
|
144
|
+
const c = index.chunks[i];
|
|
145
|
+
if (pathPrefix && !c.path.startsWith(pathPrefix)) continue;
|
|
146
|
+
scored.push({ i, cos: dot(vectors, i * dims, qVec[0]) });
|
|
147
|
+
}
|
|
148
|
+
scored.sort((a, b) => b.cos - a.cos);
|
|
149
|
+
const top = scored.slice(0, Math.min(CANDIDATES, scored.length));
|
|
150
|
+
|
|
151
|
+
const results = top.map(({ i, cos }) => {
|
|
152
|
+
const c = index.chunks[i];
|
|
153
|
+
const lex = lexicalOverlap(qTokens, c);
|
|
154
|
+
return {
|
|
155
|
+
path: c.path,
|
|
156
|
+
start: c.start,
|
|
157
|
+
end: c.end,
|
|
158
|
+
symbol: c.symbol,
|
|
159
|
+
symbols: c.symbols ?? (c.symbol ? [c.symbol] : []),
|
|
160
|
+
kind: c.kind,
|
|
161
|
+
fileHash: c.fileHash,
|
|
162
|
+
score: WEIGHT_COS * cos + WEIGHT_LEX * lex,
|
|
163
|
+
cos,
|
|
164
|
+
lex,
|
|
165
|
+
};
|
|
166
|
+
});
|
|
167
|
+
results.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path) || a.start - b.start);
|
|
168
|
+
// Diversità: senza cap, un piano di 300 KB occupa tutti gli slot e nasconde il codice.
|
|
169
|
+
const perFile = new Map();
|
|
170
|
+
const final = [];
|
|
171
|
+
for (const r of results) {
|
|
172
|
+
const n = perFile.get(r.path) ?? 0;
|
|
173
|
+
if (n >= maxPerFile) continue;
|
|
174
|
+
perFile.set(r.path, n + 1);
|
|
175
|
+
final.push(r);
|
|
176
|
+
if (final.length >= k) break;
|
|
177
|
+
}
|
|
178
|
+
for (const r of final) r.preview = await previewOf(repo, r);
|
|
179
|
+
if (checkStale && final.length) await markStale(repo, final);
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
repo,
|
|
183
|
+
results: final,
|
|
184
|
+
tookMs: Math.round((performance.now() - t0) * 10) / 10,
|
|
185
|
+
candidates: top.length,
|
|
186
|
+
chunks: index.chunks.length,
|
|
187
|
+
stale: final.some((r) => r.stale),
|
|
188
|
+
};
|
|
189
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// Client leggero: solo JSON e socket loopback. Nessuna pipeline locale.
|
|
2
|
+
import { fork } from 'node:child_process';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
5
|
+
import { loadConfig } from './config.js';
|
|
6
|
+
import { assertCompatibleIndex, validateEmbeddings } from './backend.js';
|
|
7
|
+
import { serviceAddress, readServiceRecord, rpc, disconnected, serviceError } from './service-protocol.js';
|
|
8
|
+
import { validateServiceConfig, validateTexts } from './service-policy.js';
|
|
9
|
+
|
|
10
|
+
export function createEmbeddingClient({ env = process.env, settings = () => loadConfig({ env }),
|
|
11
|
+
workerFile = new URL('./service-worker.js', import.meta.url), startupTimeoutMs = 10_000, requestTimeoutMs = 15 * 60_000 } = {}) {
|
|
12
|
+
let config = null, configLoading = null, starting = null, pinned = null;
|
|
13
|
+
let pending = 0, error = null, remote = null;
|
|
14
|
+
async function getConfig() {
|
|
15
|
+
if (config) return config;
|
|
16
|
+
if (!configLoading) configLoading = Promise.resolve().then(settings).then((loaded) => {
|
|
17
|
+
config = validateServiceConfig(loaded.config);
|
|
18
|
+
return config;
|
|
19
|
+
});
|
|
20
|
+
try { return await configLoading; } finally { configLoading = null; }
|
|
21
|
+
}
|
|
22
|
+
async function probe(address) {
|
|
23
|
+
const record = await readServiceRecord(address);
|
|
24
|
+
if (!record) return { record: null };
|
|
25
|
+
try {
|
|
26
|
+
const response = await rpc(address, record, { op: 'status' }, { timeoutMs: 2000 });
|
|
27
|
+
return { record, status: response.result };
|
|
28
|
+
} catch (e) {
|
|
29
|
+
if (disconnected(e)) return { record: null };
|
|
30
|
+
if (e.code === 'SERVICE_STOPPING') return { record, status: { running: false } };
|
|
31
|
+
throw e;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function start(address) {
|
|
35
|
+
if (starting) return starting;
|
|
36
|
+
starting = (async () => {
|
|
37
|
+
let launchError, childRunning = false, launches = 0, lastLaunchAt = 0;
|
|
38
|
+
const launch = () => {
|
|
39
|
+
launches++; childRunning = true; lastLaunchAt = Date.now();
|
|
40
|
+
const child = fork(workerFile instanceof URL ? fileURLToPath(workerFile) : workerFile,
|
|
41
|
+
[address.directory, String(address.port)], { detached: true, execArgv: [], env,
|
|
42
|
+
stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
|
|
43
|
+
child.on('error', (e) => { launchError = e; childRunning = false; });
|
|
44
|
+
child.on('exit', () => { childRunning = false; });
|
|
45
|
+
child.on('message', (message) => {
|
|
46
|
+
if (message?.ready === false && message.code !== 'EADDRINUSE') launchError = serviceError(message.code ?? 'START_FAILED', message.error);
|
|
47
|
+
});
|
|
48
|
+
child.unref(); child.channel?.unref();
|
|
49
|
+
};
|
|
50
|
+
const deadline = Date.now() + startupTimeoutMs;
|
|
51
|
+
while (Date.now() < deadline) {
|
|
52
|
+
if (launchError) throw launchError;
|
|
53
|
+
const { record, status } = await probe(address);
|
|
54
|
+
if (record && status.running) return record;
|
|
55
|
+
// Il vecchio processo tiene la porta durante dispose. Non tentare
|
|
56
|
+
// un secondo modello mentre quel processo sta ancora liberando VRAM.
|
|
57
|
+
if (!record && !childRunning && launches < 3 && Date.now() - lastLaunchAt >= 100) launch();
|
|
58
|
+
await delay(40);
|
|
59
|
+
}
|
|
60
|
+
throw serviceError('START_FAILED', `servizio locale non avviabile su 127.0.0.1:${address.port}: porta occupata o avvio fallito. Controlla service status e servicePort. Nessun fallback in-process.`);
|
|
61
|
+
})();
|
|
62
|
+
try { return await starting; } finally { starting = null; }
|
|
63
|
+
}
|
|
64
|
+
function remember(info) { if (info) remote = { ...info, lastSeenAt: Date.now() }; }
|
|
65
|
+
async function call(op, payload = {}) {
|
|
66
|
+
const active = await getConfig();
|
|
67
|
+
const address = serviceAddress(active);
|
|
68
|
+
pending++;
|
|
69
|
+
error = null;
|
|
70
|
+
try {
|
|
71
|
+
let record = await readServiceRecord(address);
|
|
72
|
+
if (!record) record = await start(address);
|
|
73
|
+
let response;
|
|
74
|
+
const request = { op, config: active, expectedProfile: pinned, ...payload };
|
|
75
|
+
try { response = await rpc(address, record, request, { timeoutMs: requestTimeoutMs }); }
|
|
76
|
+
catch (e) {
|
|
77
|
+
// Questi errori provano che l'inferenza non è iniziata. Un crash
|
|
78
|
+
// dopo l'invio non viene ritentato automaticamente.
|
|
79
|
+
if (!['ECONNREFUSED', 'SERVICE_STOPPING'].includes(e.code)) throw e;
|
|
80
|
+
record = await start(address);
|
|
81
|
+
response = await rpc(address, record, request, { timeoutMs: requestTimeoutMs });
|
|
82
|
+
}
|
|
83
|
+
remember(response.info);
|
|
84
|
+
const profile = response.info?.profile;
|
|
85
|
+
if (!profile) throw serviceError('BAD_FRAME', 'il servizio non ha restituito il profilo del modello');
|
|
86
|
+
if (pinned) {
|
|
87
|
+
try { assertCompatibleIndex(pinned, profile); }
|
|
88
|
+
catch { throw serviceError('PROFILE_CHANGED', 'profilo del servizio cambiato: riavvia pi e verifica/reindicizza; nessun vettore accettato'); }
|
|
89
|
+
}
|
|
90
|
+
pinned ??= Object.freeze({ ...profile });
|
|
91
|
+
return response.result;
|
|
92
|
+
} catch (e) {
|
|
93
|
+
remember(e.info);
|
|
94
|
+
error = e.message;
|
|
95
|
+
throw e;
|
|
96
|
+
} finally { pending--; }
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
getProfile: async () => { await call('profile'); return pinned; },
|
|
100
|
+
embed: async (texts, { normalize = true } = {}) => {
|
|
101
|
+
if (Array.isArray(texts) && !texts.length) return [];
|
|
102
|
+
validateTexts(texts, normalize);
|
|
103
|
+
const rows = await call('embed', { texts, normalize });
|
|
104
|
+
return validateEmbeddings(rows, texts.length).map((row) => Float32Array.from(row));
|
|
105
|
+
},
|
|
106
|
+
info: () => ({ profile: pinned, config, loading: pending > 0, attempts: (remote?.attempts ?? []).map((a) => ({ ...a })),
|
|
107
|
+
error, shared: true,
|
|
108
|
+
warnings: pinned?.device === 'cpu' && config && !['auto', 'cpu'].includes(config.device)
|
|
109
|
+
? [`fallback CPU esplicitamente consentito: richiesto ${config.device}, riutilizzato il servizio ${pinned.device}/${pinned.dtype}`] : [],
|
|
110
|
+
service: remote ? { pid: remote.pid, lastSeenAt: remote.lastSeenAt } : null }),
|
|
111
|
+
status: async () => {
|
|
112
|
+
const address = serviceAddress(await getConfig());
|
|
113
|
+
const record = await readServiceRecord(address);
|
|
114
|
+
if (!record) return { running: false, shared: true, port: address.port };
|
|
115
|
+
try { return (await rpc(address, record, { op: 'status' }, { timeoutMs: 2000 })).result; }
|
|
116
|
+
catch (e) { if (disconnected(e)) return { running: false, shared: true, port: address.port }; throw e; }
|
|
117
|
+
},
|
|
118
|
+
stop: async () => {
|
|
119
|
+
const address = serviceAddress(await getConfig());
|
|
120
|
+
const record = await readServiceRecord(address);
|
|
121
|
+
if (!record) return { stopping: false };
|
|
122
|
+
try { return (await rpc(address, record, { op: 'stop' }, { timeoutMs: 5000 })).result; }
|
|
123
|
+
catch (e) { if (disconnected(e)) return { stopping: false }; throw e; }
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { ENV_KEYS, validateConfig } from './config.js';
|
|
3
|
+
import { assertCompatibleIndex, backendPlan, formatProfile, webgpuBackendOf } from './backend.js';
|
|
4
|
+
import { MAX_TEXTS, MAX_TEXT_BYTES, serviceError } from './service-protocol.js';
|
|
5
|
+
|
|
6
|
+
export function validateServiceConfig(config) {
|
|
7
|
+
validateConfig(config, 'client del servizio');
|
|
8
|
+
if (Object.keys(ENV_KEYS).some((key) => !Object.hasOwn(config, key))) throw serviceError('BAD_CONFIG', 'configurazione client incompleta');
|
|
9
|
+
if (!path.isAbsolute(config.cacheDir) || !path.isAbsolute(config.serviceDir)) throw serviceError('BAD_CONFIG', 'cacheDir e serviceDir devono essere assoluti');
|
|
10
|
+
return Object.freeze({ ...config });
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function assertServiceCompatible(config, activeConfig, profile, expectedProfile) {
|
|
14
|
+
const permitted = backendPlan(config, [{ device: profile.device, available: true }]).some((p) =>
|
|
15
|
+
p.device === profile.device && p.dtype === profile.dtype &&
|
|
16
|
+
(p.device !== 'webgpu' || config.webgpuBackend === webgpuBackendOf(profile)));
|
|
17
|
+
const differs = ['cacheDir', 'cpuThreads', 'serviceIdleMs'].filter((key) => config[key] !== activeConfig[key]);
|
|
18
|
+
if (!permitted || differs.length) {
|
|
19
|
+
throw serviceError('PROFILE_CONFLICT', `servizio condiviso già configurato per ${formatProfile(profile)}${differs.length ? `; impostazioni diverse: ${differs.join(', ')}` : ''}. Nessun secondo modello avviato. Usa /codeindex service status; per cambiare profilo, service stop quando inattivo e riavvia pi.`);
|
|
20
|
+
}
|
|
21
|
+
if (expectedProfile) {
|
|
22
|
+
try { assertCompatibleIndex(expectedProfile, profile); }
|
|
23
|
+
catch { throw serviceError('PROFILE_CHANGED', `profilo del servizio cambiato dopo una riconnessione: ${formatProfile(profile)}. Riavvia pi e verifica/reindicizza; nessun vettore è stato accettato.`); }
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function validateTexts(texts, normalize) {
|
|
28
|
+
if (!Array.isArray(texts) || !texts.length || texts.length > MAX_TEXTS || texts.some((t) => typeof t !== 'string')) {
|
|
29
|
+
throw serviceError('BAD_TEXTS', `embedding: attesi da 1 a ${MAX_TEXTS} testi`);
|
|
30
|
+
}
|
|
31
|
+
if (typeof normalize !== 'boolean') throw serviceError('BAD_TEXTS', 'normalize deve essere booleano');
|
|
32
|
+
if (texts.reduce((n, t) => n + Buffer.byteLength(t), 0) > MAX_TEXT_BYTES) throw serviceError('BAD_TEXTS', `batch di embedding troppo grande (massimo ${MAX_TEXT_BYTES} byte)`);
|
|
33
|
+
}
|