@alucard_24/pi-codeindex 0.0.0-stage → 1.0.0

package/src/config.js

@@ -0,0 +1,126 @@
+// Configurazione della macchina: mai lettura implicita di file nel repository.
+// Precedenza: default < file utente (o CODEINDEX_CONFIG) < ambiente.
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+
+export const DEVICES = ['auto', 'cpu', 'cuda', 'webgpu', 'dml', 'coreml'];
+export const DTYPES = ['auto', 'q8', 'q4', 'fp16'];
+export const WEBGPU_BACKENDS = ['auto', 'vulkan'];
+export const ENV_KEYS = {
+  device: 'CODEINDEX_DEVICE', dtype: 'CODEINDEX_DTYPE', fallback: 'CODEINDEX_FALLBACK',
+  offline: 'CODEINDEX_OFFLINE', cacheDir: 'CODEINDEX_MODEL_CACHE', cpuThreads: 'CODEINDEX_CPU_THREADS',
+  webgpuBackend: 'CODEINDEX_WEBGPU_BACKEND',
+};
+
+export function configPath(env = process.env, home = os.homedir()) {
+  if (env.CODEINDEX_CONFIG) return expandPath(env.CODEINDEX_CONFIG, home);
+  const base = env.XDG_CONFIG_HOME || (process.platform === 'win32' ? env.APPDATA : null) || path.join(home, '.config');
+  return path.resolve(base, 'pi-codeindex', 'config.json');
+}
+
+function expandPath(value, home = os.homedir()) {
+  return path.resolve(value === '~' ? home : value.startsWith('~/') ? path.join(home, value.slice(2)) : value);
+}
+
+export function defaults(home = os.homedir()) {
+  return { device: 'auto', dtype: 'auto', webgpuBackend: 'auto', fallback: 'error', offline: false,
+    cacheDir: path.join(home, '.cache', 'pi-codeindex', 'text-model'), cpuThreads: 0 };
+}
+
+export function validateConfig(value, source = 'config') {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${source}: atteso un oggetto JSON`);
+  for (const [key, v] of Object.entries(value)) {
+    if (key === '$schema') {
+      if (typeof v !== 'string') throw new Error(`${source}: $schema deve essere una stringa`);
+      continue;
+    }
+    if (!Object.hasOwn(ENV_KEYS, key)) throw new Error(`${source}: chiave sconosciuta "${key}"`);
+    if (key === 'device' && v === 'vulkan') {
+      throw new Error(`${source}: Vulkan non è un provider ONNX autonomo. Usa device webgpu e webgpuBackend vulkan per forzarlo (setup-vulkan richiesto).`);
+    }
+    const choices = key === 'device' ? DEVICES : key === 'dtype' ? DTYPES
+      : key === 'webgpuBackend' ? WEBGPU_BACKENDS : key === 'fallback' ? ['error', 'cpu'] : null;
+    if (choices && !choices.includes(v)) throw new Error(`${source}: ${key} deve essere uno di: ${choices.join(', ')}`);
+    if (key === 'offline' && typeof v !== 'boolean') throw new Error(`${source}: offline deve essere booleano`);
+    if (key === 'cacheDir' && (typeof v !== 'string' || !v.trim())) throw new Error(`${source}: cacheDir deve essere un percorso non vuoto`);
+    if (key === 'cpuThreads' && (!Number.isInteger(v) || v < 0 || v > 256)) throw new Error(`${source}: cpuThreads deve essere un intero tra 0 e 256`);
+  }
+  return value;
+}
+
+export function parseSetting(key, text) {
+  if (!Object.hasOwn(ENV_KEYS, key)) throw new Error(`chiave sconosciuta "${key}"`);
+  let value = text;
+  if (key === 'offline') {
+    if (!['true', 'false', '1', '0'].includes(text)) throw new Error('offline: usa true oppure false');
+    value = text === 'true' || text === '1';
+  } else if (key === 'cpuThreads') {
+    if (!/^\d+$/.test(text)) throw new Error('cpuThreads: atteso un intero tra 0 e 256');
+    value = Number(text);
+  }
+  validateConfig({ [key]: value });
+  return value;
+}
+
+async function readConfig(file) {
+  try {
+    return validateConfig(JSON.parse(await fs.readFile(file, 'utf8')), file);
+  } catch (e) {
+    if (e.code === 'ENOENT') return {};
+    throw new Error(`configurazione non valida: ${e.message}`);
+  }
+}
+
+export async function loadConfig({ env = process.env, home = os.homedir() } = {}) {
+  const file = configPath(env, home);
+  const stored = await readConfig(file);
+  const config = { ...defaults(home), ...stored };
+  delete config.$schema;
+  const sources = Object.fromEntries(Object.keys(config).map((key) => [key, key in stored ? file : 'default']));
+  for (const [key, name] of Object.entries(ENV_KEYS)) {
+    if (env[name] === undefined) continue;
+    config[key] = parseSetting(key, env[name]);
+    sources[key] = name;
+  }
+  validateConfig(config, file);
+  config.cacheDir = expandPath(config.cacheDir, home);
+  return { config: Object.freeze(config), sources, file };
+}
+
+// Salva solo valori espliciti, mai gli override dell'ambiente. Temp unico per
+// non interferire con altre sessioni; rename atomico e permessi privati.
+export async function updateConfig(patch, { unset = [], env = process.env, home = os.homedir() } = {}) {
+  validateConfig(patch);
+  for (const key of unset) if (!Object.hasOwn(ENV_KEYS, key)) throw new Error(`chiave sconosciuta "${key}"`);
+  const file = configPath(env, home);
+  const stored = { ...await readConfig(file), ...patch };
+  for (const key of unset) delete stored[key];
+  validateConfig(stored, file);
+  await fs.mkdir(path.dirname(file), { recursive: true });
+  const tmp = `${file}.${randomUUID()}.tmp`;
+  try {
+    await fs.writeFile(tmp, `${JSON.stringify(stored, null, 2)}\n`, { mode: 0o600, flag: 'wx' });
+    await fs.rename(tmp, file);
+  } finally {
+    await fs.rm(tmp, { force: true });
+  }
+  return file;
+}
+
+export function formatConfig({ config, sources, file }) {
+  return [`file: ${file}`, ...Object.entries(config).map(([key, value]) =>
+    `${key.padEnd(12)}: ${value} (${sources[key]})`)].join('\n');
+}
+
+export async function configCommand(args, options = {}) {
+  const [action = 'show', key, ...rest] = args;
+  if (action === 'path' && args.length === 1) return configPath(options.env, options.home);
+  if (action === 'show' && args.length <= 1) return formatConfig(await loadConfig(options));
+  let file;
+  if (action === 'set' && key && rest.length) file = await updateConfig({ [key]: parseSetting(key, rest.join(' ')) }, options);
+  else if (action === 'unset' && key && !rest.length) file = await updateConfig({}, { ...options, unset: [key] });
+  else throw new Error('uso: config [show|path|set CHIAVE VALORE|unset CHIAVE]');
+  return `salvato: ${file}\nRiavvia pi (nuovo processo) per applicare. Gli override CODEINDEX_* hanno precedenza.\n${formatConfig(await loadConfig(options))}`;
+}

package/src/doctor.js

@@ -0,0 +1,96 @@
+// Diagnostica senza download per default. Le prove reali sono isolate in un
+// processo figlio con timeout: un crash nativo non abbatte la sessione pi.
+import { fork } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { loadConfig, parseSetting } from './config.js';
+import { nativeBackends, formatProfile } from './backend.js';
+import { vulkanStatus } from './vulkan.js';
+
+export function parseDoctorArgs(args) {
+  const options = { probe: false, all: false, json: false };
+  for (let i = 0; i < args.length; i++) {
+    const flag = args[i];
+    if (flag === '--probe') options.probe = true;
+    else if (flag === '--all') options.all = true;
+    else if (flag === '--json') options.json = true;
+    else if (flag === '--device' || flag === '--dtype' || flag === '--webgpu-backend') {
+      const value = args[++i];
+      if (!value || value.startsWith('--')) throw new Error(`manca il valore di ${flag}`);
+      const key = flag === '--webgpu-backend' ? 'webgpuBackend' : flag.slice(2);
+      options[key] = parseSetting(key, value);
+    } else throw new Error(`opzione doctor sconosciuta: ${flag}`);
+  }
+  if (options.all && !options.probe) throw new Error('--all richiede --probe');
+  if (options.all && options.device) throw new Error('scegli --all oppure --device, non entrambi');
+  if (options.webgpuBackend && !options.all) {
+    if (options.device && options.device !== 'webgpu') throw new Error('--webgpu-backend richiede --device webgpu oppure --all');
+    options.device = 'webgpu';
+  }
+  return options;
+}
+
+export function probeBackend({ device, dtype, webgpuBackend, timeoutMs = 120_000, env = process.env } = {}) {
+  return new Promise((resolve) => {
+    const childEnv = { ...env, CODEINDEX_FALLBACK: 'error' };
+    if (device) childEnv.CODEINDEX_DEVICE = device;
+    if (dtype) childEnv.CODEINDEX_DTYPE = dtype;
+    if (webgpuBackend) childEnv.CODEINDEX_WEBGPU_BACKEND = webgpuBackend;
+    const child = fork(fileURLToPath(new URL('./probe.js', import.meta.url)), [], { env: childEnv, silent: true, execArgv: [] });
+    let response = null, stderr = '', spawnError = null, timedOut = false;
+    const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, timeoutMs);
+    child.stderr.on('data', (s) => { stderr = (stderr + s).slice(-6000); });
+    child.stdout.on('data', () => {}); // drena eventuali messaggi del binding
+    child.on('message', (value) => { if (value && typeof value.ok === 'boolean') response = value; });
+    child.on('error', (e) => { spawnError = e.message; });
+    child.on('close', (code, signal) => {
+      clearTimeout(timer);
+      const ok = !timedOut && code === 0 && response?.ok === true;
+      resolve({ ...response, requested: device ?? 'auto', ok, code, signal,
+        ...(!ok ? { error: timedOut ? `timeout dopo ${timeoutMs} ms` : response?.error ?? spawnError ?? `processo terminato (${signal ?? code})`, stderr } : {}) });
+    });
+  });
+}
+
+export async function doctor({ probe = false, all = false, device, dtype, webgpuBackend, timeoutMs, env = process.env } = {}) {
+  if (device) parseSetting('device', device);
+  if (dtype) parseSetting('dtype', dtype);
+  if (webgpuBackend) {
+    parseSetting('webgpuBackend', webgpuBackend);
+    if (!all) {
+      if (device && device !== 'webgpu') throw new Error('--webgpu-backend richiede --device webgpu oppure --all');
+      device = 'webgpu';
+    }
+  }
+  const settings = await loadConfig({ env });
+  const native = nativeBackends();
+  const report = { ok: true, platform: process.platform, arch: process.arch, node: process.version,
+    runtime: native.version, settings,
+    requested: { device: device ?? settings.config.device, dtype: dtype ?? settings.config.dtype,
+      webgpuBackend: webgpuBackend ?? settings.config.webgpuBackend },
+    backends: native.backends, vulkan: vulkanStatus(), probes: [] };
+  if (probe) {
+    const devices = all ? native.backends.filter((b) => b.available).map((b) => b.device) : [device ?? settings.config.device];
+    for (const selected of devices) {
+      report.probes.push(await probeBackend({ device: selected, dtype, webgpuBackend, timeoutMs, env }));
+    }
+    report.ok = report.probes.length > 0 && report.probes.every((p) => p.ok);
+  }
+  return report;
+}
+
+export function formatDoctor(report) {
+  const lines = [`${report.platform}/${report.arch} · Node ${report.node} · ONNX Runtime ${report.runtime}`,
+    `config: ${report.settings.file}`,
+    `richiesto: ${(report.requested ?? report.settings.config).device}/${(report.requested ?? report.settings.config).dtype} · fallback ${report.settings.config.fallback} · offline ${report.settings.config.offline}`,
+    ...report.backends.map((b) => `${b.device.padEnd(8)}: ${b.available ? 'presente' : 'assente'} — ${b.reason}`),
+    `backend WebGPU richiesto: ${(report.requested ?? report.settings.config).webgpuBackend} (solo quando il provider selezionato è WebGPU)`,
+    `Vulkan forzato: ${report.vulkan?.available ? 'presente' : 'assente'} — ${report.vulkan?.reason ?? 'non verificato'}`];
+  if (!report.probes.length) lines.push('Nessuna inferenza eseguita. Usa doctor --probe (può scaricare i pesi), oppure --probe --all.');
+  for (const p of report.probes) {
+    lines.push(p.ok ? `PASS ${p.requested}: ${formatProfile(p.profile)} · ${p.profile.dims}d · ${p.elapsedMs} ms · cos(stesso) ${p.cosSame.toFixed(6)}`
+      : `FAIL ${p.requested}: ${p.error}`);
+    for (const a of p.attempts ?? []) if (!a.ok) lines.push(`  ${formatProfile(a)}: ${a.error}`);
+  }
+  lines.push('Un provider presente non prova compatibilità/velocità del modello; ORT può eseguire alcuni operatori su CPU.');
+  return lines.join('\n');
+}

package/src/embed.js

@@ -0,0 +1,88 @@
+// Embedding locale in-process. Selezione del provider verificata con il
+// modello reale; il profilo resta fisso per tutta la vita del processo.
+import { loadConfig } from './config.js';
+import { createEmbeddingRuntime, nativeBackends, validateEmbeddings, MODEL_ID, DIMS, QUERY_PREFIX, docPrefix } from './backend.js';
+import { ensureTextModel, modelPath } from './model.js';
+
+export { MODEL_ID, DIMS, QUERY_PREFIX, docPrefix };
+
+const runtime = createEmbeddingRuntime({
+  settings: () => loadConfig(),
+  inventory: async () => nativeBackends().backends,
+  log: (s) => process.stderr.write(`${s}\n`),
+  initialize: async (profile, config) => {
+    const sessionOptions = {};
+    if (profile.webgpuBackend === 'vulkan') {
+      // Il setup mancante fallisce PRIMA di scaricare il modello. Backend unico,
+      // esplicito e non registrato come fallback dei provider standard.
+      const { registerVulkanBackend } = await import('./vulkan.js');
+      sessionOptions.executionProviders = [await registerVulkanBackend()];
+    }
+    const dir = await ensureTextModel(profile, config);
+    // Import lazy: config/status/doctor senza --probe non caricano il modello.
+    const { pipeline } = await import('@huggingface/transformers');
+    if (config.cpuThreads) sessionOptions.intraOpNumThreads = config.cpuThreads;
+    if (profile.device === 'dml') {
+      sessionOptions.executionMode = 'sequential';
+      sessionOptions.enableMemPattern = false;
+    }
+    let pipe;
+    try {
+      pipe = await pipeline('feature-extraction', dir, {
+        dtype: profile.dtype, device: profile.device, local_files_only: true,
+        session_options: sessionOptions,
+      });
+      const out = await pipe([QUERY_PREFIX + 'code retrieval smoke test'], { pooling: 'mean', normalize: true });
+      validateEmbeddings(out.tolist(), 1);
+      return pipe;
+    } catch (e) {
+      try { await pipe?.dispose(); } catch { /* conserva l'errore originale */ }
+      throw e;
+    }
+  },
+});
+
+export async function getEmbedder() {
+  return (await runtime.get()).pipe;
+}
+
+export async function getEmbeddingProfile() {
+  return (await runtime.get()).profile;
+}
+
+export function embeddingStatus() {
+  return runtime.info();
+}
+
+/** Testi -> Float32Array[] finiti, normalizzati L2 per default. */
+export async function embed(texts, { normalize = true } = {}) {
+  if (!Array.isArray(texts) || texts.length === 0) return [];
+  const { pipe } = await runtime.get();
+  // Non fare fallback qui: l'indice è già associato al profilo selezionato.
+  const out = await pipe(texts, { pooling: 'mean', normalize });
+  const rows = validateEmbeddings(out.tolist(), texts.length);
+  return rows.map((row) => Float32Array.from(row));
+}
+
+export function embedQuery(q) {
+  return embed([QUERY_PREFIX + q]);
+}
+
+export function embedDocs(docs) {
+  return embed(docs.map((d) => typeof d === 'string' ? d : docPrefix(d.path, d.title) + d.text));
+}
+
+export function cosine(a, b) {
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1);
+}
+
+export async function modelDir() {
+  const { profile, config } = await runtime.get();
+  return modelPath(config, profile.dtype);
+}

package/src/eval.js

@@ -0,0 +1,72 @@
+// Eval minima: query a risposta nota -> recall@k + latenza.
+// Fixture JSONL: {"query": "...", "expects": ["substring/path"], "why": "..."}
+
+import fs from 'node:fs/promises';
+import { search } from './search.js';
+import { repoRoot } from './util.js';
+
+const percentile = (arr, p) => {
+  if (arr.length === 0) return 0;
+  const s = [...arr].sort((a, b) => a - b);
+  return s[Math.min(s.length - 1, Math.floor((s.length - 1) * p))];
+};
+
+export async function loadQueries(file) {
+  const text = await fs.readFile(file, 'utf8');
+  return text
+    .split('\n')
+    .filter((l) => l.trim() && !l.trim().startsWith('//'))
+    .map((l, i) => {
+      try {
+        return JSON.parse(l);
+      } catch (e) {
+        throw new Error(`riga ${i + 1} di ${file}: JSON non valido (${e.message})`);
+      }
+    });
+}
+
+export async function runEval({ root, file, k = 8, quiet = false } = {}) {
+  const repo = root ?? repoRoot(process.cwd());
+  if (!repo) throw new Error('non è un repo git: passa --root');
+  const queries = await loadQueries(file);
+
+  const t0 = performance.now();
+  await search('warmup', { root: repo, k: 1 }); // carica modello e indice
+  const coldMs = Math.round(performance.now() - t0);
+
+  const rows = [];
+  const latencies = [];
+  for (const q of queries) {
+    const res = await search(q.query, { root: repo, k });
+    latencies.push(res.tookMs);
+    const top = res.results;
+    const matches = (n) =>
+      top.slice(0, n).some((r) => q.expects.some((e) => r.path.toLowerCase().includes(e.toLowerCase())));
+    rows.push({
+      query: q.query,
+      hit3: matches(3),
+      hit8: matches(8),
+      rank: top.findIndex((r) => q.expects.some((e) => r.path.toLowerCase().includes(e.toLowerCase()))) + 1,
+      top: top.slice(0, 3).map((r) => `${r.path}:${r.start}-${r.end}`),
+      tookMs: res.tookMs,
+      why: q.why,
+    });
+    if (!quiet) {
+      const mark = rows.at(-1).hit3 ? 'OK  ' : rows.at(-1).hit8 ? 'ok@8' : 'MISS';
+      console.log(`${mark} ${res.tookMs.toFixed(0).padStart(4)} ms · ${q.query}`);
+      console.log(`     attesi: ${q.expects.join(' | ')}`);
+      console.log(`     top:    ${rows.at(-1).top.join('  ·  ')}`);
+    }
+  }
+
+  return {
+    repo,
+    queries: rows.length,
+    recall3: rows.filter((r) => r.hit3).length / rows.length,
+    recall8: rows.filter((r) => r.hit8).length / rows.length,
+    coldMs,
+    latencyP50: Math.round(percentile(latencies, 0.5)),
+    latencyP95: Math.round(percentile(latencies, 0.95)),
+    rows,
+  };
+}