@alucard_24/pi-codeindex 0.0.0-stage → 1.1.0

package/scripts/setup-vulkan.mjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+// Setup esplicito, nessun postinstall. Compila solo il binding Node, non ORT/Dawn.
+// Sorgenti MIT upstream fissati; copia ORT privata con SONAME distinto, senza
+// toccare node_modules o il contesto WebGPU di altre estensioni.
+import fs from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { createHash, randomUUID } from 'node:crypto';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+import { ORT_VERSION, ORT_SOURCE, VULKAN_REVISION, fileHash, ortLibrary, vulkanCacheDir,
+  vulkanStatus, verifyVulkanArtifacts } from '../src/vulkan.js';
+
+const require = createRequire(import.meta.url);
+const ADDON_URL = 'https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz';
+const ADDON_INTEGRITY = '+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==';
+
+export function patchBindingSource(source) {
+  const anchor = 'if (name == "preferredLayout" ||';
+  if (source.split(anchor).length !== 2) throw new Error('patch Vulkan: sorgente upstream inatteso');
+  return source.replace(anchor, 'if (name == "dawnBackendType" ||\n              name == "preferredLayout" ||');
+}
+
+function run(command, args, options = {}) {
+  const result = spawnSync(command, args, { stdio: 'inherit', timeout: 180_000, ...options });
+  if (result.error) throw result.error;
+  if (result.status !== 0) throw new Error(`${command} terminato con ${result.signal ?? result.status}`);
+}
+async function download(url) {
+  const response = await fetch(url, { signal: AbortSignal.timeout(60_000) });
+  if (!response.ok) throw new Error(`download ${response.status}: ${url}`);
+  return Buffer.from(await response.arrayBuffer());
+}
+
+export async function setupVulkan(args = []) {
+  let headers, force = false;
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--force') force = true;
+    else if (args[i] === '--node-headers' && args[i + 1] && !args[i + 1].startsWith('--')) headers = path.resolve(args[++i]);
+    else throw new Error('uso: setup-vulkan [--force] [--node-headers DIR]');
+  }
+  if (process.platform !== 'linux' || process.arch !== 'x64') throw new Error('setup-vulkan inizialmente disponibile solo su Linux x64; Windows/macOS non verificati');
+  if (require('onnxruntime-node/package.json').version !== ORT_VERSION) throw new Error(`setup-vulkan richiede onnxruntime-node ${ORT_VERSION}`);
+  if (!force) {
+    const status = vulkanStatus();
+    if (status.available) {
+      try { await verifyVulkanArtifacts(status); console.log(`Vulkan: binding già pronto in ${status.dir}`); return; }
+      catch (e) { console.log(e.message); }
+    }
+  }
+  const compiler = process.env.CXX || 'c++';
+  run(compiler, ['--version'], { stdio: 'ignore' });
+  run('patchelf', ['--version'], { stdio: 'ignore' });
+  run('tar', ['--version'], { stdio: 'ignore' });
+  headers ??= [path.resolve(path.dirname(process.execPath), '../include/node'), '/usr/include/node']
+    .find((dir) => existsSync(path.join(dir, 'node_api.h')));
+  if (!headers || !existsSync(path.join(headers, 'node_api.h'))) throw new Error('header Node mancanti: installa gli header di sviluppo o usa --node-headers DIR');
+
+  const dir = vulkanCacheDir();
+  await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+  const work = await fs.mkdtemp(path.join(dir, '.build-'));
+  try {
+    const sourceManifest = JSON.parse(await fs.readFile(new URL('./vulkan-sources.json', import.meta.url), 'utf8'));
+    if (sourceManifest.revision !== ORT_SOURCE) throw new Error('revision sorgenti Vulkan incoerente');
+    console.log(`Vulkan: sorgenti ORT ${ORT_VERSION} @ ${ORT_SOURCE.slice(0, 12)} (hash Git verificati)`);
+    const sourceDir = path.join(work, 'source');
+    const entries = Object.entries(sourceManifest.files);
+    // Download limitato: solo binding e header pubblici, niente clone del repository.
+    for (let i = 0; i < entries.length; i += 4) await Promise.all(entries.slice(i, i + 4).map(async ([file, expected]) => {
+      const bytes = await download(`https://raw.githubusercontent.com/microsoft/onnxruntime/${ORT_SOURCE}/${file}`);
+      const actual = createHash('sha1').update(`blob ${bytes.length}\0`).update(bytes).digest('hex');
+      if (actual !== expected) throw new Error(`hash sorgente inatteso: ${file}`);
+      const dest = path.join(sourceDir, file);
+      await fs.mkdir(path.dirname(dest), { recursive: true });
+      await fs.writeFile(dest, bytes);
+    }));
+    const helper = path.join(sourceDir, 'js/node/src/session_options_helper.cc');
+    await fs.writeFile(helper, patchBindingSource(await fs.readFile(helper, 'utf8')));
+    const tarball = await download(ADDON_URL);
+    if (createHash('sha512').update(tarball).digest('base64') !== ADDON_INTEGRITY) throw new Error('integrità node-addon-api non valida');
+    const archive = path.join(work, 'addon-api.tgz');
+    await fs.writeFile(archive, tarball);
+    const addonDir = path.join(work, 'addon-api');
+    await fs.mkdir(addonDir);
+    run('tar', ['-xzf', archive, '--strip-components=1', '-C', addonDir]);
+
+    const out = path.join(work, 'artifact');
+    await fs.mkdir(out);
+    const sourceLibraryHash = await fileHash(ortLibrary());
+    const library = `libpi-codeindex-ort-vulkan-${sourceLibraryHash.slice(0, 16)}.so.1`;
+    const libraryFile = path.join(out, library);
+    await fs.copyFile(ortLibrary(), libraryFile);
+    run('patchelf', ['--set-soname', library, libraryFile]);
+    const nodeDir = path.join(sourceDir, 'js/node/src');
+    const cpp = (await fs.readdir(nodeDir)).filter((f) => f.endsWith('.cc')).sort().map((f) => path.join(nodeDir, f));
+    console.log('Vulkan: compilazione binding isolato (C++/N-API 6), runtime precompilato riutilizzato');
+    const bindingFile = path.join(out, 'onnxruntime_binding.node');
+    run(compiler, ['-shared', '-fPIC', '-fvisibility=hidden', '-fvisibility-inlines-hidden', '-std=c++17', '-O2',
+      '-DNAPI_VERSION=6', '-DUSE_WEBGPU', '-DNAPI_CPP_EXCEPTIONS', '-DORT_API_MANUAL_INIT',
+      `-I${headers}`, `-I${addonDir}`, `-I${path.join(sourceDir, 'include/onnxruntime')}`,
+      `-I${path.join(sourceDir, 'include/onnxruntime/core/session')}`, ...cpp,
+      `-L${out}`, `-l:${library}`, '-Wl,-rpath,$ORIGIN', '-o', bindingFile]);
+    await fs.copyFile(path.join(sourceDir, 'LICENSE'), path.join(out, 'LICENSE-onnxruntime'));
+    await fs.copyFile(path.join(addonDir, 'LICENSE.md'), path.join(out, 'LICENSE-node-addon-api'));
+    // Il controllo negativo non richiede GPU o pesi. Nuovo processo, nessun
+    // addon tenuto aperto durante la pubblicazione degli artefatti.
+    const moduleUrl = new URL('../src/vulkan.js', import.meta.url).href;
+    run(process.execPath, ['--input-type=module', '-e', `
+      import { createRequire } from 'node:module';
+      import { assertVulkanSelector } from ${JSON.stringify(moduleUrl)};
+      process.env.ORT_DISABLE_TELEMETRY = '1';
+      const req = createRequire(${JSON.stringify(import.meta.url)});
+      const ort = req('onnxruntime-node');
+      const binding = req(${JSON.stringify(bindingFile)});
+      binding.initOrtOnce(2, ort.Tensor, true);
+      assertVulkanSelector(binding);
+      console.log('Vulkan: controllo negativo del selettore superato');
+    `]);
+    const bindingHash = await fileHash(bindingFile);
+    const libraryHash = await fileHash(libraryFile);
+    // Directory nuova ad ogni build: non riusare/sovrascrivere artefatti
+    // corrotti o ancora mappati da un altro processo. Il manifest si pubblica ultimo.
+    const artifact = createHash('sha256').update(bindingHash + libraryHash + randomUUID()).digest('hex');
+    const target = path.join(dir, artifact);
+    await fs.rename(out, target);
+    const manifest = { runtimeVersion: ORT_VERSION, sourceRevision: ORT_SOURCE, revision: VULKAN_REVISION,
+      platform: process.platform, arch: process.arch, artifact, library, sourceLibraryHash, bindingHash, libraryHash };
+    const tmp = path.join(dir, `manifest.${randomUUID()}.tmp`);
+    try {
+      await fs.writeFile(tmp, JSON.stringify(manifest, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
+      await fs.rename(tmp, path.join(dir, 'manifest.json'));
+    } finally { await fs.rm(tmp, { force: true }); }
+    console.log(`Vulkan: installato in ${target}\nNessun file in node_modules modificato. Riavvia pi se già aperto. Ora esegui doctor --probe --device webgpu --webgpu-backend vulkan.`);
+  } finally { await fs.rm(work, { recursive: true, force: true }); }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  setupVulkan(process.argv.slice(2)).catch((e) => { console.error(`setup-vulkan: ${e.message}`); process.exitCode = 1; });
+}

package/scripts/vulkan-sources.json

@@ -0,0 +1,29 @@
+{
+  "revision": "f2c39fe2f838cf35ce7da92824f5a5e3ee6e88a7",
+  "files": {
+    "js/node/src/common.h": "b60d059bb673b9685b132a6189ff32976b492c04",
+    "js/node/src/inference_session_wrap.cc": "648a4586ababc855be8ca77a0ee655552c1eb71c",
+    "js/node/src/inference_session_wrap.h": "13132a8a2d744d5e471db72037c825928f19c7c9",
+    "js/node/src/main.cc": "c6c2a5cb8dc42c7e92b0c87181663b68c7d7c639",
+    "js/node/src/ort_instance_data.cc": "825cb953e01371baec5ab627e6f1ad92ecaee0cf",
+    "js/node/src/ort_instance_data.h": "68cb6028548d4cb2656d8f9e0d7cb44bfac75681",
+    "js/node/src/ort_singleton_data.cc": "d1d57a10340c5a90996b6bc5affbff8869a7a85c",
+    "js/node/src/ort_singleton_data.h": "a0577cb71e98bd37851852040ffc7e244b7d41dd",
+    "js/node/src/run_options_helper.cc": "352f828970c66767b1a80a34a29da06a272d111d",
+    "js/node/src/run_options_helper.h": "104fae150bb0eb6661b670926974c81eabc454fa",
+    "js/node/src/session_options_helper.cc": "9cfb44eb9acb75d38e845b30f847c6fdd3632592",
+    "js/node/src/session_options_helper.h": "c6338f28e03c6f21f657c1ff654864d696a5bbb8",
+    "js/node/src/tensor_helper.cc": "c4bf5628ce6fe0e662f60f2340adb83773dabc90",
+    "js/node/src/tensor_helper.h": "4a51e5240602accae3d5e75fd249f756ef34dea7",
+    "include/onnxruntime/core/session/onnxruntime_c_api.h": "a254e990237d5575186cf3871392db8c70835a56",
+    "include/onnxruntime/core/session/onnxruntime_cxx_api.h": "8a64ec9159063d97f3c5085716dd8b456bd9b768",
+    "include/onnxruntime/core/session/onnxruntime_cxx_inline.h": "0db4a7029a16389c4140c497840b83492313d0fb",
+    "include/onnxruntime/core/session/onnxruntime_ep_c_api.h": "0424a95c2d2c1102f328b4e3aeb77ec72601a120",
+    "include/onnxruntime/core/session/onnxruntime_float16.h": "625a1e82a6504577da24e83074643f7dfc13303d",
+    "include/onnxruntime/core/session/onnxruntime_error_code.h": "732fcd7a82eaa0208c7c97aa5f2fb2b0c694456f",
+    "include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h": "f40ea6591059eb4afb574037732de8f7bc7575c1",
+    "include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h": "84e16a692d33178bcf29de7eb410c45680ab84b5",
+    "include/onnxruntime/core/providers/webgpu/webgpu_provider_factory.h": "0b45b847d651ff922a8e72856354e4ea6f95f439",
+    "LICENSE": "48bc6bb4996ac924359e8e28b9ae88970e5ed3fc"
+  }
+}

package/src/backend.js

@@ -0,0 +1,120 @@
+// Policy dei backend separata dall'inferenza: testabile senza GPU né download.
+import { createRequire } from 'node:module';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+
+export const MODEL_ID = 'onnx-community/embeddinggemma-2-ONNX';
+export const DIMS = 768;
+export const QUERY_PREFIX = 'task: code retrieval | query: ';
+export const docPrefix = (filePath, title) => `title: ${title ?? filePath ?? 'none'} | text: `;
+export const BACKENDS = ['cpu', 'cuda', 'webgpu', 'dml', 'coreml'];
+const PLATFORMS = { cpu: ['linux', 'darwin', 'win32'], cuda: ['linux'],
+  webgpu: ['linux', 'darwin', 'win32'], dml: ['win32'], coreml: ['darwin'] };
+
+export function nativeBackends() {
+  const req = createRequire(import.meta.url);
+  const ort = req('onnxruntime-node');
+  const dir = path.dirname(req.resolve('onnxruntime-node/package.json'));
+  const cudaLibrary = existsSync(path.join(dir, 'bin', 'napi-v6', process.platform, process.arch, 'libonnxruntime_providers_cuda.so'));
+  return { version: ort.env.versions.node, backends: backendInventory(ort.listSupportedBackends(), { cudaLibrary }) };
+}
+
+export function backendInventory(listed, { platform = process.platform, arch = process.arch, cudaLibrary = false } = {}) {
+  return BACKENDS.map((device) => {
+    const entry = listed.find((b) => b.name === device);
+    const platformOk = PLATFORMS[device].includes(platform) && (device !== 'cuda' || arch === 'x64');
+    const available = platformOk && Boolean(entry && (entry.bundled || (device === 'cuda' && cudaLibrary)));
+    const reason = !platformOk ? `non supportato su ${platform}/${arch}` : !entry ? 'non esposto dal binding nativo'
+      : !available ? 'provider non installato (CUDA/Linux: npm run setup-gpu)' : 'provider presente; inferenza da verificare';
+    return { device, available, reason };
+  });
+}
+
+export function backendPlan(config, inventory, platform = process.platform) {
+  const order = platform === 'win32' ? ['dml', 'webgpu', 'cpu']
+    : platform === 'darwin' ? ['coreml', 'webgpu', 'cpu'] : ['cuda', 'webgpu', 'cpu'];
+  const devices = config.device === 'auto' ? order.filter((d) => inventory.some((b) => b.device === d && b.available))
+    : [config.device, ...(config.fallback === 'cpu' && config.device !== 'cpu' ? ['cpu'] : [])];
+  return devices.map((device) => ({
+    device,
+    dtype: config.dtype === 'auto' ? (device === 'cpu' ? 'q8' : 'fp16') : config.dtype,
+    ...inventory.find((b) => b.device === device),
+  }));
+}
+
+export function webgpuBackendOf(profile) {
+  return profile.device === 'webgpu' ? (profile.webgpuBackend ?? 'auto') : null;
+}
+
+export function formatProfile(profile) {
+  return `${profile.device}${profile.device === 'webgpu' ? `→${webgpuBackendOf(profile)}` : ''}/${profile.dtype}`;
+}
+
+export function assertCompatibleIndex(index, profile) {
+  const mismatches = ['model', 'dtype', 'dims', 'device'].filter((key) => index[key] !== profile[key]);
+  if (webgpuBackendOf(index) !== webgpuBackendOf(profile)) mismatches.push('webgpuBackend');
+  if (mismatches.length) throw new Error(`indice incompatibile (${mismatches.join(', ')}): esegui /codeindex reindex o il comando index con questo profilo`);
+}
+
+export function validateEmbeddings(rows, count, dims = DIMS) {
+  if (!Array.isArray(rows) || rows.length !== count) throw new Error('embedding: numero di righe inatteso');
+  for (const row of rows) {
+    if (!row || row.length !== dims) throw new Error(`embedding: attese ${dims} dimensioni`);
+    let norm = 0;
+    for (const n of row) {
+      if (!Number.isFinite(n)) throw new Error('embedding: valori non finiti');
+      norm += n * n;
+    }
+    if (norm < 1e-12) throw new Error('embedding: vettore nullo');
+  }
+  return rows;
+}
+
+// Il profilo si fissa SOLO dopo un'inferenza riuscita. Non cambia in seguito,
+// neanche dopo OOM/errori: nessun cambio di dtype durante un'indicizzazione.
+export function createEmbeddingRuntime({ settings, inventory, initialize, platform = process.platform, log = () => {} }) {
+  let ready = null;
+  let loading = null;
+  let attempts = [];
+  let error = null;
+  let activeConfig = null;
+  async function get() {
+    if (ready) return ready;
+    if (loading) return loading;
+    loading = (async () => {
+      attempts = [];
+      error = null;
+      try {
+        const loaded = await settings();
+        activeConfig = loaded.config;
+        const plan = backendPlan(activeConfig, await inventory(), platform);
+        for (const candidate of plan) {
+          const profile = Object.freeze({ model: MODEL_ID, dims: DIMS, device: candidate.device, dtype: candidate.dtype,
+            webgpuBackend: candidate.device === 'webgpu' ? (activeConfig.webgpuBackend ?? 'auto') : null });
+          try {
+            if (!candidate.available) throw new Error(candidate.reason ?? 'provider assente');
+            const pipe = await initialize(profile, activeConfig);
+            attempts.push({ ...profile, ok: true });
+            ready = { pipe, profile, config: activeConfig };
+            log(`[codeindex] modello pronto · ${formatProfile(profile)}`);
+            return ready;
+          } catch (e) {
+            const message = String(e.message ?? e).slice(0, 1000);
+            attempts.push({ ...profile, ok: false, error: message });
+            log(`[codeindex] ${formatProfile(profile)} non disponibile: ${message}`);
+          }
+        }
+        throw new Error(`nessun backend utilizzabile: ${attempts.map((a) => `${formatProfile(a)}: ${a.error}`).join('; ')}. Usa doctor --probe per la diagnostica.`);
+      } catch (e) {
+        error = e.message;
+        throw e;
+      }
+    })();
+    try { return await loading; } finally { loading = null; }
+  }
+  return {
+    get,
+    info: () => ({ profile: ready?.profile ?? null, config: activeConfig, loading: Boolean(loading),
+      attempts: attempts.map((a) => ({ ...a })), error }),
+  };
+}

package/src/chunker.js

@@ -0,0 +1,276 @@
+// Chunker declaration-aware + fallback a finestre.
+// Puro: nessun I/O, nessuna dipendenza esterna. Versionato: cambiare la logica
+// richiede un bump di CHUNKER_VERSION (invalida i vettori esistenti).
+
+export const CHUNKER_VERSION = 1;
+export const MAX_LINES = 120; // oltre: split a finestre
+export const SPLIT_LINES = 60;
+export const MAX_EMBED_CHARS = 4000;
+
+export const LANG_BY_EXT = {
+  '.js': 'js', '.mjs': 'js', '.cjs': 'js', '.jsx': 'js',
+  '.ts': 'js', '.tsx': 'js', '.mts': 'js', '.cts': 'js',
+  '.py': 'py', '.pyi': 'py',
+  '.go': 'go',
+  '.rs': 'rust',
+  '.c': 'c', '.h': 'c',
+  '.cc': 'cpp', '.cpp': 'cpp', '.cxx': 'cpp', '.hpp': 'cpp', '.hh': 'cpp', '.hxx': 'cpp',
+  '.cs': 'csharp',
+  '.srw': 'powerbuilder', '.sru': 'powerbuilder', '.srf': 'powerbuilder',
+  '.srs': 'powerbuilder', '.srm': 'powerbuilder', '.srd': 'powerbuilder',
+  '.sra': 'powerbuilder',
+  '.md': 'md', '.mdx': 'md',
+};
+
+export function langOf(filePath) {
+  const m = /\.[^./]+$/.exec(filePath);
+  return m ? (LANG_BY_EXT[m[0].toLowerCase()] ?? 'generic') : 'generic';
+}
+
+const COMMENT = /^\s*(?:\/\/|\/\*|\*|#|--|;|'|<!--)/;
+
+const JS_KEYWORDS = new Set([
+  'if', 'for', 'while', 'switch', 'catch', 'return', 'function', 'else', 'do',
+  'try', 'with', 'await', 'yield', 'typeof', 'new', 'super', 'this',
+]);
+
+const C_KEYWORDS = new Set([
+  'if', 'for', 'while', 'switch', 'return', 'sizeof', 'do', 'else', 'case',
+  'typedef', 'struct', 'union', 'enum', 'class', 'namespace', 'template',
+]);
+
+/** Riconosce l'inizio di una dichiarazione. Ritorna {symbol, kind} o null. */
+export function matchBoundary(lang, line) {
+  let m;
+  switch (lang) {
+    case 'js': {
+      if ((m = /^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s*\*?\s*([A-Za-z_$][\w$]*)/.exec(line)))
+        return { symbol: m[1], kind: 'function' };
+      if ((m = /^\s*(?:export\s+)?(?:default\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)/.exec(line)))
+        return { symbol: m[1], kind: 'class' };
+      if ((m = /^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*(?::[^=]+)?=\s*(?:async\s*)?(?:function\b|\()/.exec(line)))
+        return { symbol: m[1], kind: 'function' };
+      if ((m = /^\s*(?:export\s+)?(?:declare\s+)?(?:interface|type|enum)\s+([A-Za-z_$][\w$]*)/.exec(line)))
+        return { symbol: m[1], kind: 'type' };
+      if ((m = /^ {2,}(?:(?:public|private|protected|static|async|readonly|override|get|set)\s+)*([A-Za-z_$][\w$]*)\s*\([^;]*\)\s*(?::[^{]+)?\{/.exec(line))) {
+        // `if (x) {`, `for (...) {` ecc. non sono metodi: spezzerebbero il corpo.
+        if (JS_KEYWORDS.has(m[1])) return null;
+        return { symbol: m[1], kind: 'method' };
+      }
+      return null;
+    }
+    case 'py': {
+      if (/^\s*(?:async\s+)?def\s+(\w+)/.test(line)) {
+        const mm = /^\s*(?:async\s+)?def\s+(\w+)/.exec(line);
+        return { symbol: mm[1], kind: 'function' };
+      }
+      if ((m = /^class\s+(\w+)/.exec(line))) return { symbol: m[1], kind: 'class' };
+      return null;
+    }
+    case 'go': {
+      if ((m = /^func\s+(?:\([^)]*\)\s*)?(\w+)/.exec(line))) return { symbol: m[1], kind: 'function' };
+      if ((m = /^type\s+(\w+)/.exec(line))) return { symbol: m[1], kind: 'type' };
+      return null;
+    }
+    case 'rust': {
+      if ((m = /^\s*(?:pub(?:\([^)]*\))?\s+)?(?:const\s+)?(?:unsafe\s+)?(?:async\s+)?(?:extern\s+"[^"]*"\s+)?fn\s+(\w+)/.exec(line)))
+        return { symbol: m[1], kind: 'function' };
+      if ((m = /^\s*(?:pub(?:\([^)]*\))?\s+)?(struct|enum|union|trait|mod)\s+(\w+)/.exec(line)))
+        return { symbol: m[2], kind: m[1] };
+      if ((m = /^\s*(?:pub(?:\([^)]*\))?\s+)?impl\b[^{]*?(?:for\s+)?([\w:<>, ]+?)\s*(?:where|$|\{)/.exec(line)))
+        return { symbol: m[1].trim(), kind: 'impl' };
+      return null;
+    }
+    case 'c':
+    case 'cpp': {
+      if (line.startsWith('#') || /^\s*\/[/*]/.test(line)) return null;
+      if ((m = /^\s*(?:typedef\s+)?(?:class|struct|enum(?:\s+class)?|namespace|union)\s+(\w+)/.exec(line)))
+        return { symbol: m[1], kind: 'type' };
+      if ((m = /^([A-Za-z_][\w\s\*&:<>,]*?[\s\*&])([A-Za-z_]\w*)\s*\([^;]*$/.exec(line))) {
+        if (C_KEYWORDS.has(m[2])) return null;
+        if (/^\s*(if|for|while|switch|return|else|do|case)\b/.test(line)) return null;
+        return { symbol: m[2], kind: 'function' };
+      }
+      return null;
+    }
+    case 'csharp': {
+      if ((m = /^\s*(?:\[[^\]]*\]\s*)*(?:(?:public|private|protected|internal|static|sealed|abstract|partial|readonly|unsafe|file)\s+)*(?:class|struct|interface|enum|record)\s+(\w+)/.exec(line)))
+        return { symbol: m[1], kind: 'type' };
+      if ((m = /^\s*(?:(?:public|private|protected|internal|static|async|virtual|override|sealed|new|extern|unsafe|partial|abstract)\s+)+[\w<>\[\],\.\?]+\s+(\w+)\s*\(/.exec(line)))
+        return { symbol: m[1], kind: 'function' };
+      if ((m = /^\s*(?:(?:public|private|protected|internal|static|virtual|override|abstract|sealed)\s+)+[\w<>\[\],\.\?]+\s+(\w+)\s*\{\s*(?:get|set)/.exec(line)))
+        return { symbol: m[1], kind: 'property' };
+      return null;
+    }
+    case 'powerbuilder': {
+      if ((m = /^global\s+type\s+(\w+)/i.exec(line))) return { symbol: m[1], kind: 'type' };
+      if (/^forward\s+prototypes/i.test(line)) return { symbol: 'forward prototypes', kind: 'prototypes' };
+      if ((m = /^(?:public|private|protected|global)\s+function\s+[\w\[\]]+\s+(\w+)/i.exec(line)))
+        return { symbol: m[1], kind: 'function' };
+      if ((m = /^event\s+(\w+)/i.exec(line))) return { symbol: m[1], kind: 'event' };
+      if ((m = /^on\s+(\w+)/i.exec(line))) return { symbol: m[1], kind: 'event-handler' };
+      return null;
+    }
+    case 'md': {
+      if ((m = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line)))
+        return { symbol: m[2].slice(0, 80), kind: `h${m[1].length}` };
+      return null;
+    }
+    default:
+      return null;
+  }
+}
+
+function trimRange(lines, start, end) {
+  while (start <= end && lines[start - 1].trim() === '') start++;
+  while (end >= start && lines[end - 1].trim() === '') end--;
+  return { start, end };
+}
+
+/** Estende l'inizio verso l'alto includendo il blocco di commento adiacente. */
+function includeLeadingComments(lines, start) {
+  let s = start;
+  while (s > 1 && COMMENT.test(lines[s - 2]) && lines[s - 2].trim() !== '') {
+    s--;
+    if (s > 1 && lines[s - 2].trim() === '') break; // una riga vuota chiude il blocco
+  }
+  return s;
+}
+
+/** Split di un range troppo lungo: finestre a ridosso di righe vuote. */
+function splitRange(lines, start, end, kind) {
+  const out = [];
+  let cur = start;
+  while (cur <= end) {
+    const targetEnd = Math.min(cur + SPLIT_LINES - 1, end);
+    let cut = targetEnd;
+    if (targetEnd < end) {
+      for (let l = targetEnd; l > cur + 10; l--) {
+        if (lines[l - 1].trim() === '') {
+          cut = l;
+          break;
+        }
+      }
+    }
+    out.push({ start: cur, end: cut, kind });
+    cur = cut + 1;
+    while (cur <= end && lines[cur - 1].trim() === '') cur++;
+  }
+  return out;
+}
+
+/**
+ * Fonduzione di chunk adiacenti: riduce il numero di forward pass (le sequenze
+ * corte sprecano padding) senza perdere righe. Con `smallOnly`, fonde solo se
+ * ENTRAMBI i chunk sono più corti della soglia (per non perdere granularità
+ * sulle dichiarazioni di codice).
+ */
+export function mergeRanges(lines, ranges, { maxLines = 40, maxChars = 1000, smallOnly = 0 } = {}) {
+  const len = (r) => r.end - r.start + 1;
+  const out = [];
+  for (const r of ranges) {
+    const prev = out[out.length - 1];
+    if (!prev) {
+      out.push({ ...r, symbols: r.symbol ? [r.symbol] : [] });
+      continue;
+    }
+    const joinedLines = r.end - prev.start + 1;
+    const joinedChars = lines.slice(prev.start - 1, r.end).join('\n').length;
+    if (prev.kind === 'preamble') {
+      out.push({ ...r, symbols: r.symbol ? [r.symbol] : [] });
+      continue;
+    }
+    const smallEnough = !smallOnly || (len(prev) <= smallOnly && len(r) <= smallOnly);
+    if (smallEnough && joinedLines <= maxLines && joinedChars <= maxChars) {
+      prev.end = r.end;
+      if (r.symbol) prev.symbols.push(r.symbol);
+      if (prev.kind !== r.kind) prev.kind = 'group';
+      prev.symbol = prev.symbols[0] ?? null;
+    } else {
+      out.push({ ...r, symbols: r.symbol ? [r.symbol] : [] });
+    }
+  }
+  return out.map((c) => ({
+    start: c.start,
+    end: c.end,
+    symbol: c.symbol,
+    symbols: c.symbols,
+    kind: c.kind,
+  }));
+}
+
+/** Politica di merge per linguaggio. Markdown: sezioni intere. Codice: solo micro-chunk. */
+const MERGE_POLICY = {
+  md: { maxLines: 80, maxChars: 1500 },
+  generic: { maxLines: 0, maxChars: 0 }, // finestre da 60 righe: già ottimali
+  default: { maxLines: 30, maxChars: 900, smallOnly: 6 },
+};
+
+/**
+ * Testo -> chunk. Copertura completa del file (nessuna riga persa).
+ * @returns {{start:number,end:number,symbol:string|null,kind:string}[]}
+ */
+export function chunkText(text, lang = 'generic') {
+  const lines = text.split('\n');
+  if (lines.length > 0 && lines[lines.length - 1] === '') lines.pop();
+  if (lines.length === 0) return [];
+  return chunkLines(lines, lang);
+}
+
+/** Variante su array di righe (per test e per il chunker di file già letti). */
+export function chunkLines(lines, lang = 'generic') {
+  if (lines.length === 0) return [];
+
+  const boundaries = [];
+  for (let i = 0; i < lines.length; i++) {
+    const hit = matchBoundary(lang, lines[i]);
+    if (hit) boundaries.push({ line: i + 1, ...hit });
+  }
+
+  if (boundaries.length === 0) {
+    return splitRange(lines, 1, lines.length, 'window').map((r) => ({ ...r, symbol: null }));
+  }
+
+  const ranges = [];
+
+  // Preambolo (import, header, prime righe) prima della prima dichiarazione.
+  // Il confine include l'eventuale blocco di commento della dichiarazione:
+  // altrimenti il commento finirebbe nel chunk sbagliato.
+  const firstStart = includeLeadingComments(lines, boundaries[0].line);
+  const preamble = trimRange(lines, 1, firstStart - 1);
+  if (preamble.start <= preamble.end && lines.slice(preamble.start - 1, preamble.end).some((l) => l.trim() !== ''))
+    ranges.push({ ...preamble, symbol: null, kind: 'preamble' });
+
+  for (let i = 0; i < boundaries.length; i++) {
+    const b = boundaries[i];
+    const nextLine = i + 1 < boundaries.length ? boundaries[i + 1].line : lines.length + 1;
+    let start = includeLeadingComments(lines, b.line);
+    const prevEnd = ranges.length ? ranges[ranges.length - 1].end : 0;
+    if (start <= prevEnd) start = b.line; // non sovrapporre al chunk precedente
+    const r = trimRange(lines, start, Math.min(nextLine - 1, lines.length));
+    if (r.start > r.end) continue;
+    const body = r.end - r.start + 1;
+    if (body > MAX_LINES) {
+      for (const s of splitRange(lines, r.start, r.end, b.kind)) ranges.push({ ...s, symbol: b.symbol });
+    } else {
+      ranges.push({ ...r, symbol: b.symbol, kind: b.kind });
+    }
+  }
+  const policy = MERGE_POLICY[lang] ?? MERGE_POLICY.default;
+  if (!policy.maxLines) return mergeRanges(lines, ranges, { maxLines: 1, maxChars: 0 });
+  return mergeRanges(lines, ranges, policy);
+}
+
+/** Righe del chunk + eventuale troncamento a MAX_EMBED_CHARS. */
+export function chunkCode(lines, chunk) {
+  const code = lines.slice(chunk.start - 1, chunk.end).join('\n');
+  return code.length > MAX_EMBED_CHARS ? code.slice(0, MAX_EMBED_CHARS) : code;
+}
+
+/** Payload testuale per l'embedding (senza prefisso modello: lo aggiunge il chiamante). */
+export function chunkPayload(chunk, code) {
+  const symbols = chunk.symbols?.length ? chunk.symbols : chunk.symbol ? [chunk.symbol] : [];
+  const head = `${chunk.start}-${chunk.end} ${chunk.kind}${symbols.length ? ` ${symbols.slice(0, 5).join(', ')}` : ''}`;
+  const body = code.length > MAX_EMBED_CHARS ? code.slice(0, MAX_EMBED_CHARS) : code;
+  return `${head}\n${body}`;
+}