@ngockhoale/ukit 2.6.7 → 2.6.9

package/src/core/codeintel/semanticProvider.js

@@ -0,0 +1,235 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { getArtifactPath, INDEX_ARTIFACTS } from '../../index/paths.js';
+import { createEdge, NullSemanticProvider } from './providers.js';
+
+const NO_CAPS = Object.freeze({ defs: false, refs: false, types: false });
+const ALL_CAPS = Object.freeze({ defs: true, refs: true, types: true });
+const DEFAULT_LIMIT = 50;
+const MAX_FILES = 200;
+const SEMANTIC_EXTS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mts', '.cts']);
+
+function defaultLoader() {
+  return import('typescript');
+}
+
+function isSemanticFile(filePath) {
+  return typeof filePath === 'string' && SEMANTIC_EXTS.has(path.extname(filePath));
+}
+
+/**
+ * SemanticProvider backed by the TypeScript language service, loaded via an
+ * injectable async loader (default: dynamic `import('typescript')`). When the
+ * loader fails — package absent, throw, any reason — the provider degrades to
+ * `available()===false`, `resolve()→null`, all-false capabilities. Never throws.
+ */
+export class TypeScriptSemanticProvider {
+  constructor({ projectRoot = process.cwd(), loader = defaultLoader } = {}) {
+    this.name = 'typescript-semantic';
+    this.projectRoot = projectRoot;
+    this._loader = loader;
+    this._probe = null;
+    this._caps = NO_CAPS;
+  }
+
+  #probeTs() {
+    if (!this._probe) {
+      this._probe = Promise.resolve()
+        .then(() => this._loader())
+        .then((ts) => {
+          this._caps = ts ? ALL_CAPS : NO_CAPS;
+          return ts ?? null;
+        })
+        .catch(() => {
+          this._caps = NO_CAPS;
+          return null;
+        });
+    }
+    return this._probe;
+  }
+
+  async available() {
+    return (await this.#probeTs()) !== null;
+  }
+
+  capabilities() {
+    return { ...this._caps };
+  }
+
+  /**
+   * ctx = { file?, mode?: 'definition'|'references'|'both', limit?, snapshot? }
+   * → edge[] (createEdge shape) | null when unavailable or on any failure.
+   */
+  async resolve(symbol, ctx = {}) {
+    try {
+      const ts = await this.#probeTs();
+      if (!ts || typeof symbol !== 'string' || symbol.length === 0) {
+        return null;
+      }
+      const safeCtx = ctx && typeof ctx === 'object' ? ctx : {};
+      const mode = safeCtx.mode ?? 'both';
+      const limit = Number.isInteger(safeCtx.limit) && safeCtx.limit > 0 ? safeCtx.limit : DEFAULT_LIMIT;
+      const files = this.#boundedFiles(safeCtx);
+      if (files.length === 0) {
+        return [];
+      }
+      return this.#resolveWith(ts, symbol, files, mode, limit, safeCtx.snapshot ?? null);
+    } catch {
+      return null;
+    }
+  }
+
+  #boundedFiles(ctx) {
+    if (isSemanticFile(ctx.file)) {
+      return [path.resolve(this.projectRoot, ctx.file)];
+    }
+    try {
+      const filesPath = getArtifactPath(this.projectRoot, INDEX_ARTIFACTS.files);
+      const raw = JSON.parse(fs.readFileSync(filesPath, 'utf8'));
+      const list = Array.isArray(raw) ? raw : (raw.files ?? []);
+      return list
+        .map((entry) => (typeof entry === 'string' ? entry : entry?.filePath ?? entry?.path))
+        .filter(isSemanticFile)
+        .slice(0, MAX_FILES)
+        .map((file) => path.resolve(this.projectRoot, file));
+    } catch {
+      return [];
+    }
+  }
+
+  #resolveWith(ts, symbol, files, mode, limit, snapshot) {
+    const sources = new Map();
+    const readSource = (fileName) => {
+      if (!sources.has(fileName)) {
+        try {
+          sources.set(fileName, fs.readFileSync(fileName, 'utf8'));
+        } catch {
+          sources.set(fileName, null);
+        }
+      }
+      return sources.get(fileName);
+    };
+
+    const host = {
+      getScriptFileNames: () => files,
+      getScriptVersion: () => '1',
+      getScriptSnapshot: (fileName) => {
+        const text = readSource(fileName);
+        if (text === null) {
+          return undefined;
+        }
+        return ts.ScriptSnapshot?.fromString ? ts.ScriptSnapshot.fromString(text) : text;
+      },
+      getCurrentDirectory: () => this.projectRoot,
+      getCompilationSettings: () => ({ allowJs: true, checkJs: false }),
+      getDefaultLibFileName: () => 'lib.d.ts',
+      readFile: (fileName) => readSource(fileName) ?? undefined,
+      fileExists: (fileName) => readSource(fileName) !== null,
+    };
+
+    let service = null;
+    const edges = [];
+    const seen = new Set();
+    const pushEdge = (kind, fileName, start) => {
+      if (edges.length >= limit) {
+        return;
+      }
+      const source = readSource(fileName);
+      const line = source === null ? 0 : lineOf(source, start);
+      const key = `${kind}:${fileName}:${start}`;
+      if (seen.has(key)) {
+        return;
+      }
+      seen.add(key);
+      edges.push(createEdge({
+        from: symbol,
+        to: `${fileName}:${line}`,
+        kind,
+        confidence: 1,
+        provider: this.name,
+        evidence: 'typescript-ls',
+        snapshot,
+      }));
+    };
+
+    try {
+      service = ts.createLanguageService(host);
+      const wantsDefs = mode === 'definition' || mode === 'both';
+      const wantsRefs = mode === 'references' || mode === 'both';
+      for (const fileName of files) {
+        if (edges.length >= limit) {
+          break;
+        }
+        const source = readSource(fileName);
+        if (source === null) {
+          continue;
+        }
+        for (const pos of occurrencesOf(source, symbol)) {
+          if (edges.length >= limit) {
+            break;
+          }
+          if (wantsDefs && typeof service.getDefinitionAtPosition === 'function') {
+            for (const entry of service.getDefinitionAtPosition(fileName, pos) ?? []) {
+              if (entry?.fileName && entry?.textSpan) {
+                pushEdge('def', entry.fileName, entry.textSpan.start);
+              }
+            }
+          }
+          if (wantsRefs && typeof service.findReferences === 'function') {
+            for (const group of service.findReferences(fileName, pos) ?? []) {
+              for (const entry of group?.references ?? []) {
+                if (entry?.fileName && entry?.textSpan) {
+                  pushEdge('ref', entry.fileName, entry.textSpan.start);
+                }
+              }
+            }
+          }
+        }
+      }
+    } finally {
+      try {
+        service?.dispose?.();
+      } catch {
+        // dispose is best-effort
+      }
+    }
+    return edges;
+  }
+}
+
+function occurrencesOf(source, symbol) {
+  const positions = [];
+  const pattern = new RegExp(`\\b${symbol.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g');
+  let match;
+  while ((match = pattern.exec(source)) !== null && positions.length < 50) {
+    positions.push(match.index);
+  }
+  return positions;
+}
+
+function lineOf(source, offset) {
+  let line = 1;
+  for (let i = 0; i < offset && i < source.length; i += 1) {
+    if (source[i] === '\n') {
+      line += 1;
+    }
+  }
+  return line;
+}
+
+/**
+ * Factory honoring `config.codeIntel.providers.semantic`:
+ *   'typescript' (or 'auto' when typescript resolves) → TypeScriptSemanticProvider
+ *   anything else / missing ('null' default)         → NullSemanticProvider
+ */
+export function createSemanticProvider({ projectRoot = process.cwd(), config = {}, loader } = {}) {
+  const setting = config?.codeIntel?.providers?.semantic ?? 'null';
+  if (setting === 'typescript' || setting === 'auto') {
+    return new TypeScriptSemanticProvider({
+      projectRoot,
+      loader: loader ?? defaultLoader,
+    });
+  }
+  return new NullSemanticProvider();
+}

package/src/core/codeintel/summaries.js

@@ -0,0 +1,194 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { getSyntaxProvider } from './providers.js';
+import { resolveProjectRelativePath } from '../fileOps.js';
+
+// Deterministic extractive summaries (SPEC §7/§8, CI-304) — the L3 detail tier.
+// No model calls, no clock/random, no new deps. File reads are bounded to the
+// first MAX_READ_LINES lines; extraction is pure text scanning over docblocks,
+// declaration regexes, and the index-file symbol provider.
+
+const MAX_READ_LINES = 200;
+const DEFAULT_MAX_CHARS = 240;
+const DEFAULT_SYMBOL_MAX_CHARS = 160;
+const DEFAULT_MAX_SYMBOLS = 8;
+
+const DECL_RE = /^\s*(?:export\s+default\s+|export\s+)?(?:async\s+)?(?:function\*?|class|const|let|var)\s+([A-Za-z_$][\w$]*)/;
+
+async function readHead(rootDir, relPath) {
+  try {
+    const abs = resolveProjectRelativePath(rootDir, relPath);
+    if (!abs) return null;
+    const content = await fs.readFile(abs, 'utf8');
+    return { content, lines: content.split('\n').slice(0, MAX_READ_LINES), bytes: Buffer.byteLength(content, 'utf8') };
+  } catch {
+    return null;
+  }
+}
+
+// Extract the first sentence of a docblock: with `endIndex` omitted, reads the
+// leading file-level docblock at the top of `lines`; with `endIndex`, finds the
+// docblock ending right before a declaration line (lines above decl passed in).
+function docblockFirstSentence(lines, endIndex = lines.length) {
+  if (endIndex === lines.length) {
+    // File-level lane: the docblock must start at (or near) the top of file.
+    let start = 0;
+    while (start < lines.length && (lines[start].trim() === '' || lines[start].startsWith('#!'))) start += 1;
+    const first = (lines[start] ?? '').trim();
+    const isBlock = first.startsWith('/*');
+    const isLine = first.startsWith('//');
+    if (!isBlock && !isLine) return null;
+    const block = [];
+    if (isLine) {
+      for (let i = start; i < lines.length; i += 1) {
+        const t = lines[i].trim();
+        if (!t.startsWith('//')) break;
+        block.push(t.replace(/^\/\/+\s?/, ''));
+      }
+    } else {
+      for (let i = start; i < lines.length; i += 1) {
+        const t = lines[i].trim();
+        block.push(t.replace(/^\/\*\*?\s?/, '').replace(/\*\/\s?$/, '').replace(/^\*\s?/, ''));
+        if (t.endsWith('*/')) break;
+      }
+    }
+    const text = block.join(' ').replace(/\s+/g, ' ').trim();
+    if (!text) return null;
+    const m = text.match(/^.+?[.!?](?:\s|$)/);
+    return (m ? m[0] : text).trim();
+  }
+
+  // Find the last docblock ending at or before endIndex, scanning backwards
+  // for a '*/' terminator so decl-adjacent blocks are preferred.
+  let end = -1;
+  for (let i = Math.min(endIndex, lines.length) - 1; i >= 0; i -= 1) {
+    const t = lines[i].trim();
+    if (t === '' || t.startsWith('//') || t.startsWith('/*') || t.startsWith('*') || t.endsWith('*/')) {
+      if (t.endsWith('*/')) { end = i; break; }
+      if (t.startsWith('//')) { end = i; break; }
+      continue;
+    }
+    break;
+  }
+  if (end < 0) return null;
+
+  const block = [];
+  const tail = lines[end].trim();
+  if (tail.startsWith('//')) {
+    for (let i = end; i >= 0; i -= 1) {
+      const t = lines[i].trim();
+      if (!t.startsWith('//')) break;
+      block.unshift(t.replace(/^\/\/+\s?/, ''));
+    }
+  } else {
+    for (let i = end; i >= 0; i -= 1) {
+      const t = lines[i].trim();
+      const isBoundary = t.startsWith('/**') || t.startsWith('/*');
+      block.unshift(t.replace(/^\/\*\*?\s?/, '').replace(/\*\/\s?$/, '').replace(/^\*\s?/, ''));
+      if (isBoundary) break;
+    }
+  }
+
+  const text = block.join(' ').replace(/\s+/g, ' ').trim();
+  if (!text) return null;
+  const m = text.match(/^.+?[.!?](?:\s|$)/);
+  return (m ? m[0] : text).trim();
+}
+
+// Truncate on a word boundary — never mid-word.
+function truncateWords(text, maxChars) {
+  if (text.length <= maxChars) return text;
+  const slice = text.slice(0, maxChars);
+  const lastSpace = slice.lastIndexOf(' ');
+  const cut = lastSpace > 0 ? slice.slice(0, lastSpace) : slice;
+  return cut.trimEnd();
+}
+
+// Regex fallback when the index symbols artifact is absent — bounded, dep-free.
+function scanDeclarations(lines, maxSymbols) {
+  const names = [];
+  for (const line of lines) {
+    const m = line.match(DECL_RE);
+    if (m && !names.includes(m[1])) names.push(m[1]);
+    if (names.length >= maxSymbols) break;
+  }
+  return names;
+}
+
+async function symbolNames(rootDir, relPath, lines, maxSymbols) {
+  try {
+    const provider = getSyntaxProvider(relPath);
+    if (provider) {
+      const syms = await provider.symbols(relPath);
+      if (Array.isArray(syms) && syms.length > 0) {
+        return syms.slice(0, maxSymbols).map((s) => s.name).filter(Boolean);
+      }
+    }
+  } catch {
+    // provider failure falls through to the regex lane — never throws
+  }
+  return scanDeclarations(lines, maxSymbols);
+}
+
+/**
+ * summarizeFile(rootDir, relPath, { maxSymbols=8, maxChars=240 })
+ *   → { path, summary, symbolCount, bytes } | null
+ *
+ * Extractive summary: leading docblock first sentence + top symbol names.
+ * Falls back to a path/size description for docblock-free files. Returns null
+ * only when the file cannot be read. Deterministic across calls.
+ */
+export async function summarizeFile(rootDir, relPath, opts = {}) {
+  if (!relPath || typeof relPath !== 'string') return null;
+  const maxChars = typeof opts.maxChars === 'number' ? opts.maxChars : DEFAULT_MAX_CHARS;
+  const maxSymbols = typeof opts.maxSymbols === 'number' ? opts.maxSymbols : DEFAULT_MAX_SYMBOLS;
+
+  const head = await readHead(rootDir, relPath);
+  if (!head) return null;
+
+  const parts = [];
+  const doc = docblockFirstSentence(head.lines);
+  if (doc) parts.push(doc);
+
+  const names = await symbolNames(rootDir, relPath, head.lines, maxSymbols);
+  if (names.length > 0) parts.push(`exports: ${names.join(', ')}`);
+
+  const summary = parts.length > 0
+    ? parts.join(' — ')
+    : `${relPath} (${head.bytes} bytes)`;
+
+  return {
+    path: relPath,
+    summary: truncateWords(summary, maxChars),
+    symbolCount: names.length,
+    bytes: head.bytes,
+  };
+}
+
+/**
+ * summarizeSymbol(rootDir, relPath, symbolName, { maxChars=160 })
+ *   → { path, symbol, summary } | null
+ *
+ * Docblock-above-decl + the declaration signature line. Null when the file is
+ * unreadable or the symbol is absent. Deterministic.
+ */
+export async function summarizeSymbol(rootDir, relPath, symbolName, opts = {}) {
+  if (!relPath || !symbolName) return null;
+  const maxChars = typeof opts.maxChars === 'number' ? opts.maxChars : DEFAULT_SYMBOL_MAX_CHARS;
+
+  const head = await readHead(rootDir, relPath);
+  if (!head) return null;
+
+  const declIndex = head.lines.findIndex((line) => {
+    const m = line.match(DECL_RE);
+    return m && m[1] === symbolName;
+  });
+  if (declIndex < 0) return null;
+
+  const signature = head.lines[declIndex].trim();
+  const doc = docblockFirstSentence(head.lines, declIndex);
+  const summary = doc ? `${doc} ${signature}` : signature;
+
+  return { path: relPath, symbol: symbolName, summary: truncateWords(summary, maxChars) };
+}

package/src/core/codeintel/vectorProvider.js

@@ -0,0 +1,213 @@
+import fs from 'node:fs/promises';
+
+import { getArtifactPath, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION } from '../../index/paths.js';
+import { createEdge } from './providers.js';
+
+// Dep-free embedding lane (SPEC §2): feature hashing (FNV-1a 32-bit) over word
+// tokens + word-boundary trigrams, L2-normalized, cosine-ranked against the
+// same doc text the BM25 lane uses (file paths + symbol names). No file reads,
+// no deps — real embeddings plug in later through the injectable `loader`
+// seam (config.codeIntel.embedding.provider === 'auto'), degrading to the
+// hashed provider on ANY failure. Never throws.
+
+const DEFAULT_DIMENSIONS = 256;
+const DEFAULT_LIMIT = 20;
+
+function fnv1a(str) {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i += 1) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193) >>> 0;
+  }
+  return h >>> 0;
+}
+
+function wordTokens(text) {
+  return String(text ?? '')
+    .split(/[^A-Za-z0-9_$]+/)
+    .map((t) => t.toLowerCase())
+    .filter(Boolean);
+}
+
+function trigrams(token) {
+  const padded = `#${token}#`;
+  const grams = [];
+  for (let i = 0; i + 3 <= padded.length; i += 1) {
+    grams.push(padded.slice(i, i + 3));
+  }
+  return grams;
+}
+
+/**
+ * hashVectorize(text, { dimensions }) → number[] of length `dimensions`,
+ * L2-normalized (zero vector for empty/null input). Deterministic.
+ */
+export function hashVectorize(text, { dimensions = DEFAULT_DIMENSIONS } = {}) {
+  const dims = Number.isInteger(dimensions) && dimensions > 0 ? dimensions : DEFAULT_DIMENSIONS;
+  const vec = new Array(dims).fill(0);
+  const tokens = wordTokens(text);
+  for (const token of tokens) {
+    vec[fnv1a(`w:${token}`) % dims] += 1;
+    for (const gram of trigrams(token)) {
+      vec[fnv1a(`t:${gram}`) % dims] += 0.1;
+    }
+  }
+  const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
+  if (norm > 0) {
+    for (let i = 0; i < dims; i += 1) vec[i] /= norm;
+  }
+  return vec;
+}
+
+/** cosineSimilarity(a, b) → number in [-1, 1]; 0 on dim mismatch/empty. */
+export function cosineSimilarity(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length) {
+    return 0;
+  }
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+async function readArtifact(rootDir, name) {
+  try {
+    const raw = await fs.readFile(getArtifactPath(rootDir, name), 'utf8');
+    const parsed = JSON.parse(raw);
+    if (parsed?.schemaVersion !== undefined && parsed.schemaVersion !== INDEX_SCHEMA_VERSION) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+export class HashedEmbeddingProvider {
+  constructor({ projectRoot = process.cwd(), dimensions = DEFAULT_DIMENSIONS, embedFn = null } = {}) {
+    this.name = 'hashed-vector';
+    this.projectRoot = projectRoot;
+    this.dimensions = Number.isInteger(dimensions) && dimensions > 0 ? dimensions : DEFAULT_DIMENSIONS;
+    this._embedFn = typeof embedFn === 'function' ? embedFn : null;
+  }
+
+  async available() {
+    return true;
+  }
+
+  capabilities() {
+    return { embeddings: true, similarity: true };
+  }
+
+  async embed(text) {
+    if (this._embedFn) {
+      const vec = await this._embedFn(String(text ?? ''));
+      if (Array.isArray(vec)) return vec;
+    }
+    return hashVectorize(text, { dimensions: this.dimensions });
+  }
+
+  /**
+   * resolve(query, ctx { limit?, snapshot? }) → edge[] | null.
+   * Corpus: files.json paths + symbols.json names — index-bound, no file reads.
+   * Missing index → null (caller records 'unavailable'); empty index → [].
+   */
+  async resolve(query, ctx = {}) {
+    try {
+      const [filesArtifact, symbolsArtifact] = await Promise.all([
+        readArtifact(this.projectRoot, INDEX_ARTIFACTS.files),
+        readArtifact(this.projectRoot, INDEX_ARTIFACTS.symbols),
+      ]);
+      if (!filesArtifact && !symbolsArtifact) {
+        return null;
+      }
+      const limit = Number.isInteger(ctx?.limit) && ctx.limit > 0 ? ctx.limit : DEFAULT_LIMIT;
+      const queryVec = await this.embed(query);
+      const docs = new Map();
+      for (const item of filesArtifact?.items ?? []) {
+        if (!item?.filePath) continue;
+        docs.set(item.filePath, item.filePath);
+      }
+      for (const sym of symbolsArtifact?.items ?? []) {
+        if (!sym?.filePath || !sym?.name) continue;
+        docs.set(sym.filePath, `${docs.get(sym.filePath) ?? sym.filePath} ${sym.name}`);
+      }
+      const snapshot = ctx?.snapshot ?? null;
+      const scored = await Promise.all([...docs.entries()].map(async ([filePath, docText]) => ({
+        filePath,
+        score: cosineSimilarity(queryVec, await this.embed(docText)),
+      })));
+      return scored
+        .filter((entry) => entry.score > 0)
+        .sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath))
+        .slice(0, limit)
+        .map((entry) => createEdge({
+          from: String(query ?? ''),
+          to: `${entry.filePath}:0`,
+          kind: 'vector',
+          confidence: entry.score,
+          provider: this.name,
+          evidence: 'hash-vectorizer',
+          snapshot,
+        }));
+    } catch {
+      return null;
+    }
+  }
+}
+
+export class NullEmbeddingProvider {
+  constructor() {
+    this.name = 'null';
+  }
+
+  async available() {
+    return false;
+  }
+
+  capabilities() {
+    return { embeddings: false, similarity: false };
+  }
+
+  async embed() {
+    return [];
+  }
+
+  async resolve() {
+    return null;
+  }
+}
+
+/**
+ * createEmbeddingProvider({ projectRoot, config, loader }) → provider.
+ * config.codeIntel.embedding.provider:
+ *   'hashed' (default) → HashedEmbeddingProvider
+ *   'auto'             → loader() module exposing embed(); ANY failure → Hashed
+ *   'null'             → NullEmbeddingProvider
+ */
+export function createEmbeddingProvider({ projectRoot = process.cwd(), config, loader } = {}) {
+  const embedding = config?.codeIntel?.embedding;
+  const setting = embedding?.provider ?? 'hashed';
+  const dimensions = embedding?.dimensions;
+  if (setting === 'null') {
+    return new NullEmbeddingProvider();
+  }
+  if (setting === 'auto' && typeof loader === 'function') {
+    try {
+      const mod = loader();
+      if (mod && typeof mod.embed === 'function') {
+        return new HashedEmbeddingProvider({ projectRoot, dimensions, embedFn: mod.embed });
+      }
+    } catch {
+      // fall through — degrade to hashed
+    }
+    return new HashedEmbeddingProvider({ projectRoot, dimensions });
+  }
+  return new HashedEmbeddingProvider({ projectRoot, dimensions });
+}