@ngockhoale/ukit 2.6.7 → 2.6.9

package/src/core/codeintel/retriever.js

@@ -0,0 +1,418 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { getArtifactPath, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION, normalizeRelative } from '../../index/paths.js';
+import { loadRuntimeConfig } from '../runtimeConfig.js';
+import { createEdge, getSemanticProvider, IndexFileSyntaxProvider } from './providers.js';
+import { createSemanticProvider } from './semanticProvider.js';
+import { createEmbeddingProvider } from './vectorProvider.js';
+
+// Hybrid Retriever (SPEC §3): each lane produces a ranked list of file paths,
+// then lanes merge via Reciprocal Rank Fusion (k=60, per-lane weights from
+// codeIntel.retriever.weights). merge:'concat' preserves the exact C26
+// sequential ordering as an escape hatch. A missing/failing lane is skipped
+// and the reason lands in `omitted` — never fatal, never fabricated anchors.
+
+const DEFAULT_LIMIT = 20;
+const DEFAULT_BM25 = { k1: 1.2, b: 0.75 };
+const DEFAULT_RRF_K = 60;
+const DEFAULT_WEIGHTS = { exact: 1.0, symbol: 1.2, bm25: 0.8, vector: 0.6, semantic: 1.0 };
+const LANE_ORDER = ['exact', 'symbol', 'bm25', 'vector', 'semantic'];
+
+async function readArtifact(rootDir, name) {
+  try {
+    const raw = await fs.readFile(getArtifactPath(rootDir, name), 'utf8');
+    const parsed = JSON.parse(raw);
+    if (parsed?.schemaVersion !== undefined && parsed.schemaVersion !== INDEX_SCHEMA_VERSION) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function tokenize(text) {
+  return String(text ?? '')
+    .split(/[^A-Za-z0-9_$]+/)
+    .map((t) => t.toLowerCase())
+    .filter(Boolean);
+}
+
+function retrieverDefaults(config) {
+  const retriever = config?.codeIntel?.retriever;
+  const configWeights = retriever?.weights;
+  const weight = (lane) => (
+    typeof configWeights?.[lane] === 'number' ? configWeights[lane] : DEFAULT_WEIGHTS[lane]
+  );
+  return {
+    limit: typeof retriever?.limit === 'number' ? retriever.limit : DEFAULT_LIMIT,
+    merge: retriever?.merge === 'concat' ? 'concat' : 'rrf',
+    rrfK: typeof retriever?.rrfK === 'number' && retriever.rrfK > 0 ? retriever.rrfK : DEFAULT_RRF_K,
+    bm25: {
+      k1: typeof retriever?.bm25?.k1 === 'number' ? retriever.bm25.k1 : DEFAULT_BM25.k1,
+      b: typeof retriever?.bm25?.b === 'number' ? retriever.bm25.b : DEFAULT_BM25.b,
+    },
+    weights: {
+      exact: weight('exact'),
+      symbol: weight('symbol'),
+      bm25: weight('bm25'),
+      vector: weight('vector'),
+      semantic: weight('semantic'),
+    },
+  };
+}
+
+function bm25Score(queryTokens, docTokens, avgDocLen, { k1, b }) {
+  if (docTokens.length === 0) return 0;
+  const freq = new Map();
+  for (const t of docTokens) freq.set(t, (freq.get(t) ?? 0) + 1);
+  let score = 0;
+  const lenNorm = 1 - b + (b * docTokens.length) / (avgDocLen || 1);
+  for (const qt of queryTokens) {
+    const f = freq.get(qt) ?? 0;
+    if (f === 0) continue;
+    score += ((f * (k1 + 1)) / (f + k1 * lenNorm));
+  }
+  return score;
+}
+
+function resolveRelativeSpecifier(fromFile, specifier, fileSet) {
+  const candidates = [];
+  if (specifier.startsWith('.')) {
+    const base = path.posix.normalize(path.posix.join(path.posix.dirname(fromFile), specifier));
+    candidates.push(base, `${base}.js`, `${base}.ts`, `${base}.mjs`, `${base}.jsx`, `${base}.tsx`, `${base}/index.js`, `${base}/index.ts`);
+  } else {
+    candidates.push(specifier);
+  }
+  for (const candidate of candidates) {
+    if (fileSet.has(candidate)) return candidate;
+  }
+  return null;
+}
+
+function makeEvidence({ path: filePath, why, excerpt }) {
+  const item = { level: 'L1', source: 'index', why };
+  if (filePath) item.path = filePath;
+  if (excerpt) item.excerpt = excerpt;
+  return item;
+}
+
+// Lane helpers produce ranked, deduped lists of { path, why, extra }.
+const laneEntry = (filePath, why, extra = {}) => ({ path: filePath, why, extra });
+
+function exactLane(fileItems, normalizedQuery, queryBase) {
+  const lane = [];
+  const seen = new Set();
+  for (const item of fileItems) {
+    if (!item?.filePath || seen.has(item.filePath)) continue;
+    const base = path.posix.basename(item.filePath).toLowerCase();
+    if (item.filePath === normalizedQuery || base === queryBase || base === normalizedQuery.toLowerCase()) {
+      seen.add(item.filePath);
+      lane.push(laneEntry(item.filePath, `exact-path match for "${normalizedQuery}"`));
+    }
+  }
+  return lane;
+}
+
+function symbolLane(symbolItems, normalizedQuery, fileSet) {
+  const lane = [];
+  const seen = new Set();
+  for (const sym of symbolItems) {
+    if (!sym?.name || !sym?.filePath || seen.has(sym.filePath)) continue;
+    if (sym.name === normalizedQuery || sym.name.toLowerCase() === normalizedQuery.toLowerCase()) {
+      if (fileSet.size === 0 || fileSet.has(sym.filePath)) {
+        seen.add(sym.filePath);
+        lane.push(laneEntry(
+          sym.filePath,
+          `symbol "${sym.name}" defined here`,
+          { symbol: sym.name, line: sym.line },
+        ));
+      }
+    }
+  }
+  return lane;
+}
+
+function bm25Lane(fileItems, symbolItems, queryTokens, bm25) {
+  if (queryTokens.length === 0) return null;
+  const docs = new Map();
+  for (const item of fileItems) {
+    if (!item?.filePath) continue;
+    docs.set(item.filePath, tokenize(item.filePath));
+  }
+  for (const sym of symbolItems) {
+    if (!sym?.filePath) continue;
+    const doc = docs.get(sym.filePath) ?? [];
+    doc.push(...tokenize(sym.name));
+    docs.set(sym.filePath, doc);
+  }
+  const avgLen = [...docs.values()].reduce((sum, d) => sum + d.length, 0) / (docs.size || 1);
+  return [...docs.entries()]
+    .map(([filePath, docTokens]) => ({
+      filePath,
+      score: bm25Score(queryTokens, docTokens, avgLen, bm25),
+    }))
+    .filter((entry) => entry.score > 0)
+    .sort((a, b) => b.score - a.score || a.filePath.localeCompare(b.filePath))
+    .map((entry) => laneEntry(entry.filePath, `bm25 match (score ${entry.score.toFixed(2)})`));
+}
+
+// Semantic lane (SPEC §2/§3): registered provider wins, else
+// createSemanticProvider honors config.codeIntel.providers.semantic. Edges'
+// `to` carries "file:line" — reduce to a ranked file list.
+async function semanticLane(rootDir, normalizedQuery, config, fileSet, snapshotId) {
+  const provider = getSemanticProvider()?.name === 'null'
+    ? createSemanticProvider({ projectRoot: rootDir, config })
+    : getSemanticProvider();
+  if (!provider || provider.name === 'null' || typeof provider.resolve !== 'function') {
+    return { lane: null, why: 'unavailable' };
+  }
+  let edges;
+  try {
+    edges = await provider.resolve(normalizedQuery, { mode: 'both', snapshot: snapshotId });
+  } catch {
+    return { lane: null, why: 'provider-error' };
+  }
+  if (edges === null || edges === undefined) {
+    return { lane: null, why: 'unavailable' };
+  }
+  if (!Array.isArray(edges) || edges.length === 0) {
+    return { lane: [], why: 'no-match' };
+  }
+  const lane = [];
+  const seen = new Set();
+  for (const edge of edges) {
+    const target = typeof edge?.to === 'string' ? edge.to.split(':')[0] : null;
+    if (!target) continue;
+    const rel = path.isAbsolute(target) ? normalizeRelative(rootDir, target) : target;
+    if (seen.has(rel)) continue;
+    if (fileSet.size > 0 && !fileSet.has(rel)) continue;
+    seen.add(rel);
+    lane.push(laneEntry(rel, `semantic ${edge.kind ?? 'match'} for "${normalizedQuery}"`));
+  }
+  return { lane, why: lane.length === 0 ? 'no-match' : null };
+}
+
+// Vector lane (SPEC §2/§3): dep-free hashed embedding provider; mirrors the
+// semantic lane. Edges' `to` carries "file:0" — reduce to a ranked file list.
+async function vectorLane(rootDir, normalizedQuery, config, fileSet, snapshotId) {
+  const provider = createEmbeddingProvider({ projectRoot: rootDir, config });
+  if (!provider || provider.name === 'null' || typeof provider.resolve !== 'function') {
+    return { lane: null, why: 'unavailable' };
+  }
+  let edges;
+  try {
+    edges = await provider.resolve(normalizedQuery, { limit: DEFAULT_LIMIT, snapshot: snapshotId });
+  } catch {
+    return { lane: null, why: 'provider-error' };
+  }
+  if (edges === null || edges === undefined) {
+    return { lane: null, why: 'unavailable' };
+  }
+  if (!Array.isArray(edges) || edges.length === 0) {
+    return { lane: [], why: 'no-match' };
+  }
+  const lane = [];
+  const seen = new Set();
+  for (const edge of edges) {
+    const target = typeof edge?.to === 'string' ? edge.to.split(':')[0] : null;
+    if (!target) continue;
+    const rel = path.isAbsolute(target) ? normalizeRelative(rootDir, target) : target;
+    if (seen.has(rel)) continue;
+    if (fileSet.size > 0 && !fileSet.has(rel)) continue;
+    seen.add(rel);
+    lane.push(laneEntry(rel, `vector ${edge.kind ?? 'match'} for "${normalizedQuery}"`));
+  }
+  return { lane, why: lane.length === 0 ? 'no-match' : null };
+}
+
+function rrfMerge(lanes, weights, rrfK) {
+  const scores = new Map();
+  const meta = new Map();
+  for (const laneName of LANE_ORDER) {
+    const lane = lanes.get(laneName);
+    if (!lane) continue;
+    const w = weights[laneName] ?? 0;
+    if (w <= 0) continue;
+    lane.forEach((entry, index) => {
+      scores.set(entry.path, (scores.get(entry.path) ?? 0) + w / (rrfK + index + 1));
+      if (!meta.has(entry.path)) meta.set(entry.path, entry);
+    });
+  }
+  return [...scores.entries()]
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([filePath]) => meta.get(filePath));
+}
+
+function concatMerge(lanes) {
+  const merged = [];
+  const seen = new Set();
+  for (const laneName of LANE_ORDER) {
+    const lane = lanes.get(laneName);
+    if (!lane) continue;
+    for (const entry of lane) {
+      if (seen.has(entry.path)) continue;
+      seen.add(entry.path);
+      merged.push(entry);
+    }
+  }
+  return merged;
+}
+
+/**
+ * retrieve(projectRoot, query, { mode, limit, snapshot, merge, weights }) →
+ *   { anchors, evidence, relations, omitted }
+ *
+ * modes: 'search' (default) | 'impact' (adds 1-hop reverse-import edges).
+ * merge: 'rrf' (default) | 'concat' (exact C26 sequential ordering).
+ * weights: per-lane RRF weight overrides; defaults come from
+ *   codeIntel.retriever.weights (in-code defaults when config is absent).
+ * Never throws on a missing/partial index — returns empty lanes with reasons
+ * in `omitted`.
+ */
+export async function retrieve(projectRoot, query, { mode = 'search', limit, snapshot, merge, weights } = {}) {
+  const rootDir = path.resolve(projectRoot ?? process.cwd());
+  const omitted = [];
+
+  let config = null;
+  try {
+    config = await loadRuntimeConfig(rootDir);
+  } catch {
+    config = null;
+  }
+  const defaults = retrieverDefaults(config);
+  const effectiveLimit = typeof limit === 'number' && limit > 0 ? limit : defaults.limit;
+  const effectiveMerge = merge === 'concat' || merge === 'rrf' ? merge : defaults.merge;
+  const effectiveWeights = { ...defaults.weights };
+  if (weights && typeof weights === 'object') {
+    for (const laneName of LANE_ORDER) {
+      if (typeof weights[laneName] === 'number') effectiveWeights[laneName] = weights[laneName];
+    }
+  }
+
+  const [filesArtifact, symbolsArtifact, importsArtifact, metaArtifact] = await Promise.all([
+    readArtifact(rootDir, INDEX_ARTIFACTS.files),
+    readArtifact(rootDir, INDEX_ARTIFACTS.symbols),
+    readArtifact(rootDir, INDEX_ARTIFACTS.imports),
+    readArtifact(rootDir, INDEX_ARTIFACTS.meta),
+  ]);
+
+  const snapshotId = snapshot ?? metaArtifact?.headSha ?? metaArtifact?.snapshot ?? null;
+
+  if (!filesArtifact && !symbolsArtifact) {
+    omitted.push({ what: 'retrieval', why: 'index-missing' });
+    return { anchors: [], evidence: [], relations: [], omitted };
+  }
+
+  const fileItems = filesArtifact?.items ?? [];
+  const symbolItems = symbolsArtifact?.items ?? [];
+  const fileSet = new Set(fileItems.map((f) => f.filePath));
+  const normalizedQuery = String(query ?? '').trim();
+  const queryTokens = tokenize(normalizedQuery);
+  const queryBase = path.posix.basename(normalizedQuery).toLowerCase();
+
+  const lanes = new Map();
+
+  // Lane 1 — exact path / basename match.
+  const exact = exactLane(fileItems, normalizedQuery, queryBase);
+  if (exact.length > 0) lanes.set('exact', exact);
+
+  // Lane 2 — symbol-name match via the syntax provider (index-file fallback).
+  const provider = new IndexFileSyntaxProvider({ rootDir });
+  if (!symbolsArtifact) {
+    omitted.push({ what: 'symbol-lane', why: 'index-missing' });
+  }
+  const symbol = symbolLane(symbolItems, normalizedQuery, fileSet);
+  if (symbol.length > 0) lanes.set('symbol', symbol);
+  // Enrich anchors with provider symbols when the lane matched (best-effort).
+  for (const entry of symbol) {
+    try {
+      if (provider.supports(entry.path)) {
+        await provider.symbols(entry.path);
+      }
+    } catch {
+      // Provider enrichment is best-effort; anchors already stand.
+    }
+  }
+
+  // Lane 3 — BM25-lite over file/symbol tokens.
+  const bm25 = bm25Lane(fileItems, symbolItems, queryTokens, defaults.bm25);
+  if (bm25 === null) {
+    if (exact.length === 0 && symbol.length === 0) {
+      omitted.push({ what: 'bm25-lane', why: 'empty-query-tokens' });
+    }
+  } else if (bm25.length > 0) {
+    // Lane pool is wider than the output limit so deep-ranked bm25 hits still
+    // get an RRF rank instead of silently dropping out of the merge.
+    lanes.set('bm25', bm25.slice(0, Math.max(effectiveLimit, 100)));
+  }
+
+  // Lane 4 — dep-free hashed vector provider (index-bound; absent → omitted).
+  const vector = await vectorLane(rootDir, normalizedQuery, config, fileSet, snapshotId);
+  if (vector.lane === null) {
+    omitted.push({ what: 'vector-lane', why: vector.why });
+  } else if (vector.lane.length > 0) {
+    lanes.set('vector', vector.lane);
+  } else {
+    omitted.push({ what: 'vector-lane', why: vector.why });
+  }
+
+  // Lane 5 — semantic provider (defensive; absent → skipped, never fatal).
+  const semantic = await semanticLane(rootDir, normalizedQuery, config, fileSet, snapshotId);
+  if (semantic.lane === null) {
+    omitted.push({ what: 'semantic-lane', why: semantic.why });
+  } else if (semantic.lane.length > 0) {
+    lanes.set('semantic', semantic.lane);
+  } else {
+    omitted.push({ what: 'semantic-lane', why: semantic.why });
+  }
+
+  const merged = effectiveMerge === 'concat'
+    ? concatMerge(lanes)
+    : rrfMerge(lanes, effectiveWeights, defaults.rrfK);
+
+  const anchors = [];
+  const evidence = [];
+  for (const entry of merged) {
+    if (anchors.length >= effectiveLimit) break;
+    anchors.push({ path: entry.path, ...entry.extra });
+    evidence.push(makeEvidence({ path: entry.path, why: entry.why }));
+  }
+
+  if (anchors.length === 0) {
+    omitted.push({ what: 'anchors', why: 'no-match' });
+  }
+
+  const relations = [];
+
+  // Impact mode — 1-hop reverse imports: who imports each anchor file.
+  if (mode === 'impact') {
+    const importItems = importsArtifact?.items ?? [];
+    if (!importsArtifact) {
+      omitted.push({ what: 'impact-lane', why: 'index-missing' });
+    } else {
+      for (const anchor of anchors) {
+        for (const imp of importItems) {
+          if (!imp?.from || !imp?.to) continue;
+          const resolved = resolveRelativeSpecifier(imp.from, imp.to, fileSet);
+          if (resolved !== anchor.path) continue;
+          relations.push(createEdge({
+            from: imp.from,
+            to: anchor.path,
+            kind: imp.kind ?? 'import',
+            confidence: 1,
+            provider: 'index-file',
+            evidence: INDEX_ARTIFACTS.imports,
+            snapshot: snapshotId,
+          }));
+        }
+      }
+      if (relations.length === 0) {
+        omitted.push({ what: 'impact-lane', why: 'no-match' });
+      }
+    }
+  }
+
+  return { anchors, evidence, relations, omitted };
+}

package/src/core/codeintel/router.js

@@ -0,0 +1,149 @@
+// Task Router v1 (SPEC §6) — rule-based gate, no model calls.
+// First-match wins, top→bottom. Output: { mode, reasons[], budget, depth }.
+
+const DEFAULT_BUDGETS = Object.freeze({
+  none: 0,
+  peek: 500,
+  targeted: 2000,
+  impact: 3000,
+  explore: 4000,
+  deep_flow: 6000,
+  analogy: 4000,
+});
+
+const DEFAULT_DEPTHS = Object.freeze({
+  none: 0,
+  peek: 1,
+  targeted: 1,
+  impact: 2,
+  explore: 2,
+  deep_flow: 3,
+  analogy: 2,
+});
+
+export const ROUTER_MODES = Object.freeze([
+  'none',
+  'peek',
+  'targeted',
+  'explore',
+  'impact',
+  'deep_flow',
+  'analogy',
+]);
+
+const TRIVIAL_RE = /\b(typo|label|rename|spacing|toggle|whitespace|comment fix|small tweak)\b/i;
+const ERROR_RE = /\b(error|exception|stack ?trace|bug|crash|failing|fails|failure|regression|TypeError|ReferenceError|SyntaxError|ENOENT|segfault)\b/i;
+const EXPLORE_RE = /\b(where|how does|how do|find|explore|understand|what does|which file|locate|handled)\b/i;
+const DEEP_FLOW_RE = /\b(refactor|migrate|migration|flow|end[- ]?to[- ]?end|trace|across|pipeline|lifecycle)\b/i;
+const ANALOGY_RE = /\b(similar to|analogy|like\b|same as|pattern|equivalent)\b/i;
+const PATH_RE = /(?:[\w.-]+\/)+[\w.-]+|[\w.-]+\.(?:js|ts|mjs|cjs|jsx|tsx|py|rb|go|rs|java|md|json|yml|yaml|css|html|sh)\b/i;
+
+function resolveBudgets(options) {
+  const configured = options?.config?.codeIntel?.budgets ?? options?.budgets;
+  const budgets = { ...DEFAULT_BUDGETS };
+  if (configured && typeof configured === 'object') {
+    for (const mode of ROUTER_MODES) {
+      const value = configured[mode];
+      if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
+        budgets[mode] = value;
+      }
+    }
+  }
+  return budgets;
+}
+
+function decide(prompt, { hasError = false, filesHinted } = {}) {
+  const hintedFiles = Array.isArray(filesHinted) ? filesHinted : [];
+  const reasons = [];
+
+  // Rule 2 — trivial signals: typo/label/rename/toggle, <=1 hinted file, no error.
+  if (!hasError && hintedFiles.length <= 1 && TRIVIAL_RE.test(prompt)) {
+    reasons.push('trivial signal matched (typo/label/rename/toggle) with no error context');
+    return { mode: 'none', reasons };
+  }
+
+  // Rule 3 — explicit path/symbol mention, clear scope, no error.
+  const hasExplicitTarget = hintedFiles.length > 0 || PATH_RE.test(prompt);
+  const errorSignalled = hasError || ERROR_RE.test(prompt);
+  if (!errorSignalled && hasExplicitTarget) {
+    reasons.push('explicit path or symbol mention with clear scope');
+    return { mode: 'targeted', reasons };
+  }
+
+  // Rule 4 — error signature / stack / bug wording.
+  if (errorSignalled) {
+    reasons.push('error signature or failure wording detected');
+    return { mode: 'impact', reasons };
+  }
+
+  // Rule 5 — open-ended exploration wording.
+  if (EXPLORE_RE.test(prompt)) {
+    reasons.push('open-ended where/how/find/explore wording');
+    return { mode: 'explore', reasons };
+  }
+
+  // Rule 6 — cross-cutting refactor/migrate/flow wording.
+  if (DEEP_FLOW_RE.test(prompt)) {
+    reasons.push('cross-cutting refactor/migrate/flow wording');
+    return { mode: 'deep_flow', reasons };
+  }
+
+  // Rule 7 — analogy / pattern search wording.
+  if (ANALOGY_RE.test(prompt)) {
+    reasons.push('similar-to / analogy pattern-search wording');
+    return { mode: 'analogy', reasons };
+  }
+
+  // Rule 8 — fallback: single-file verb (single hinted file or lone path mention).
+  if (hintedFiles.length === 1 || PATH_RE.test(prompt)) {
+    reasons.push('fallback: single-file hint');
+    return { mode: 'peek', reasons };
+  }
+
+  // Rule 9 — fallback otherwise.
+  reasons.push('fallback: no rule matched');
+  return { mode: 'explore', reasons };
+}
+
+/**
+ * routeTask({ prompt, hasError?, filesHinted?, flags? }, options?)
+ * options: { config } (runtime config — uses config.codeIntel.budgets) or { budgets }.
+ * Returns { mode, reasons, budget, depth }.
+ */
+export function routeTask(input = {}, options = {}) {
+  const safeInput = input && typeof input === 'object' ? input : {};
+  const prompt = typeof safeInput.prompt === 'string' ? safeInput.prompt : '';
+  const flags = safeInput.flags && typeof safeInput.flags === 'object' ? safeInput.flags : {};
+  const budgets = resolveBudgets(options);
+
+  // Rule 1 — forced mode wins; unknown forced mode falls back to explore gracefully.
+  if (typeof flags.mode === 'string' && flags.mode.trim() !== '') {
+    const forced = flags.mode.trim();
+    if (ROUTER_MODES.includes(forced)) {
+      return {
+        mode: forced,
+        reasons: [`flags.mode forced to '${forced}'`],
+        budget: budgets[forced],
+        depth: DEFAULT_DEPTHS[forced],
+      };
+    }
+    return {
+      mode: 'explore',
+      reasons: [`flags.mode '${forced}' is not a valid mode; graceful fallback to explore`],
+      budget: budgets.explore,
+      depth: DEFAULT_DEPTHS.explore,
+    };
+  }
+
+  const { mode, reasons } = decide(prompt, {
+    hasError: safeInput.hasError === true,
+    filesHinted: safeInput.filesHinted,
+  });
+
+  return {
+    mode,
+    reasons,
+    budget: budgets[mode],
+    depth: DEFAULT_DEPTHS[mode],
+  };
+}