@ngockhoale/ukit 2.6.7 → 2.6.9

package/src/core/codeintel/freshness.js

@@ -0,0 +1,295 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { spawnSync } from 'node:child_process';
+import { getIndexDir, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION } from '../../index/paths.js';
+import { loadRuntimeConfig } from '../runtimeConfig.js';
+
+const GIT_TIMEOUT_MS = 3000;
+const NON_GIT_MANIFEST_CAP = 2000;
+const LEVELS = ['L0', 'L1', 'L2', 'L3', 'L4'];
+const SNAPSHOT_FILE = 'freshness.json';
+const DIRTY_FILE = 'dirty.json';
+
+function sha1(input) {
+  return crypto.createHash('sha1').update(input).digest('hex');
+}
+
+// Deterministic JSON: object keys sorted recursively so equal values hash equal.
+function canonicalJson(value) {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
+  const keys = Object.keys(value).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
+}
+
+// Git spawns must never throw (SPEC §7): any failure/timeout yields null.
+function runGit(rootDir, args) {
+  try {
+    const result = spawnSync('git', args, {
+      cwd: rootDir,
+      encoding: 'utf8',
+      timeout: GIT_TIMEOUT_MS,
+      maxBuffer: 16 * 1024 * 1024,
+    });
+    if (result.error || result.status !== 0) return null;
+    return result.stdout ?? '';
+  } catch {
+    return null;
+  }
+}
+
+export function getHeadSha(rootDir) {
+  const out = runGit(rootDir, ['rev-parse', 'HEAD']);
+  const sha = (out || '').trim();
+  return /^[0-9a-f]{40}$/i.test(sha) ? sha.toLowerCase() : 'nogit';
+}
+
+// Branch key for per-branch snapshot isolation (SPEC §5):
+// sanitized branch name | 'detached-<head8>' | 'nogit'. Never throws.
+export function getBranchKey(rootDir) {
+  const out = runGit(rootDir, ['branch', '--show-current']);
+  const branch = (out || '').trim();
+  if (branch) return branch.replace(/[^A-Za-z0-9._-]/g, '-');
+  const head = (runGit(rootDir, ['rev-parse', 'HEAD']) || '').trim();
+  if (/^[0-9a-f]{40}$/i.test(head)) return `detached-${head.slice(0, 8).toLowerCase()}`;
+  return 'nogit';
+}
+
+function parsePorcelainPaths(output) {
+  const paths = [];
+  for (const line of output.split('\n')) {
+    if (!line.trim()) continue;
+    let p = line.slice(3);
+    // rename entries: "old -> new" — track the new path
+    const arrow = p.indexOf(' -> ');
+    if (arrow !== -1) p = p.slice(arrow + 4);
+    p = p.trim();
+    if (p.startsWith('"') && p.endsWith('"')) p = p.slice(1, -1);
+    if (p) paths.push(p);
+  }
+  return paths;
+}
+
+async function statEntry(rootDir, rel) {
+  try {
+    const stat = await fs.stat(path.join(rootDir, rel));
+    return `${stat.mtimeMs}:${stat.size}`;
+  } catch {
+    return 'deleted:0';
+  }
+}
+
+// Read dirty.json on-disk contract {paths[], saturated} (SPEC §6).
+// Returns null when the file is missing/corrupt — dirty tracking inactive.
+async function readDirtySet(rootDir) {
+  try {
+    const raw = await fs.readFile(path.join(getIndexDir(rootDir), DIRTY_FILE), 'utf8');
+    const dirty = JSON.parse(raw);
+    if (!dirty || typeof dirty !== 'object') return null;
+    const paths = Array.isArray(dirty.paths)
+      ? dirty.paths.filter((p) => typeof p === 'string' && p.length > 0)
+      : [];
+    return { paths, saturated: dirty.saturated === true };
+  } catch {
+    return null;
+  }
+}
+
+async function overlayEntriesGit(rootDir, statusOutput, extraPaths = []) {
+  const entries = {};
+  const seen = new Set();
+  for (const rel of [...parsePorcelainPaths(statusOutput), ...extraPaths]) {
+    // Skip index-internal artifacts (.cache/) — snapshot/dirty writes must not
+    // self-invalidate the overlay.
+    if (!rel || rel.startsWith('.cache/') || seen.has(rel)) continue;
+    seen.add(rel);
+    entries[rel] = await statEntry(rootDir, rel);
+  }
+  return entries;
+}
+
+// Non-git fallback: hash LIVE mtimes for index-covered files (files.json),
+// capped at NON_GIT_MANIFEST_CAP entries. Stored mtimeMs from files.json is
+// stale by definition — only live fs.stat detects post-index edits.
+// Incremental mode: only when dirty-set tracking is active (dirty.json exists,
+// not saturated) AND a prior snapshot carries overlayEntries — reuse stored
+// entries for paths proven unchanged by the dirty set; live-stat dirty,
+// added, and unknown paths. Stored files.json mtimes are never trusted.
+async function overlayEntriesManifest(rootDir, { incremental, snapshot, dirty } = {}) {
+  let items = [];
+  try {
+    const raw = await fs.readFile(path.join(getIndexDir(rootDir), INDEX_ARTIFACTS.files), 'utf8');
+    const manifest = JSON.parse(raw);
+    items = Array.isArray(manifest?.items) ? manifest.items : [];
+  } catch {
+    return {};
+  }
+  const manifestPaths = [];
+  const seen = new Set();
+  for (const item of items.slice(0, NON_GIT_MANIFEST_CAP)) {
+    const rel = typeof item?.filePath === 'string' ? item.filePath : null;
+    if (!rel || seen.has(rel)) continue;
+    seen.add(rel);
+    manifestPaths.push(rel);
+  }
+
+  const stored = snapshot && typeof snapshot.overlayEntries === 'object' && snapshot.overlayEntries !== null
+    ? snapshot.overlayEntries
+    : null;
+  const dirtyPaths = dirty && !dirty.saturated ? new Set(dirty.paths) : null;
+  const canReuse = incremental === true && stored !== null && dirtyPaths !== null;
+
+  const entries = {};
+  for (const rel of manifestPaths) {
+    // Reuse only when dirty tracking proves the path unchanged and a stored
+    // entry exists; otherwise stat live (never trust files.json mtimeMs).
+    if (canReuse && !dirtyPaths.has(rel) && typeof stored[rel] === 'string') {
+      entries[rel] = stored[rel];
+      continue;
+    }
+    entries[rel] = await statEntry(rootDir, rel);
+  }
+  // Dirty paths outside the manifest still affect identity (stat live);
+  // index-internal artifacts are skipped.
+  if (dirtyPaths) {
+    for (const rel of dirtyPaths) {
+      if (rel.startsWith('.cache/')) continue;
+      if (entries[rel] === undefined) entries[rel] = await statEntry(rootDir, rel);
+    }
+  }
+  return entries;
+}
+
+function hashEntries(entries) {
+  const lines = Object.keys(entries)
+    .sort()
+    .map((rel) => `${rel}:${entries[rel]}`);
+  return sha1(lines.join('\n'));
+}
+
+async function computeOverlayEntries(rootDir, headSha, { incremental = true, snapshot } = {}) {
+  const dirty = await readDirtySet(rootDir);
+  if (headSha !== 'nogit') {
+    const status = runGit(rootDir, ['status', '--porcelain']);
+    if (status !== null) {
+      // Git mode: porcelain output IS the diff — dirty.json unioned defensively.
+      const extra = dirty && !dirty.saturated ? dirty.paths : [];
+      return overlayEntriesGit(rootDir, status, extra);
+    }
+  }
+  return overlayEntriesManifest(rootDir, { incremental, snapshot, dirty });
+}
+
+export async function computeOverlayHash(rootDir, headSha, { incremental = true, snapshot } = {}) {
+  const entries = await computeOverlayEntries(rootDir, headSha, { incremental, snapshot });
+  return hashEntries(entries);
+}
+
+export async function computeConfigHash(rootDir) {
+  const config = await loadRuntimeConfig(rootDir);
+  return sha1(canonicalJson({
+    codeIntel: config?.codeIntel ?? null,
+    memoryV2: config?.memoryV2 ?? null,
+    compact: { enabled: config?.compact?.enabled ?? null },
+    INDEX_SCHEMA_VERSION,
+  }));
+}
+
+export async function computeIdentity(rootDir, { incremental } = {}) {
+  const headSha = getHeadSha(rootDir);
+  const config = await loadRuntimeConfig(rootDir);
+  const useIncremental = incremental ?? (config?.codeIntel?.freshness?.incremental !== false);
+  const configHash = sha1(canonicalJson({
+    codeIntel: config?.codeIntel ?? null,
+    memoryV2: config?.memoryV2 ?? null,
+    compact: { enabled: config?.compact?.enabled ?? null },
+    INDEX_SCHEMA_VERSION,
+  }));
+  const snapshot = await readSnapshot(rootDir);
+  const overlayEntries = await computeOverlayEntries(rootDir, headSha, {
+    incremental: useIncremental,
+    snapshot,
+  });
+  const overlayHash = hashEntries(overlayEntries);
+  const identity = sha1(`${headSha}|${overlayHash}|${configHash}`);
+  return {
+    identity,
+    headSha,
+    overlayHash,
+    configHash,
+    branch: getBranchKey(rootDir),
+    overlayEntries,
+  };
+}
+
+export function getSnapshotPath(rootDir, { branch } = {}) {
+  const key = branch ?? getBranchKey(rootDir);
+  return path.join(getIndexDir(rootDir), `freshness-${key}.json`);
+}
+
+async function readSnapshotFile(snapshotPath) {
+  try {
+    const raw = await fs.readFile(snapshotPath, 'utf8');
+    const snapshot = JSON.parse(raw);
+    if (!snapshot || typeof snapshot !== 'object' || typeof snapshot.identity !== 'string') {
+      return null;
+    }
+    return snapshot;
+  } catch {
+    return null;
+  }
+}
+
+export async function readSnapshot(rootDir) {
+  // Current branch's file first, then legacy freshness.json (C26 read-compat).
+  const branchSnapshot = await readSnapshotFile(getSnapshotPath(rootDir));
+  if (branchSnapshot) return branchSnapshot;
+  return readSnapshotFile(path.join(getIndexDir(rootDir), SNAPSHOT_FILE));
+}
+
+export async function writeSnapshot(rootDir, snapshot) {
+  const snapshotPath = getSnapshotPath(rootDir);
+  const legacyPath = path.join(getIndexDir(rootDir), SNAPSHOT_FILE);
+  await fs.mkdir(path.dirname(snapshotPath), { recursive: true });
+  const payload = JSON.stringify(snapshot, null, 2);
+  await fs.writeFile(snapshotPath, payload);
+  // Legacy alias: older readers still look at freshness.json.
+  if (legacyPath !== snapshotPath) {
+    await fs.writeFile(legacyPath, payload);
+  }
+}
+
+function levelIndex(level) {
+  return LEVELS.indexOf(level);
+}
+
+function clampLevel(level, maxLevel) {
+  const cap = levelIndex(maxLevel);
+  if (cap === -1) return level;
+  return levelIndex(level) > cap ? maxLevel : level;
+}
+
+// Lowest L that repairs the staleness:
+// - no usable snapshot → L2 (batch incremental over unknown state)
+// - configHash changed → L4 (schema/config drift needs full rebuild)
+// - headSha changed → L2 (batch incremental)
+// - overlayHash only → L1 (dirty-file refresh)
+function repairLevel(snapshot, current) {
+  if (!snapshot) return 'L2';
+  if (snapshot.configHash !== current.configHash) return 'L4';
+  if (snapshot.headSha !== current.headSha) return 'L2';
+  return 'L1';
+}
+
+export async function guardLevel(rootDir, { maxLevel = 'L4' } = {}) {
+  const [identity, snapshot] = await Promise.all([
+    computeIdentity(rootDir),
+    readSnapshot(rootDir),
+  ]);
+  const stale = !snapshot || snapshot.identity !== identity.identity;
+  const recommendedLevel = stale
+    ? clampLevel(repairLevel(snapshot, identity), maxLevel)
+    : 'L0';
+  return { identity, snapshot, stale, recommendedLevel };
+}

package/src/core/codeintel/graph.js

@@ -0,0 +1,291 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import { getArtifactPath, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION } from '../../index/paths.js';
+import { readJsonIfExists, writeJson } from '../fileOps.js';
+import { createEdge } from './providers.js';
+import { COCHANGE_SCHEMA_VERSION } from './cochange.js';
+
+// Graph contract + JSON store (SPEC §9 — CI-305). Builds `.cache/index/codegraph.json`
+// from existing index artifacts (imports.json + calls.json, + cochange.json when
+// present) and exposes a GraphProvider seam so SCIP/PDG/LSP backends can slot in
+// later without touching consumers. No SQLite, no new deps — providers other than
+// 'json' are contract-only NullGraphProvider stubs this cycle.
+
+export const GRAPH_SCHEMA_VERSION = 1;
+
+const VALID_GRAPH_STORES = new Set(['json', 'scip', 'pdg', 'lsp', 'null']);
+
+async function readArtifact(rootDir, name, expectedSchemaVersion) {
+  try {
+    const raw = await fs.readFile(getArtifactPath(rootDir, name), 'utf8');
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== 'object') return null;
+    if (expectedSchemaVersion !== undefined && parsed.schemaVersion !== expectedSchemaVersion) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+// Same specifier-resolution discipline as impact.js — local copy so this lane
+// does not import same-wave sibling internals.
+function resolveRelativeSpecifier(fromFile, specifier, fileSet) {
+  const candidates = [];
+  if (specifier.startsWith('.')) {
+    const base = path.posix.normalize(path.posix.join(path.posix.dirname(fromFile), specifier));
+    candidates.push(base, `${base}.js`, `${base}.ts`, `${base}.mjs`, `${base}.jsx`, `${base}.tsx`, `${base}/index.js`, `${base}/index.ts`);
+  } else {
+    candidates.push(specifier);
+  }
+  for (const candidate of candidates) {
+    if (fileSet.has(candidate)) return candidate;
+  }
+  return null;
+}
+
+/**
+ * buildGraphArtifact(projectRoot) → artifact | null
+ *
+ * Merges imports.json + calls.json (+ cochange.json when present) into a
+ * codegraph.json artifact: { schemaVersion, generatedAt, nodes, edges }.
+ * Node ids: files → their relative path, symbols → `symbol:<name>`.
+ * Edges use the canonical createEdge shape. Writes the artifact atomically via
+ * fileOps.writeJson; returns null when no source artifacts exist. Never throws
+ * on missing/partial sources.
+ */
+export async function buildGraphArtifact(projectRoot) {
+  const rootDir = path.resolve(projectRoot ?? process.cwd());
+
+  const [importsArtifact, callsArtifact, cochangeArtifact, filesArtifact] = await Promise.all([
+    readArtifact(rootDir, INDEX_ARTIFACTS.imports, INDEX_SCHEMA_VERSION),
+    readArtifact(rootDir, INDEX_ARTIFACTS.calls, INDEX_SCHEMA_VERSION),
+    readArtifact(rootDir, INDEX_ARTIFACTS.cochange, COCHANGE_SCHEMA_VERSION),
+    readArtifact(rootDir, INDEX_ARTIFACTS.files, INDEX_SCHEMA_VERSION),
+  ]);
+
+  if (!importsArtifact && !callsArtifact && !cochangeArtifact) {
+    return null;
+  }
+
+  const fileSet = new Set();
+  for (const file of filesArtifact?.items ?? []) {
+    if (file?.filePath) fileSet.add(file.filePath);
+  }
+  for (const imp of importsArtifact?.items ?? []) {
+    if (imp?.from) fileSet.add(imp.from);
+  }
+  for (const call of callsArtifact?.items ?? []) {
+    if (call?.filePath) fileSet.add(call.filePath);
+  }
+
+  const nodes = new Map(); // id → node
+  const edges = new Map(); // dedupe key → edge
+  const addFileNode = (filePath) => {
+    if (filePath && !nodes.has(filePath)) {
+      nodes.set(filePath, { id: filePath, kind: 'file', path: filePath });
+    }
+  };
+  const addSymbolNode = (name, filePath) => {
+    if (!name) return null;
+    const id = `symbol:${name}`;
+    if (!nodes.has(id)) {
+      nodes.set(id, { id, kind: 'symbol', path: filePath ?? null });
+    }
+    return id;
+  };
+  const addEdge = (edge, dedupeExtra = '') => {
+    const key = `${edge.from}»${edge.to}»${edge.kind}»${dedupeExtra}`;
+    if (!edges.has(key)) edges.set(key, edge);
+  };
+
+  // import edges: from-file → resolved target file
+  for (const imp of importsArtifact?.items ?? []) {
+    if (!imp?.from || !imp?.to) continue;
+    addFileNode(imp.from);
+    const target = resolveRelativeSpecifier(imp.from, imp.to, fileSet) ?? imp.to;
+    if (target === imp.from) continue;
+    addFileNode(target);
+    addEdge(createEdge({
+      from: imp.from,
+      to: target,
+      kind: 'import',
+      provider: 'json-graph',
+      evidence: INDEX_ARTIFACTS.imports,
+    }));
+  }
+
+  // call edges: caller file → callee file (resolved via symbol→file map);
+  // symbol nodes recorded regardless so queries can address them.
+  const symbolToFile = new Map();
+  for (const call of callsArtifact?.items ?? []) {
+    if (call?.filePath && call?.symbol && !symbolToFile.has(call.symbol)) {
+      symbolToFile.set(call.symbol, call.filePath);
+    }
+  }
+  for (const call of callsArtifact?.items ?? []) {
+    if (!call?.filePath) continue;
+    addFileNode(call.filePath);
+    if (call?.symbol) addSymbolNode(call.symbol, call.filePath);
+    for (const name of call.calls ?? []) {
+      const target = symbolToFile.get(name);
+      if (!target || target === call.filePath) continue;
+      addEdge(createEdge({
+        from: call.filePath,
+        to: target,
+        kind: 'call',
+        provider: 'json-graph',
+        evidence: INDEX_ARTIFACTS.calls,
+      }), name);
+    }
+  }
+
+  // co-change edges: pair endpoints become file nodes; count feeds confidence.
+  for (const pair of cochangeArtifact?.pairs ?? []) {
+    if (!pair?.a || !pair?.b) continue;
+    addFileNode(pair.a);
+    addFileNode(pair.b);
+    const confidence = Math.min(1, (pair.count ?? 1) / 10);
+    addEdge(createEdge({
+      from: pair.a,
+      to: pair.b,
+      kind: 'cochange',
+      confidence,
+      provider: 'json-graph',
+      evidence: INDEX_ARTIFACTS.cochange,
+    }), String(pair.count ?? ''));
+    addEdge(createEdge({
+      from: pair.b,
+      to: pair.a,
+      kind: 'cochange',
+      confidence,
+      provider: 'json-graph',
+      evidence: INDEX_ARTIFACTS.cochange,
+    }), String(pair.count ?? ''));
+  }
+
+  const artifact = {
+    schemaVersion: GRAPH_SCHEMA_VERSION,
+    generatedAt: new Date().toISOString(),
+    nodes: [...nodes.values()],
+    edges: [...edges.values()],
+  };
+
+  try {
+    // Content-gated write: unchanged graph (ignoring generatedAt) must not
+    // bump the artifact mtime — refresh's "no rewrite when nothing changed"
+    // guarantee covers derived artifacts too.
+    const graphPath = getArtifactPath(rootDir, INDEX_ARTIFACTS.codegraph);
+    const existing = await readJsonIfExists(graphPath);
+    const strip = ({ generatedAt, ...rest }) => rest;
+    if (!existing || JSON.stringify(strip(existing)) !== JSON.stringify(strip(artifact))) {
+      await writeJson(graphPath, artifact);
+    }
+  } catch {
+    // Artifact-write failure (read-only .cache, EACCES) is non-fatal — the
+    // in-memory artifact is still returned for the caller/summary.
+  }
+  return artifact;
+}
+
+/**
+ * JsonGraphStore — GraphProvider over `.cache/index/codegraph.json`.
+ * Schema-guarded (GRAPH_SCHEMA_VERSION); a missing/mismatched/corrupt artifact
+ * yields empty results per op, never throws.
+ */
+export class JsonGraphStore {
+  constructor(projectRoot) {
+    this.name = 'json';
+    this.rootDir = path.resolve(projectRoot ?? process.cwd());
+    this._artifactPromise = null;
+  }
+
+  capabilities() {
+    return { nodes: true, edges: true, queries: true };
+  }
+
+  async _load() {
+    if (!this._artifactPromise) {
+      this._artifactPromise = (async () => {
+        const artifact = await readJsonIfExists(getArtifactPath(this.rootDir, INDEX_ARTIFACTS.codegraph));
+        if (!artifact || typeof artifact !== 'object') return null;
+        if (artifact.schemaVersion !== GRAPH_SCHEMA_VERSION) return null;
+        if (!Array.isArray(artifact.nodes) || !Array.isArray(artifact.edges)) return null;
+        return artifact;
+      })();
+    }
+    return this._artifactPromise;
+  }
+
+  async nodes() {
+    const artifact = await this._load();
+    return artifact?.nodes ?? [];
+  }
+
+  async edgesFrom(id) {
+    const artifact = await this._load();
+    if (!artifact || typeof id !== 'string') return [];
+    return artifact.edges.filter((edge) => edge?.from === id);
+  }
+
+  /**
+   * query(q) → edge[] | null. Resolves a node id (file path or `symbol:` id) to
+   * its outgoing edges; null when the node is absent or no artifact exists.
+   */
+  async query(q) {
+    const artifact = await this._load();
+    if (!artifact) return null;
+    const nodeId = typeof q === 'object' && q !== null ? q.path ?? q.id ?? null : q;
+    if (typeof nodeId !== 'string' || nodeId === '') return null;
+    const exists = artifact.nodes.some((node) => node?.id === nodeId);
+    if (!exists) return null;
+    return artifact.edges.filter((edge) => edge?.from === nodeId);
+  }
+}
+
+/**
+ * NullGraphProvider — contract-only stub for SCIP/PDG/LSP/'null' backends.
+ * All capabilities false; every op returns empty/null. Never throws.
+ */
+export class NullGraphProvider {
+  constructor(backend = 'null') {
+    this.name = backend;
+  }
+
+  capabilities() {
+    return { nodes: false, edges: false, queries: false };
+  }
+
+  async nodes() {
+    return [];
+  }
+
+  async edgesFrom() {
+    return [];
+  }
+
+  async query() {
+    return null;
+  }
+}
+
+/**
+ * createGraphProvider({ projectRoot?, config? }) → GraphProvider
+ * codeIntel.graph.store: 'json' (default/unset) → JsonGraphStore;
+ * 'scip' | 'pdg' | 'lsp' | 'null' → NullGraphProvider (contract-only seam).
+ */
+export function createGraphProvider({ projectRoot, config } = {}) {
+  if (config?.codeIntel?.graph?.enabled === false) {
+    return new NullGraphProvider('disabled');
+  }
+  const store = config?.codeIntel?.graph?.store;
+  if (store === undefined || store === 'json') {
+    return new JsonGraphStore(projectRoot);
+  }
+  if (VALID_GRAPH_STORES.has(store)) {
+    return new NullGraphProvider(store);
+  }
+  return new NullGraphProvider('null');
+}