@ngockhoale/ukit 2.6.6 → 2.6.8

package/src/core/codeintel/manifest.js

@@ -0,0 +1,176 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+import { getIndexDir } from '../../index/paths.js';
+
+// Manifest scanner (SPEC §1) — walks the project, records a sha1 per file, and
+// caches the result at `getIndexDir()/manifest.json`. The cache is validated on
+// (path, mtime, size) so an unchanged tree never re-hashes, and any mtime/size
+// drift forces a full rescan.
+
+const HARD_EXCLUDES = new Set(['.git', '.cache', 'node_modules']);
+const MANIFEST_CACHE = 'manifest.json';
+
+const IGNORED_BASENAMES = new Set(['.DS_Store']);
+
+function sha1(content) {
+  return crypto.createHash('sha1').update(content).digest('hex');
+}
+
+// Minimal .gitignore subset: blank lines and `#` comments skipped, `!` negation
+// unsupported (documented — negations are rare in-repo and the safe failure mode
+// is scanning a file that should be ignored, never dropping a needed one).
+// Supported: `name` (basename match anywhere), `dir/` (directory), `/path`
+// (root-anchored), `*` single-segment wildcard.
+function parseGitignore(content) {
+  return String(content ?? '')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0 && !line.startsWith('#') && !line.startsWith('!'));
+}
+
+function gitignorePatternToRegExp(pattern) {
+  let anchored = false;
+  let dirOnly = false;
+  let body = pattern;
+  if (body.endsWith('/')) {
+    dirOnly = true;
+    body = body.slice(0, -1);
+  }
+  if (body.startsWith('/')) {
+    anchored = true;
+    body = body.slice(1);
+  }
+  const escaped = body
+    .split('*')
+    .map((part) => part.replace(/[.+^${}()|[\]\\]/g, '\\$&'))
+    .join('[^/]*');
+  if (anchored || body.includes('/')) {
+    return { re: new RegExp(`^${escaped}(?:/.*)?$`), dirOnly };
+  }
+  return { re: new RegExp(`(?:^|/)${escaped}(?:/.*)?$`), dirOnly };
+}
+
+function loadIgnoreMatchers(rootDir, extraIgnore) {
+  return fs.readFile(path.join(rootDir, '.gitignore'), 'utf8')
+    .then((content) => [...parseGitignore(content), ...(extraIgnore ?? [])])
+    .then((patterns) => patterns.map(gitignorePatternToRegExp))
+    .catch(() => (extraIgnore ?? []).map(gitignorePatternToRegExp));
+}
+
+function isIgnored(relPath, isDir, matchers) {
+  return matchers.some(({ re, dirOnly }) => (!dirOnly || isDir) && re.test(relPath));
+}
+
+async function walk(rootDir, matchers, options) {
+  const results = [];
+  const stack = [''];
+  while (stack.length > 0) {
+    const relDir = stack.pop();
+    const absDir = path.join(rootDir, relDir);
+    let entries;
+    try {
+      entries = await fs.readdir(absDir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const relPath = relDir ? `${relDir}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        if (HARD_EXCLUDES.has(entry.name)) {
+          continue;
+        }
+        if (isIgnored(relPath, true, matchers)) {
+          continue;
+        }
+        stack.push(relPath);
+      } else if (entry.isFile()) {
+        if (IGNORED_BASENAMES.has(entry.name)) {
+          continue;
+        }
+        if (isIgnored(relPath, false, matchers)) {
+          continue;
+        }
+        if (options?.filter && !options.filter(relPath)) {
+          continue;
+        }
+        try {
+          const stat = await fs.stat(path.join(rootDir, relPath));
+          results.push({ path: relPath, mtime: stat.mtimeMs, size: stat.size });
+        } catch {
+          // Vanished mid-walk — skip.
+        }
+      }
+    }
+  }
+  results.sort((a, b) => a.path.localeCompare(b.path));
+  return results;
+}
+
+async function readCachedManifest(rootDir) {
+  try {
+    const raw = await fs.readFile(path.join(getIndexDir(rootDir), MANIFEST_CACHE), 'utf8');
+    const parsed = JSON.parse(raw);
+    if (!parsed || !Array.isArray(parsed.files)) {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function snapshotMatches(cachedFiles, scannedFiles) {
+  if (!Array.isArray(cachedFiles) || cachedFiles.length !== scannedFiles.length) {
+    return false;
+  }
+  const byPath = new Map(cachedFiles.map((f) => [f.path, f]));
+  return scannedFiles.every((entry) => {
+    const cached = byPath.get(entry.path);
+    return cached && cached.mtime === entry.mtime && cached.size === entry.size
+      && typeof cached.sha1 === 'string';
+  });
+}
+
+/**
+ * scanManifest(projectRoot, { ignore, filter }) →
+ *   { files: [{ path, sha1, mtime, size }], generatedAt, fromCache }
+ *
+ * `ignore` adds .gitignore-style patterns on top of the repo .gitignore.
+ * `filter(relPath)` optionally keeps a subset of scanned files.
+ * Never throws on a missing/unreadable tree — returns an empty manifest.
+ */
+export async function scanManifest(projectRoot, { ignore, filter } = {}) {
+  const rootDir = path.resolve(projectRoot ?? process.cwd());
+  const matchers = await loadIgnoreMatchers(rootDir, ignore);
+  const scanned = await walk(rootDir, matchers, { filter });
+
+  const cached = await readCachedManifest(rootDir);
+  if (cached && snapshotMatches(cached.files, scanned)) {
+    return { files: cached.files, generatedAt: cached.generatedAt, fromCache: true };
+  }
+
+  const files = await Promise.all(scanned.map(async (entry) => {
+    let fileSha = null;
+    try {
+      const content = await fs.readFile(path.join(rootDir, entry.path));
+      fileSha = sha1(content);
+    } catch {
+      fileSha = null;
+    }
+    return { path: entry.path, sha1: fileSha, mtime: entry.mtime, size: entry.size };
+  }));
+
+  const manifest = { files, generatedAt: new Date().toISOString() };
+  try {
+    await fs.mkdir(getIndexDir(rootDir), { recursive: true });
+    await fs.writeFile(
+      path.join(getIndexDir(rootDir), MANIFEST_CACHE),
+      JSON.stringify({ files, generatedAt: manifest.generatedAt }, null, 2),
+    );
+  } catch {
+    // Cache write failure must not fail the scan.
+  }
+  return { ...manifest, fromCache: false };
+}

package/src/core/codeintel/packet.js

@@ -0,0 +1,146 @@
+// Context Packet v1 — shared envelope for every `ukit code` result (SPEC §5).
+// JSON-serializable contract; no clock/random in static fields (CTX-04).
+
+export const PACKET_VERSION = 1;
+
+const EVIDENCE_LEVELS = new Set(['L0', 'L1', 'L2', 'L3', 'L4']);
+const EVIDENCE_SOURCES = new Set(['index', 'file', 'git', 'memory']);
+
+/**
+ * createPacket({ repo, snapshot, taskType, budget }) → packet
+ */
+export function createPacket({ repo, snapshot, taskType, budget } = {}) {
+  const requested = budget && typeof budget.requested === 'number' ? budget.requested : 0;
+  return {
+    version: PACKET_VERSION,
+    repo: repo ?? '',
+    snapshot: snapshot ?? '',
+    task_type: taskType ?? '',
+    freshness: { level: 'L0', headSha: '', overlayHash: '', configHash: '', stale: false },
+    budget: { requested, used: 0 },
+    anchors: [],
+    evidence: [],
+    relations: [],
+    tests: [],
+    memory: [],
+    omitted: [],
+    next_actions: [],
+  };
+}
+
+function validateEvidenceItem(item, i, errors) {
+  if (!item || typeof item !== 'object' || Array.isArray(item)) {
+    errors.push(`evidence[${i}] must be an object`);
+    return;
+  }
+  if (!EVIDENCE_LEVELS.has(item.level)) errors.push(`evidence[${i}].level must be one of L0..L4`);
+  if (!EVIDENCE_SOURCES.has(item.source)) errors.push(`evidence[${i}].source must be one of index|file|git|memory`);
+  if (typeof item.why !== 'string' || item.why.length === 0) errors.push(`evidence[${i}].why is required`);
+}
+
+/**
+ * validatePacket(packet) → { ok, errors }
+ */
+export function validatePacket(packet) {
+  const errors = [];
+  if (!packet || typeof packet !== 'object' || Array.isArray(packet)) {
+    return { ok: false, errors: ['packet must be an object'] };
+  }
+  if (packet.version !== PACKET_VERSION) errors.push(`version must be ${PACKET_VERSION}`);
+  if (typeof packet.repo !== 'string' || packet.repo.length === 0) errors.push('repo is required');
+  if (typeof packet.snapshot !== 'string' || packet.snapshot.length === 0) errors.push('snapshot is required');
+  if (typeof packet.task_type !== 'string') errors.push('task_type must be a string');
+  if (!packet.freshness || typeof packet.freshness !== 'object') {
+    errors.push('freshness object is required');
+  } else if (typeof packet.freshness.stale !== 'boolean') {
+    errors.push('freshness.stale must be a boolean');
+  }
+  if (!packet.budget || typeof packet.budget !== 'object') {
+    errors.push('budget object is required');
+  } else {
+    if (typeof packet.budget.requested !== 'number') errors.push('budget.requested must be a number');
+    if (typeof packet.budget.used !== 'number') errors.push('budget.used must be a number');
+  }
+  for (const key of ['anchors', 'evidence', 'relations', 'tests', 'memory', 'omitted', 'next_actions']) {
+    if (!Array.isArray(packet[key])) errors.push(`${key} must be an array`);
+  }
+  if (Array.isArray(packet.evidence)) {
+    packet.evidence.forEach((item, i) => validateEvidenceItem(item, i, errors));
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/**
+ * addEvidence(packet, evidence) → packet — enforces shape, then appends.
+ * Throws on invalid evidence; packet left unchanged.
+ */
+export function addEvidence(packet, evidence) {
+  const errors = [];
+  validateEvidenceItem(evidence, 0, errors);
+  if (errors.length > 0) throw new Error(`invalid evidence: ${errors.join('; ')}`);
+  if (!Array.isArray(packet.evidence)) packet.evidence = [];
+  packet.evidence.push({ ...evidence });
+  return packet;
+}
+
+/**
+ * packetToText(packet) → string — deterministic human/LLM-facing render.
+ */
+export function packetToText(packet) {
+  const lines = [];
+  lines.push('## Context Packet');
+  lines.push(`version: ${packet.version}`);
+  lines.push(`repo: ${packet.repo}`);
+  lines.push(`snapshot: ${packet.snapshot}`);
+  lines.push(`task_type: ${packet.task_type}`);
+  const f = packet.freshness || {};
+  lines.push(`freshness: level=${f.level ?? ''} stale: ${f.stale === true}`);
+  lines.push(`budget: ${packet.budget?.used ?? 0}/${packet.budget?.requested ?? 0}`);
+
+  lines.push('');
+  lines.push('## Anchors');
+  for (const a of packet.anchors ?? []) {
+    lines.push(`- ${a.path ?? ''}${typeof a.line === 'number' ? `:${a.line}` : ''}${a.symbol ? ` (${a.symbol})` : ''}`);
+  }
+  if ((packet.anchors ?? []).length === 0) lines.push('(none)');
+
+  lines.push('');
+  lines.push('## Evidence');
+  for (const e of packet.evidence ?? []) {
+    lines.push(`- [${e.level}/${e.source}] ${e.path ?? ''} — ${e.why ?? ''}`);
+    if (e.excerpt) lines.push(`  ${e.excerpt}`);
+  }
+  if ((packet.evidence ?? []).length === 0) lines.push('(none)');
+
+  lines.push('');
+  lines.push('## Relations');
+  for (const r of packet.relations ?? []) {
+    lines.push(`- ${r.from ?? ''} -[${r.kind ?? ''}]-> ${r.to ?? ''} (confidence: ${r.confidence ?? 0}${r.provider ? `, ${r.provider}` : ''})`);
+  }
+  if ((packet.relations ?? []).length === 0) lines.push('(none)');
+
+  lines.push('');
+  lines.push('## Tests');
+  for (const t of packet.tests ?? []) lines.push(`- ${t}`);
+  if ((packet.tests ?? []).length === 0) lines.push('(none)');
+
+  lines.push('');
+  lines.push('## Omitted');
+  for (const o of packet.omitted ?? []) lines.push(`- ${o.what ?? ''} — ${o.why ?? ''}`);
+  if ((packet.omitted ?? []).length === 0) lines.push('(none)');
+
+  lines.push('');
+  lines.push('## Next Actions');
+  for (const n of packet.next_actions ?? []) {
+    if (n && typeof n === 'object') {
+      // Object form (SPEC §8): - [<kind>] <file(s)> — <message>
+      const target = Array.isArray(n.files) ? n.files.join(', ') : (n.file ?? '');
+      lines.push(`- [${n.kind ?? 'action'}] ${target} — ${n.message ?? n.why ?? ''}`);
+    } else {
+      lines.push(`- ${n}`);
+    }
+  }
+  if ((packet.next_actions ?? []).length === 0) lines.push('(none)');
+
+  return lines.join('\n');
+}

package/src/core/codeintel/providers.js

@@ -0,0 +1,201 @@
+import fs from 'node:fs';
+
+import { getFileOutline } from '../../index/queryIndex.js';
+import { getArtifactPath, INDEX_ARTIFACTS } from '../../index/paths.js';
+
+/**
+ * Provider contracts (SPEC §8) — the future LSP adapter seam. Contract only, no new deps.
+ *
+ * SyntaxProvider (plain object or class instance):
+ *   .name                         → string, unique across the registry
+ *   .supports(filePath)           → bool (sync, cheap — no I/O beyond an existence check)
+ *   .symbols(filePath, src?)      → Promise<[{ name, kind, line }]>
+ *   .outline(filePath, src?)      → Promise<[{ kind, line }]>
+ *
+ * SemanticProvider:
+ *   .name                         → string
+ *   .capabilities()               → { defs: bool, refs: bool, types: bool }
+ *   .resolve(symbol, ctx)         → Promise<edge[] | null>
+ *
+ * Every returned edge carries: { from, to, kind, confidence, provider, evidence, snapshot }.
+ */
+
+export class UnsupportedOperation extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'UnsupportedOperation';
+  }
+}
+
+/**
+ * Canonical edge shape for codeintel results (SPEC §8). `confidence` is 0..1,
+ * `provider` names the producing provider, `evidence` records where the edge came
+ * from (artifact, rule, probe), `snapshot` ties it to the index/content revision.
+ */
+export function createEdge({ from, to, kind, confidence = 1, provider = 'unknown', evidence = null, snapshot = null } = {}) {
+  return Object.freeze({
+    from: from ?? null,
+    to: to ?? null,
+    kind: kind ?? 'unknown',
+    confidence: typeof confidence === 'number' ? confidence : 1,
+    provider,
+    evidence,
+    snapshot,
+  });
+}
+
+/**
+ * Syntax provider backed by the existing `.cache/index/` artifacts — zero new
+ * parsing. Reads `symbols.json` through `getFileOutline` (`src/index/queryIndex.js`),
+ * which already handles the artifact cache, schema guard, and never-throws contract.
+ */
+export class IndexFileSyntaxProvider {
+  constructor({ rootDir = process.cwd() } = {}) {
+    this.name = 'index-file';
+    this.rootDir = rootDir;
+  }
+
+  // Sync existence check only — the expensive read stays inside getFileOutline's
+  // cached artifact load, and a missing index must never throw.
+  supports(filePath) {
+    if (!filePath) {
+      return false;
+    }
+    try {
+      return fs.existsSync(getArtifactPath(this.rootDir, INDEX_ARTIFACTS.symbols));
+    } catch {
+      return false;
+    }
+  }
+
+  async symbols(filePath) {
+    const outline = await this.#outline(filePath);
+    return outline.map((item) => ({
+      name: item.name,
+      kind: item.type,
+      line: item.line,
+    }));
+  }
+
+  async outline(filePath) {
+    const outline = await this.#outline(filePath);
+    return outline.map((item) => ({ kind: item.type, line: item.line }));
+  }
+
+  async #outline(filePath) {
+    if (!filePath) {
+      return [];
+    }
+    try {
+      return await getFileOutline({ rootDir: this.rootDir, filePath, limit: Number.MAX_SAFE_INTEGER });
+    } catch {
+      return [];
+    }
+  }
+}
+
+/**
+ * Default semantic provider: advertises no capabilities and resolves nothing.
+ * Keeps call sites uniform until a real provider (e.g. LspSemanticProvider, CI-2xx)
+ * registers over it.
+ */
+export class NullSemanticProvider {
+  constructor() {
+    this.name = 'null';
+  }
+
+  capabilities() {
+    return { defs: false, refs: false, types: false };
+  }
+
+  async resolve() {
+    return null;
+  }
+}
+
+const registeredProviders = [];
+const registeredNames = new Set();
+let semanticProvider = null;
+
+/**
+ * Register a provider instance. Custom providers dispatch ahead of the built-in
+ * `index-file` provider (which is appended lazily so it is always the fallback).
+ * Duplicate names are rejected — a name collision would make dispatch ambiguous.
+ */
+export function registerProvider(provider) {
+  if (!provider || typeof provider !== 'object') {
+    throw new TypeError('registerProvider expects a provider object');
+  }
+  if (typeof provider.name !== 'string' || provider.name.length === 0) {
+    throw new TypeError('provider.name must be a non-empty string');
+  }
+  if (registeredNames.has(provider.name)) {
+    throw new UnsupportedOperation(`provider already registered: ${provider.name}`);
+  }
+  registeredNames.add(provider.name);
+
+  if (typeof provider.resolve === 'function' && typeof provider.capabilities === 'function'
+    && typeof provider.symbols !== 'function') {
+    semanticProvider = provider;
+    return provider;
+  }
+
+  const indexFile = registeredProviders.find((entry) => entry.name === 'index-file');
+  if (indexFile) {
+    registeredProviders.splice(registeredProviders.indexOf(indexFile), 0, provider);
+  } else {
+    registeredProviders.push(provider);
+  }
+  return provider;
+}
+
+export function listProviders() {
+  ensureBuiltins();
+  return [...registeredProviders];
+}
+
+/**
+ * First registered provider whose `supports(filePath)` returns true, checked in
+ * registration order with `index-file` always last (the fallback).
+ */
+export function getSyntaxProvider(filePath) {
+  ensureBuiltins();
+  for (const provider of registeredProviders) {
+    try {
+      if (provider.supports(filePath)) {
+        return provider;
+      }
+    } catch {
+      // A throwing supports() must not break dispatch — treat as unsupported.
+    }
+  }
+  return null;
+}
+
+/**
+ * The registered SemanticProvider, or NullSemanticProvider when none was
+ * registered (the `codeIntel.providers.semantic: "null"` default).
+ */
+export function getSemanticProvider() {
+  return semanticProvider ?? new NullSemanticProvider();
+}
+
+let builtinsEnsured = false;
+function ensureBuiltins() {
+  if (builtinsEnsured) {
+    return;
+  }
+  builtinsEnsured = true;
+  if (!registeredProviders.some((entry) => entry.name === 'index-file')) {
+    registeredProviders.push(new IndexFileSyntaxProvider({}));
+    registeredNames.add('index-file');
+  }
+}
+
+// Exported for tests: clears registry state back to defaults.
+export function resetProvidersForTest() {
+  registeredProviders.length = 0;
+  registeredNames.clear();
+  semanticProvider = null;
+  builtinsEnsured = false;
+}