@flyingrobots/graft 0.6.1 → 0.7.0

package/src/warp/index-head.ts

@@ -0,0 +1,348 @@
+// ---------------------------------------------------------------------------
+// Index HEAD — emit compact structural facts into WARP.
+//
+// Also emits a commit node and commit→sym edges (adds/changes/removes)
+// so the structural query layer can detect what changed between ticks.
+// ---------------------------------------------------------------------------
+
+import { Buffer } from "node:buffer";
+import type { GitClient } from "../ports/git.js";
+import type { PathOps } from "../ports/paths.js";
+import type { WarpContext } from "./context.js";
+import type { PatchBuilderV2 } from "@git-stunts/git-warp";
+import { patchGraph, observeGraph, materializeGraph } from "./context.js";
+import { detectLang } from "../parser/lang.js";
+import { parseStructuredTree } from "../parser/runtime.js";
+import { extractOutlineForFile } from "../parser/outline.js";
+import { attachAstSnapshot, emitAstNodes } from "./ast-emitter.js";
+import { resolveImportEdges } from "./ast-import-resolver.js";
+import type { OutlineEntry } from "../parser/types.js";
+import { getCommitMeta } from "./commit-meta.js";
+
+export const DEFAULT_INDEX_MAX_FILES_PER_CALL = 64;
+export const DEFAULT_INDEX_MAX_PATCH_JSON_BYTES = 2 * 1024 * 1024;
+
+// ---------------------------------------------------------------------------
+// Sym-node emission from outline entries (richer than the AST-only walker:
+// includes signatures for change detection).
+// ---------------------------------------------------------------------------
+
+function emitOutlineSyms(
+  patch: PatchBuilderV2,
+  filePath: string,
+  entries: readonly OutlineEntry[],
+  jumpLookup: ReadonlyMap<string, { start: number; end: number }>,
+  parentPath = "",
+): Set<string> {
+  const emitted = new Set<string>();
+  const fileId = `file:${filePath}`;
+
+  for (const entry of entries) {
+    const symbolPath = parentPath.length > 0 ? `${parentPath}.${entry.name}` : entry.name;
+    const symId = `sym:${filePath}:${symbolPath}`;
+    emitted.add(symId);
+
+    patch.addNode(symId);
+    patch.setProperty(symId, "name", entry.name);
+    patch.setProperty(symId, "kind", entry.kind);
+    patch.setProperty(symId, "exported", entry.exported);
+    patch.setProperty(symId, "filePath", filePath);
+    if (entry.signature !== undefined) {
+      patch.setProperty(symId, "signature", entry.signature);
+    }
+    const jump = jumpLookup.get(entry.name);
+    if (jump !== undefined) {
+      patch.setProperty(symId, "startLine", jump.start);
+      patch.setProperty(symId, "endLine", jump.end);
+    }
+    patch.addEdge(fileId, symId, "contains");
+
+    if (entry.children !== undefined && entry.children.length > 0) {
+      for (const childId of emitOutlineSyms(patch, filePath, entry.children, jumpLookup, symbolPath)) {
+        emitted.add(childId);
+      }
+    }
+  }
+  return emitted;
+}
+
+
+// ---------------------------------------------------------------------------
+// Directory chain emission (inlined from indexer-graph.ts to avoid dep on
+// the legacy indexer module).
+// ---------------------------------------------------------------------------
+
+function emitDirectoryChain(patch: PatchBuilderV2, filePath: string): void {
+  const parts = filePath.split("/");
+  if (parts.length <= 1) return;
+
+  let current = "";
+  for (let index = 0; index < parts.length - 1; index++) {
+    const parent = current;
+    const part = parts[index] ?? "";
+    current = current.length > 0 ? `${current}/${part}` : part;
+    const dirId = `dir:${current}`;
+    patch.addNode(dirId);
+    patch.setProperty(dirId, "path", current);
+
+    if (parent.length > 0) {
+      patch.addEdge(`dir:${parent}`, dirId, "contains");
+    }
+  }
+
+  patch.addEdge(`dir:${current}`, `file:${filePath}`, "contains");
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export interface IndexHeadOptions {
+  readonly cwd: string;
+  readonly git: GitClient;
+  readonly pathOps: PathOps;
+  readonly ctx: WarpContext;
+  readonly paths?: readonly string[] | undefined;
+  readonly maxFilesPerCall?: number | undefined;
+  readonly maxPatchJsonBytes?: number | undefined;
+}
+
+export interface IndexHeadResult {
+  readonly ok: true;
+  readonly filesIndexed: number;
+  readonly nodesEmitted: number;
+}
+
+interface HeadCommitMetadata {
+  readonly message: string;
+  readonly author: string;
+  readonly email: string;
+  readonly timestamp: string;
+}
+
+/**
+ * Index the current HEAD by writing one bounded patch per file.
+ *
+ * Full tree-sitter syntax is attached as file-node blob content. The graph
+ * itself only stores compact query facts: files, symbols, import/reference
+ * anchors, directories, and commit-to-symbol touch edges.
+ *
+ * Emits a `commit:{sha}` node with edges to sym nodes labelled
+ * "adds", "changes", or "removes" so structural queries can detect
+ * what changed between ticks.
+ */
+export async function indexHead(opts: IndexHeadOptions): Promise<IndexHeadResult> {
+  const { cwd, git, pathOps, ctx } = opts;
+  const maxFilesPerCall = opts.maxFilesPerCall ?? DEFAULT_INDEX_MAX_FILES_PER_CALL;
+  const maxPatchJsonBytes = opts.maxPatchJsonBytes ?? DEFAULT_INDEX_MAX_PATCH_JSON_BYTES;
+
+  const headResult = await git.run({ args: ["rev-parse", "HEAD"], cwd });
+  if (headResult.status !== 0) {
+    throw new Error(`git rev-parse HEAD failed: ${headResult.stderr}`);
+  }
+  const headSha = headResult.stdout.trim();
+  const commitMeta = await getCommitMeta(git, headSha, cwd);
+
+  const allFiles = await listTrackedFiles(cwd, git);
+  const knownFiles = new Set(allFiles);
+  const parseableFiles = selectParseableFiles(allFiles, opts.paths);
+
+  if (parseableFiles.length > maxFilesPerCall) {
+    throw new Error(
+      `indexHead refused to index ${String(parseableFiles.length)} files in one call. ` +
+      `Lazy indexing policy allows at most ${String(maxFilesPerCall)} files; ` +
+      `pass explicit paths or index on read.`,
+    );
+  }
+
+  let filesIndexed = 0;
+  for (const filePath of parseableFiles) {
+    const indexed = await indexHeadFile({
+      cwd,
+      git,
+      pathOps,
+      ctx,
+      headSha,
+      commitMeta,
+      knownFiles,
+      filePath,
+      maxPatchJsonBytes,
+    });
+    if (indexed) filesIndexed++;
+  }
+
+  return { ok: true, filesIndexed, nodesEmitted: filesIndexed };
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function collectSymIds(
+  target: Map<string, string | undefined>,
+  filePath: string,
+  entries: readonly OutlineEntry[],
+  parentPath = "",
+): void {
+  for (const entry of entries) {
+    const symbolPath = parentPath.length > 0 ? `${parentPath}.${entry.name}` : entry.name;
+    target.set(`sym:${filePath}:${symbolPath}`, entry.signature);
+    if (entry.children !== undefined && entry.children.length > 0) {
+      collectSymIds(target, filePath, entry.children, symbolPath);
+    }
+  }
+}
+
+async function listTrackedFiles(cwd: string, git: GitClient): Promise<readonly string[]> {
+  const result = await git.run({ args: ["ls-files"], cwd });
+  if (result.status !== 0) {
+    throw new Error(`git ls-files failed: ${result.stderr}`);
+  }
+  return result.stdout.trim().split("\n").filter((filePath) => filePath.length > 0);
+}
+
+function selectParseableFiles(
+  allFiles: readonly string[],
+  requestedPaths: readonly string[] | undefined,
+): readonly string[] {
+  const tracked = new Set(allFiles);
+  const candidates = requestedPaths !== undefined && requestedPaths.length > 0
+    ? requestedPaths.map((filePath) => filePath.trim()).filter((filePath) => filePath.length > 0 && tracked.has(filePath))
+    : allFiles;
+
+  return candidates.filter((filePath) => detectLang(filePath) !== null);
+}
+
+async function readPriorSymsForFile(
+  ctx: WarpContext,
+  filePath: string,
+): Promise<Map<string, string | undefined>> {
+  const priorSyms = new Map<string, string | undefined>();
+  const priorObs = await observeGraph(ctx, { match: `sym:${filePath}:*`, expose: ["signature"] });
+  const priorNodes = await priorObs.getNodes();
+  for (const nodeId of priorNodes) {
+    const props = await priorObs.getNodeProps(nodeId);
+    const sig = typeof props?.["signature"] === "string" ? props["signature"] : undefined;
+    priorSyms.set(nodeId, sig);
+  }
+  return priorSyms;
+}
+
+async function readPriorAstAnchorsForFile(
+  ctx: WarpContext,
+  filePath: string,
+): Promise<readonly string[]> {
+  const obs = await observeGraph(ctx, { match: `ast:${filePath}:*`, expose: [] });
+  return obs.getNodes();
+}
+
+function patchJsonByteLength(patch: PatchBuilderV2): number {
+  return Buffer.byteLength(JSON.stringify(patch.build()), "utf8");
+}
+
+function assertPatchWithinBudget(
+  patch: PatchBuilderV2,
+  filePath: string,
+  maxPatchJsonBytes: number,
+): void {
+  const bytes = patchJsonByteLength(patch);
+  if (bytes > maxPatchJsonBytes) {
+    throw new Error(
+      `indexHead refused to write ${filePath}: estimated patch payload ${String(bytes)} bytes ` +
+      `exceeds ${String(maxPatchJsonBytes)} bytes. Index smaller slices or attach bulky data as blobs.`,
+    );
+  }
+}
+
+async function indexHeadFile(input: {
+  readonly cwd: string;
+  readonly git: GitClient;
+  readonly pathOps: PathOps;
+  readonly ctx: WarpContext;
+  readonly headSha: string;
+  readonly commitMeta: HeadCommitMetadata;
+  readonly knownFiles: ReadonlySet<string>;
+  readonly filePath: string;
+  readonly maxPatchJsonBytes: number;
+}): Promise<boolean> {
+  const { cwd, git, pathOps, ctx, headSha, commitMeta, knownFiles, filePath, maxPatchJsonBytes } = input;
+  const lang = detectLang(filePath);
+  if (lang === null) return false;
+
+  const contentResult = await git.run({ args: ["show", `HEAD:${filePath}`], cwd });
+  if (contentResult.status !== 0) return false;
+
+  const tree = parseStructuredTree(lang, contentResult.stdout);
+  try {
+    const outlineResult = extractOutlineForFile(filePath, contentResult.stdout);
+    const outline = outlineResult?.entries ?? [];
+    const jumpLookup = new Map<string, { start: number; end: number }>();
+    for (const jt of outlineResult?.jumpTable ?? []) {
+      jumpLookup.set(jt.symbol, { start: jt.start, end: jt.end });
+    }
+
+    const currentSyms = new Map<string, string | undefined>();
+    collectSymIds(currentSyms, filePath, outline);
+
+    let priorSyms = new Map<string, string | undefined>();
+    let priorAstAnchors: readonly string[] = [];
+    try {
+      await materializeGraph(ctx);
+      priorSyms = await readPriorSymsForFile(ctx, filePath);
+      priorAstAnchors = await readPriorAstAnchorsForFile(ctx, filePath);
+    } catch {
+      // No prior state: first lazy index writes adds only.
+    }
+
+    await patchGraph(ctx, async (patch) => {
+      const tick = patch.build().lamport;
+      const commitId = `commit:${headSha}`;
+      const fileId = `file:${filePath}`;
+
+      patch.addNode(commitId);
+      patch.setProperty(commitId, "sha", headSha);
+      patch.setProperty(commitId, "tick", tick);
+      patch.setProperty(commitId, "message", commitMeta.message);
+      patch.setProperty(commitId, "author", commitMeta.author);
+      patch.setProperty(commitId, "email", commitMeta.email);
+      patch.setProperty(commitId, "date", commitMeta.timestamp);
+      patch.setProperty(commitId, "timestamp", commitMeta.timestamp);
+
+      patch.addNode(fileId);
+      patch.setProperty(fileId, "path", filePath);
+      patch.setProperty(fileId, "lang", lang);
+      patch.setProperty(fileId, "indexedCommit", headSha);
+      patch.addEdge(commitId, fileId, "touches");
+
+      emitDirectoryChain(patch, filePath);
+      for (const anchorId of priorAstAnchors) {
+        patch.removeNode(anchorId);
+      }
+      emitOutlineSyms(patch, filePath, outline, jumpLookup);
+      emitAstNodes(patch, filePath, tree.root);
+      resolveImportEdges(patch, filePath, tree.root, pathOps, knownFiles);
+      await attachAstSnapshot(patch, filePath, tree.root);
+
+      for (const [symId, sig] of currentSyms) {
+        if (!priorSyms.has(symId)) {
+          patch.addEdge(commitId, symId, "adds");
+        } else if (priorSyms.get(symId) !== sig) {
+          patch.addEdge(commitId, symId, "changes");
+        }
+      }
+
+      for (const symId of priorSyms.keys()) {
+        if (!currentSyms.has(symId)) {
+          patch.addEdge(commitId, symId, "removes");
+          patch.removeNode(symId);
+        }
+      }
+
+      assertPatchWithinBudget(patch, filePath, maxPatchJsonBytes);
+    });
+
+    return true;
+  } finally {
+    tree.delete();
+  }
+}

package/src/warp/observers.ts

@@ -5,14 +5,15 @@
  * walks the graph directly, maintains shadow state, or implements
  * traversal algorithms.
  *
- * Each function returns an observer with a focused lens. The lens
- * determines the aperture — what the observer can see.
+ * Each function returns a lens or an observer with a focused lens.
+ * The lens determines the aperture — what the observer can see.
  */
 
-import type { WarpHandle, WarpObserver, WarpObserverLens } from "../ports/warp.js";
+import type { Lens, Observer } from "@git-stunts/git-warp";
+import type { WarpContext } from "./context.js";
+import { observeGraph } from "./context.js";
 
-/** Lens config for creating focused observers. */
-export type Lens = WarpObserverLens;
+export type { Lens };
 
 /**
  * Observe all symbols in a specific file.
@@ -95,6 +96,6 @@ export function commitsLens(): Lens {
  * Create an observer on the current frontier with a given lens.
  * Observers are static snapshots — create a new one after writes.
  */
-export function observe(warp: WarpHandle, lens: Lens): Promise<WarpObserver> {
-  return warp.observer(lens);
+export function observe(ctx: WarpContext, lens: Lens): Promise<Observer> {
+  return observeGraph(ctx, lens);
 }

package/src/warp/open.ts

@@ -2,92 +2,66 @@
  * WARP graph initialization — opens the graft-ast graph backed by
  * the repo's own .git directory.
  *
- * Single entry point. Returns a WARP port handle ready for patch
- * writes and observer reads.
+ * Sole construction adapter. After this, git-warp types flow directly.
  */
 
-import RawWarpApp, { GitGraphAdapter } from "@git-stunts/git-warp";
+import WarpApp, { GitGraphAdapter } from "@git-stunts/git-warp";
 import GitPlumbing from "@git-stunts/plumbing";
-import type {
-  WarpHandle,
-  WarpMaterializeReceipt,
-  WarpObserver,
-  WarpObserverLens,
-  WarpObserverOptions,
-  WarpPatchBuilder,
-} from "../ports/warp.js";
 import { DEFAULT_WARP_WRITER_ID } from "./writer-id.js";
 
 export const GRAPH_NAME = "graft-ast";
+export const DEFAULT_WARP_CHECKPOINT_EVERY = 128;
+const WARP_GIT_MAX_BUFFER_BYTES = 128 * 1024 * 1024;
 
-export interface OpenWarpOptions {
-  readonly cwd: string;
-  readonly writerId?: string;
+interface GitExecuteOptions {
+  readonly args: string[];
+  readonly input?: string | Uint8Array;
+  readonly env?: Record<string, string>;
+  readonly maxBytes?: number;
 }
 
-function toRawLens(lens: WarpObserverLens): {
-  match: string | string[];
-  expose?: string[];
-  redact?: string[];
-} {
-  return {
-    match: typeof lens.match === "string" ? lens.match : [...lens.match],
-    ...(lens.expose !== undefined ? { expose: [...lens.expose] } : {}),
-    ...(lens.redact !== undefined ? { redact: [...lens.redact] } : {}),
-  };
+interface GitStreamLike {
+  readonly finished: Promise<{ code: number; stderr: string; error?: Error | undefined }>;
+  collect(opts?: { maxBytes?: number; asString?: boolean; encoding?: string }): Promise<Uint8Array | string>;
+  [Symbol.asyncIterator](): AsyncIterator<Uint8Array>;
 }
 
-function toRawObserverOptions(options: WarpObserverOptions | undefined): {
-  source?: { kind: "live"; ceiling: number | null };
-} | undefined {
-  if (options === undefined) {
-    return undefined;
-  }
-  if (options.source === undefined) {
-    return {};
-  }
-  return {
-    source: {
-      kind: options.source.kind,
-      ceiling: options.source.ceiling,
-    },
-  };
+function raiseWarpGitBufferLimit(plumbing: GitPlumbing): GitPlumbing {
+  const execute = plumbing.execute.bind(plumbing);
+  plumbing.execute = ((options: GitExecuteOptions) => execute({
+    ...options,
+    maxBytes: options.maxBytes ?? WARP_GIT_MAX_BUFFER_BYTES,
+  })) as GitPlumbing["execute"];
+
+  const executeStream = plumbing.executeStream.bind(plumbing);
+  plumbing.executeStream = (async (options: { args: string[]; input?: string | Uint8Array; env?: Record<string, string> }) => {
+    const stream = await executeStream(options);
+    const collect = stream.collect.bind(stream);
+    stream.collect = (opts = {}) => collect({
+      ...opts,
+      maxBytes: opts.maxBytes ?? WARP_GIT_MAX_BUFFER_BYTES,
+    });
+    return stream as GitStreamLike;
+  }) as GitPlumbing["executeStream"];
+
+  return plumbing;
 }
 
-function wrapWarpApp(app: RawWarpApp): WarpHandle {
-  return {
-    async hasNode(nodeId: string): Promise<boolean> {
-      const observer = await app.observer({ match: nodeId });
-      const nodes = await observer.getNodes();
-      return nodes.includes(nodeId);
-    },
-    observer(lens, options): Promise<WarpObserver> {
-      return app.observer(toRawLens(lens), toRawObserverOptions(options)) as Promise<WarpObserver>;
-    },
-    patch(build): Promise<string> {
-      return app.patch((patch) => build(patch as unknown as WarpPatchBuilder));
-    },
-    async materialize(): Promise<void> {
-      await app.core().materialize();
-    },
-    async materializeReceipts(): Promise<readonly WarpMaterializeReceipt[]> {
-      const { receipts } = await app.core().materialize({ receipts: true });
-      return receipts as readonly WarpMaterializeReceipt[];
-    },
-  };
+export interface OpenWarpOptions {
+  readonly cwd: string;
+  readonly writerId?: string;
+  readonly checkpointEvery?: number;
 }
 
-export async function openWarp(options: OpenWarpOptions): Promise<WarpHandle> {
-  // createDefault() wires the ShellRunnerFactory (required port)
-  const plumbing = GitPlumbing.createDefault({ cwd: options.cwd });
+export async function openWarp(options: OpenWarpOptions): Promise<WarpApp> {
+  const plumbing = raiseWarpGitBufferLimit(GitPlumbing.createDefault({ cwd: options.cwd }));
   const persistence = new GitGraphAdapter({ plumbing });
 
-  const app = await RawWarpApp.open({
+  return WarpApp.open({
     persistence,
     graphName: GRAPH_NAME,
     writerId: options.writerId ?? DEFAULT_WARP_WRITER_ID,
+    checkpointPolicy: { every: options.checkpointEvery ?? DEFAULT_WARP_CHECKPOINT_EVERY },
     onDeleteWithData: "cascade",
   });
-
-  return wrapWarpApp(app);
 }

package/src/warp/outline-diff-trailer.ts

@@ -0,0 +1,100 @@
+// ---------------------------------------------------------------------------
+// Outline Diff Commit Trailer — structural summary in git commit messages
+// ---------------------------------------------------------------------------
+
+/** A single symbol change to include in the trailer. */
+export interface SymbolDiffEntry {
+  readonly name: string;
+  readonly kind: string;
+  readonly changeKind: "added" | "removed" | "changed";
+}
+
+/** Options for trailer formatting. */
+export interface TrailerOptions {
+  /** Maximum entries before truncation. Default: 20. */
+  readonly maxEntries?: number;
+}
+
+/** Short kind abbreviations for the trailer format. */
+const KIND_ABBREV: Readonly<Record<string, string>> = {
+  function: "fn",
+  class: "class",
+  interface: "iface",
+  type: "type",
+  variable: "var",
+  method: "method",
+  property: "prop",
+};
+
+function abbrevKind(kind: string): string {
+  return KIND_ABBREV[kind] ?? kind;
+}
+
+/**
+ * Format a list of symbol changes as a Git commit trailer.
+ *
+ * Format: `Structural-Diff: added fn:X; removed class:Y; changed fn:Z`
+ *
+ * Returns empty string if there are no changes.
+ */
+export function formatStructuralDiffTrailer(
+  entries: readonly SymbolDiffEntry[],
+  options?: TrailerOptions,
+): string {
+  if (entries.length === 0) return "";
+
+  const max = options?.maxEntries ?? 20;
+  const display = entries.slice(0, max);
+  const parts = display.map(
+    (e) => `${e.changeKind} ${abbrevKind(e.kind)}:${e.name}`,
+  );
+
+  let trailer = `Structural-Diff: ${parts.join("; ")}`;
+
+  if (entries.length > max) {
+    trailer += `; ... (${String(entries.length)} total)`;
+  }
+
+  return trailer;
+}
+
+// ---------------------------------------------------------------------------
+// Parser
+// ---------------------------------------------------------------------------
+
+const TRAILER_PREFIX = "Structural-Diff:";
+
+const ENTRY_RE = /^(added|removed|changed)\s+(\w+):(\w+)$/;
+
+/**
+ * Parse a Structural-Diff trailer back into structured entries.
+ *
+ * Skips malformed entries gracefully.
+ */
+export function parseStructuralDiffTrailer(text: string): SymbolDiffEntry[] {
+  const idx = text.indexOf(TRAILER_PREFIX);
+  if (idx === -1) return [];
+
+  const body = text.slice(idx + TRAILER_PREFIX.length).trim();
+  const parts = body.split(";").map((p) => p.trim());
+  const results: SymbolDiffEntry[] = [];
+
+  for (const part of parts) {
+    if (part.startsWith("...")) continue;
+    const match = ENTRY_RE.exec(part);
+    if (match === null) continue;
+
+    const changeKind = match[1] as "added" | "removed" | "changed";
+    const abbrev = match[2] ?? "unknown";
+    const name = match[3] ?? "unknown";
+
+    // Reverse the abbreviation
+    const kind = Object.entries(KIND_ABBREV).find(
+      ([, v]) => v === abbrev,
+    )?.[0] ?? abbrev;
+
+    results.push({ name, kind, changeKind });
+  }
+
+  return results;
+}

package/src/warp/plumbing.d.ts

@@ -5,8 +5,17 @@ declare module "@git-stunts/plumbing" {
     readonly emptyTree: string;
     constructor(options: { runner: unknown; cwd?: string });
     static createDefault(options?: { cwd?: string; env?: string }): GitPlumbing;
-    execute(options: { args: string[]; input?: string | Uint8Array }): Promise<string>;
-    executeStream(options: { args: string[] }): Promise<{
+    execute(options: {
+      args: string[];
+      input?: string | Uint8Array;
+      env?: Record<string, string>;
+      maxBytes?: number;
+    }): Promise<string>;
+    executeStream(options: {
+      args: string[];
+      input?: string | Uint8Array;
+      env?: Record<string, string>;
+    }): Promise<{
       finished: Promise<{ code: number; stderr: string; error?: Error }>;
       collect(opts?: { maxBytes?: number; asString?: boolean; encoding?: string }): Promise<Uint8Array | string>;
       [Symbol.asyncIterator](): AsyncIterator<Uint8Array>;