@flyingrobots/graft 0.5.0 → 0.6.0

package/src/mcp/daemon-session-host.ts

@@ -0,0 +1,230 @@
+import * as crypto from "node:crypto";
+import * as fs from "node:fs/promises";
+import * as http from "node:http";
+import * as path from "node:path";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
+import { isInitializeRequest } from "@modelcontextprotocol/sdk/types.js";
+import { createGraftServer, type GraftServer } from "./server.js";
+import type { DaemonControlPlane, DaemonStatusView } from "./daemon-control-plane.js";
+import type { DaemonJobScheduler } from "./daemon-job-scheduler.js";
+import type { ChildProcessDaemonWorkerPool } from "./daemon-worker-pool.js";
+import type { PersistentMonitorRuntime } from "./persistent-monitor-runtime.js";
+import type { RunCaptureConfig } from "./run-capture-config.js";
+import type { RuntimeObservabilityState } from "./runtime-observability.js";
+import type { WarpPool } from "./warp-pool.js";
+import { ensurePrivateDirectory } from "./daemon-bootstrap.js";
+
+const MAX_BODY_BYTES = 1024 * 1024;
+
+interface DaemonSession {
+  readonly id: string;
+  readonly graftDir: string;
+  readonly transport: StreamableHTTPServerTransport;
+  readonly server: GraftServer;
+}
+
+export interface CreateDaemonSessionHostOptions {
+  readonly graftDir: string;
+  readonly socketPath: string;
+  readonly transportKind: "unix_socket" | "named_pipe";
+  readonly healthPath: string;
+  readonly mcpPath: string;
+  readonly startedAt: string;
+  readonly warpPool: WarpPool;
+  readonly controlPlane: DaemonControlPlane;
+  readonly daemonScheduler: DaemonJobScheduler;
+  readonly daemonWorkerPool: ChildProcessDaemonWorkerPool;
+  readonly monitorRuntime: PersistentMonitorRuntime;
+  readonly getHealthStatus: () => DaemonStatusView;
+  readonly env?: Readonly<Record<string, string | undefined>> | undefined;
+  readonly runCapture?: Partial<RunCaptureConfig> | undefined;
+  readonly runtimeObservability?: Partial<RuntimeObservabilityState> | undefined;
+}
+
+export interface DaemonSessionHost {
+  handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void>;
+  close(): Promise<void>;
+}
+
+function getHeader(req: http.IncomingMessage, name: string): string | undefined {
+  const value = req.headers[name];
+  return Array.isArray(value) ? value[0] : value;
+}
+
+async function readJsonBody(req: http.IncomingMessage): Promise<unknown> {
+  const chunks: Buffer[] = [];
+  let total = 0;
+  for await (const chunk of req as AsyncIterable<Buffer | string>) {
+    const buffer = typeof chunk === "string" ? Buffer.from(chunk, "utf-8") : chunk;
+    total += buffer.length;
+    if (total > MAX_BODY_BYTES) {
+      throw new Error(`Request body exceeds ${String(MAX_BODY_BYTES)} bytes`);
+    }
+    chunks.push(buffer);
+  }
+  if (chunks.length === 0) return undefined;
+  return JSON.parse(Buffer.concat(chunks).toString("utf-8")) as unknown;
+}
+
+function sendJson(res: http.ServerResponse, statusCode: number, body: Record<string, unknown>): void {
+  if (res.headersSent) return;
+  res.writeHead(statusCode, { "content-type": "application/json" });
+  res.end(JSON.stringify(body));
+}
+
+function sendJsonRpcError(res: http.ServerResponse, code: number, message: string): void {
+  sendJson(res, code === -32700 ? 400 : 500, {
+    jsonrpc: "2.0",
+    error: { code, message },
+    id: null,
+  });
+}
+
+async function removeSessionDirectory(sessionGraftDir: string): Promise<void> {
+  try {
+    await fs.rm(sessionGraftDir, { recursive: true, force: true });
+  } catch (error: unknown) {
+    const code = error instanceof Error && "code" in error ? (error as NodeJS.ErrnoException).code : undefined;
+    if (code !== "ENOENT") {
+      console.error(`[graft] failed to remove session directory ${sessionGraftDir}: ${String(error)}`);
+    }
+  }
+}
+
+async function createDaemonSession(
+  newSessionId: string,
+  options: CreateDaemonSessionHostOptions,
+  sessions: Map<string, DaemonSession>,
+): Promise<DaemonSession> {
+  const sessionGraftDir = path.join(options.graftDir, "sessions", newSessionId);
+  await ensurePrivateDirectory(sessionGraftDir);
+  const transport = new StreamableHTTPServerTransport({
+    sessionIdGenerator: () => newSessionId,
+  });
+  const server = createGraftServer({
+    mode: "daemon",
+    sessionId: newSessionId,
+    graftDir: sessionGraftDir,
+    warpPool: options.warpPool,
+    daemonControlPlane: options.controlPlane,
+    daemonScheduler: options.daemonScheduler,
+    daemonWorkerPool: options.daemonWorkerPool,
+    daemonRuntime: () => ({
+      transport: options.transportKind,
+      sameUserOnly: true,
+      socketPath: options.socketPath,
+      mcpPath: options.mcpPath,
+      healthPath: options.healthPath,
+      activeWarpRepos: options.warpPool.size(),
+      startedAt: options.startedAt,
+    }),
+    monitorRuntime: options.monitorRuntime,
+    ...(options.env !== undefined ? { env: options.env } : {}),
+    ...(options.runCapture !== undefined ? { runCapture: options.runCapture } : {}),
+    ...(options.runtimeObservability !== undefined
+      ? { runtimeObservability: options.runtimeObservability }
+      : {}),
+  });
+  const session: DaemonSession = {
+    id: newSessionId,
+    graftDir: sessionGraftDir,
+    transport,
+    server,
+  };
+  transport.onclose = () => {
+    sessions.delete(newSessionId);
+    options.controlPlane.unregisterTransport(newSessionId);
+    void removeSessionDirectory(sessionGraftDir);
+  };
+  transport.onerror = () => {
+    sessions.delete(newSessionId);
+    options.controlPlane.unregisterTransport(newSessionId);
+    void removeSessionDirectory(sessionGraftDir);
+  };
+  sessions.set(newSessionId, session);
+  options.controlPlane.registerTransport(
+    newSessionId,
+    () => server.getWorkspaceStatus(),
+    () => server.getRuntimeCausalContext(),
+  );
+  await server.getMcpServer().connect(transport as Transport);
+  return session;
+}
+
+export function createDaemonSessionHost(options: CreateDaemonSessionHostOptions): DaemonSessionHost {
+  const sessions = new Map<string, DaemonSession>();
+
+  return {
+    async handleRequest(req, res): Promise<void> {
+      try {
+        const url = new URL(req.url ?? "/", "http://localhost");
+        if (req.method === "GET" && url.pathname === options.healthPath) {
+          sendJson(res, 200, { ...options.getHealthStatus() });
+          return;
+        }
+
+        if (url.pathname !== options.mcpPath) {
+          sendJson(res, 404, { error: "Not found" });
+          return;
+        }
+
+        const sessionId = getHeader(req, "mcp-session-id");
+
+        if (req.method === "POST") {
+          const parsedBody = await readJsonBody(req);
+          let session = sessionId !== undefined ? sessions.get(sessionId) : undefined;
+          if (session === undefined) {
+            if (sessionId !== undefined) {
+              sendJsonRpcError(res, -32000, `Unknown MCP session: ${sessionId}`);
+              return;
+            }
+            if (!isInitializeRequest(parsedBody)) {
+              sendJsonRpcError(res, -32000, "Initialization requests must start a daemon session");
+              return;
+            }
+            session = await createDaemonSession(crypto.randomUUID(), options, sessions);
+          }
+
+          options.controlPlane.touchTransport(session.id);
+          await session.transport.handleRequest(req, res, parsedBody);
+          return;
+        }
+
+        if (req.method === "GET" || req.method === "DELETE") {
+          if (sessionId === undefined) {
+            sendJson(res, 400, { error: "Missing MCP session header" });
+            return;
+          }
+          const session = sessions.get(sessionId);
+          if (session === undefined) {
+            sendJson(res, 404, { error: `Unknown MCP session: ${sessionId}` });
+            return;
+          }
+          options.controlPlane.touchTransport(session.id);
+          await session.transport.handleRequest(req, res);
+          return;
+        }
+
+        sendJson(res, 405, { error: "Method not allowed" });
+      } catch (error) {
+        if (error instanceof SyntaxError) {
+          sendJsonRpcError(res, -32700, "Invalid JSON");
+          return;
+        }
+        sendJsonRpcError(res, -32603, error instanceof Error ? error.message : String(error));
+      }
+    },
+
+    async close(): Promise<void> {
+      for (const session of [...sessions.values()]) {
+        options.controlPlane.unregisterTransport(session.id);
+        await session.transport.close().catch(() => {
+          return undefined;
+        });
+        await removeSessionDirectory(session.graftDir);
+      }
+      sessions.clear();
+    },
+  };
+}

package/src/mcp/daemon-stdio-bridge.ts

@@ -0,0 +1,260 @@
+import * as http from "node:http";
+import { spawn, type ChildProcess } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { dirname, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import {
+  isInitializeRequest,
+  isJSONRPCRequest,
+  type JSONRPCMessage,
+  type JSONRPCRequest,
+} from "@modelcontextprotocol/sdk/types.js";
+import { defaultDaemonRoot, resolveSocketPath } from "./daemon-bootstrap.js";
+
+const DEFAULT_READY_TIMEOUT_MS = 5_000;
+const DEFAULT_POLL_INTERVAL_MS = 100;
+const MCP_PATH = "/mcp";
+const HEALTH_PATH = "/healthz";
+
+function toNodeHeaders(headers: Headers): http.OutgoingHttpHeaders {
+  const outgoing: http.OutgoingHttpHeaders = {};
+  for (const [name, value] of headers.entries()) {
+    outgoing[name] = value;
+  }
+  return outgoing;
+}
+
+function toResponseHeaders(headers: http.IncomingHttpHeaders): Headers {
+  const responseHeaders = new Headers();
+  for (const [name, value] of Object.entries(headers)) {
+    if (value === undefined) continue;
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        responseHeaders.append(name, item);
+      }
+      continue;
+    }
+    responseHeaders.append(name, value);
+  }
+  return responseHeaders;
+}
+
+export function createLocalSocketFetch(socketPath: string): typeof fetch {
+  return async (input: string | URL | Request, init?: RequestInit): Promise<Response> => {
+    const request = input instanceof Request ? input : new Request(input, init);
+    const url = new URL(request.url);
+    const body = request.body === null ? undefined : Buffer.from(await request.arrayBuffer());
+
+    return await new Promise<Response>((resolvePromise, reject) => {
+      const req = http.request({
+        socketPath,
+        path: `${url.pathname}${url.search}`,
+        method: request.method,
+        headers: toNodeHeaders(request.headers),
+        signal: request.signal,
+      }, (res) => {
+        const bodyStream = Readable.toWeb(res) as ReadableStream<Uint8Array>;
+        resolvePromise(new Response(bodyStream, {
+          status: res.statusCode ?? 0,
+          statusText: res.statusMessage ?? "",
+          headers: toResponseHeaders(res.headers),
+        }));
+      });
+      req.once("error", reject);
+      if (body !== undefined && body.byteLength > 0) {
+        req.write(body);
+      }
+      req.end();
+    });
+  };
+}
+
+async function isDaemonHealthy(socketPath: string): Promise<boolean> {
+  try {
+    const response = await createLocalSocketFetch(socketPath)(
+      new URL(`http://graft${HEALTH_PATH}`),
+      {
+        method: "GET",
+        headers: {
+          accept: "application/json",
+        },
+      },
+    );
+    await response.arrayBuffer();
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+function resolveGraftBinPath(): string {
+  return resolve(dirname(fileURLToPath(import.meta.url)), "../../bin/graft.js");
+}
+
+function spawnDaemonProcess(socketPath: string): ChildProcess {
+  const child = spawn(
+    process.execPath,
+    [resolveGraftBinPath(), "daemon", "--socket", socketPath],
+    {
+      detached: true,
+      stdio: ["ignore", "ignore", "ignore"],
+      windowsHide: true,
+      env: { ...process.env },
+    },
+  );
+  child.unref();
+  return child;
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolvePromise) => {
+    setTimeout(resolvePromise, ms);
+  });
+}
+
+export interface EnsureDaemonReadyOptions {
+  socketPath?: string;
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+  spawnIfMissing?: boolean;
+  spawnDaemon?: ((socketPath: string) => ChildProcess | undefined) | undefined;
+  healthCheck?: ((socketPath: string) => Promise<boolean>) | undefined;
+}
+
+export async function ensureDaemonReady(options: EnsureDaemonReadyOptions = {}): Promise<string> {
+  const socketPath = resolveSocketPath(options.socketPath, defaultDaemonRoot());
+  const timeoutMs = options.timeoutMs ?? DEFAULT_READY_TIMEOUT_MS;
+  const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+  const healthCheck = options.healthCheck ?? isDaemonHealthy;
+
+  if (await healthCheck(socketPath)) {
+    return socketPath;
+  }
+
+  if (options.spawnIfMissing === false) {
+    throw new Error(
+      `No graft daemon is listening on ${socketPath}. Start it with \`graft daemon --socket ${socketPath}\` or use repo-local \`graft serve\`.`,
+    );
+  }
+
+  let spawnError: unknown;
+  try {
+    (options.spawnDaemon ?? spawnDaemonProcess)(socketPath);
+  } catch (error) {
+    spawnError = error;
+  }
+
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() <= deadline) {
+    if (await healthCheck(socketPath)) {
+      return socketPath;
+    }
+    await delay(pollIntervalMs);
+  }
+
+  const suffix = spawnError instanceof Error
+    ? ` Auto-start failed: ${spawnError.message}`
+    : "";
+  throw new Error(`Timed out waiting for graft daemon readiness on ${socketPath}.${suffix}`);
+}
+
+function internalErrorResponse(
+  message: JSONRPCRequest,
+  error: unknown,
+): JSONRPCMessage {
+  return {
+    jsonrpc: "2.0",
+    id: message.id,
+    error: {
+      code: -32603,
+      message: error instanceof Error ? error.message : String(error),
+    },
+  };
+}
+
+export interface StartDaemonBackedStdioBridgeOptions {
+  socketPath?: string;
+  ensureReady?: ((options?: EnsureDaemonReadyOptions) => Promise<string>) | undefined;
+}
+
+export async function startDaemonBackedStdioBridge(
+  options: StartDaemonBackedStdioBridgeOptions = {},
+): Promise<void> {
+  const socketPath = await (options.ensureReady ?? ensureDaemonReady)({
+    ...(options.socketPath !== undefined ? { socketPath: options.socketPath } : {}),
+  });
+  const stdioTransport = new StdioServerTransport();
+  const daemonTransport = new StreamableHTTPClientTransport(
+    new URL(`http://graft${MCP_PATH}`),
+    { fetch: createLocalSocketFetch(socketPath) },
+  );
+
+  let closed = false;
+  const close = async (): Promise<void> => {
+    if (closed) return;
+    closed = true;
+    process.stdin.off("end", onStdinEnd);
+    process.stdin.off("close", onStdinEnd);
+    process.off("SIGINT", onSignal);
+    process.off("SIGTERM", onSignal);
+    await daemonTransport.terminateSession().catch(() => {
+      return undefined;
+    });
+    await daemonTransport.close().catch(() => {
+      return undefined;
+    });
+    await stdioTransport.close().catch(() => {
+      return undefined;
+    });
+  };
+
+  const onSignal = (): void => {
+    void close().finally(() => {
+      process.exitCode = process.exitCode ?? 0;
+    });
+  };
+  const onStdinEnd = (): void => {
+    void close();
+  };
+
+  stdioTransport.onmessage = (message): void => {
+    if (isInitializeRequest(message)) {
+      daemonTransport.setProtocolVersion(message.params.protocolVersion);
+    }
+    void daemonTransport.send(message).catch(async (error: unknown) => {
+      if (!isJSONRPCRequest(message)) {
+        console.error(error);
+        return;
+      }
+      await stdioTransport.send(internalErrorResponse(message, error)).catch(() => {
+        return undefined;
+      });
+    });
+  };
+  stdioTransport.onerror = (error): void => {
+    console.error(error);
+    void close();
+  };
+  daemonTransport.onmessage = (message): void => {
+    void stdioTransport.send(message).catch((error: unknown) => {
+      console.error(error);
+      void close();
+    });
+  };
+  daemonTransport.onerror = (error): void => {
+    console.error(error);
+  };
+  daemonTransport.onclose = (): void => {
+    void close();
+  };
+
+  process.stdin.on("end", onStdinEnd);
+  process.stdin.on("close", onStdinEnd);
+  process.once("SIGINT", onSignal);
+  process.once("SIGTERM", onSignal);
+
+  await daemonTransport.start();
+  await stdioTransport.start();
+}

package/src/mcp/daemon-worker-child-pool.ts

@@ -0,0 +1,213 @@
+import * as crypto from "node:crypto";
+import { fork, type ChildProcess } from "node:child_process";
+import * as os from "node:os";
+import { fileURLToPath } from "node:url";
+import type { MonitorTickWorkerJob, MonitorTickWorkerResult } from "./monitor-tick-job.js";
+import type { RepoToolWorkerJob, RepoToolWorkerResult } from "./repo-tool-job.js";
+import type {
+  ActiveWorkerTask,
+  ChildProcessDaemonWorkerPoolOptions,
+  DaemonWorkerCounts,
+  DaemonWorkerPool,
+  QueuedWorkerTask,
+  WorkerMessage,
+  WorkerState,
+} from "./daemon-worker-types.js";
+
+function defaultProcessPoolSize(): number {
+  const parallelism = os.availableParallelism();
+  return Math.max(1, Math.min(4, parallelism > 1 ? parallelism - 1 : 1));
+}
+
+function normalizeProcessPoolSize(value: number | undefined): number {
+  if (value === undefined) return defaultProcessPoolSize();
+  const normalized = Math.trunc(value);
+  if (!Number.isFinite(normalized) || normalized <= 0) {
+    throw new Error("worker pool size must be a positive integer");
+  }
+  return normalized;
+}
+
+function daemonWorkerProcessPath(): string {
+  return fileURLToPath(new URL("./daemon-worker-process.ts", import.meta.url));
+}
+
+async function stopChild(child: ChildProcess): Promise<void> {
+  if (child.exitCode !== null || child.signalCode !== null) {
+    return;
+  }
+  child.kill();
+  await new Promise<void>((resolve) => {
+    child.once("exit", () => {
+      resolve();
+    });
+  });
+}
+
+export class ChildProcessDaemonWorkerPool implements DaemonWorkerPool {
+  private readonly size: number;
+  private readonly workers = new Map<string, WorkerState>();
+  private readonly queue: QueuedWorkerTask[] = [];
+  private completedTasks = 0;
+  private failedTasks = 0;
+  private closing = false;
+
+  constructor(options: ChildProcessDaemonWorkerPoolOptions = {}) {
+    this.size = normalizeProcessPoolSize(options.size);
+    for (let index = 0; index < this.size; index++) {
+      this.spawnWorker();
+    }
+  }
+
+  getCounts(): DaemonWorkerCounts {
+    const busyWorkers = [...this.workers.values()].filter((workerState) => workerState.task !== null).length;
+    return {
+      mode: "child_processes",
+      totalWorkers: this.workers.size,
+      busyWorkers,
+      idleWorkers: this.workers.size - busyWorkers,
+      queuedTasks: this.queue.length,
+      completedTasks: this.completedTasks,
+      failedTasks: this.failedTasks,
+    };
+  }
+
+  runMonitorTick(job: MonitorTickWorkerJob): Promise<MonitorTickWorkerResult> {
+    if (this.closing) {
+      return Promise.reject(new Error("daemon worker pool is closing"));
+    }
+    return new Promise<MonitorTickWorkerResult>((resolve, reject) => {
+      this.queue.push({
+        requestId: crypto.randomUUID(),
+        kind: "monitor_tick",
+        job,
+        resolve,
+        reject,
+      });
+      this.dispatch();
+    });
+  }
+
+  runRepoTool(job: RepoToolWorkerJob): Promise<RepoToolWorkerResult> {
+    if (this.closing) {
+      return Promise.reject(new Error("daemon worker pool is closing"));
+    }
+    return new Promise<RepoToolWorkerResult>((resolve, reject) => {
+      this.queue.push({
+        requestId: crypto.randomUUID(),
+        kind: "repo_tool",
+        job,
+        resolve,
+        reject,
+      });
+      this.dispatch();
+    });
+  }
+
+  async close(): Promise<void> {
+    if (this.closing) return;
+    this.closing = true;
+    const pending = [...this.queue];
+    this.queue.length = 0;
+    for (const task of pending) {
+      task.reject(new Error("daemon worker pool closed before task execution"));
+    }
+    const active = [...this.workers.values()].flatMap((workerState) => {
+      return workerState.task === null ? [] : [workerState.task];
+    });
+    for (const task of active) {
+      task.reject(new Error("daemon worker pool closed during task execution"));
+    }
+    await Promise.all([...this.workers.values()].map(async (workerState) => {
+      await stopChild(workerState.child);
+    }));
+    this.workers.clear();
+  }
+
+  private spawnWorker(): void {
+    const workerId = crypto.randomUUID();
+    const child = fork(daemonWorkerProcessPath(), [], {
+      cwd: process.cwd(),
+      execArgv: ["--import", "tsx"],
+      stdio: ["ignore", "ignore", "ignore", "ipc"],
+    });
+    const workerState: WorkerState = {
+      workerId,
+      child,
+      task: null,
+    };
+    child.on("message", (message: WorkerMessage) => {
+      this.handleWorkerMessage(workerState, message);
+    });
+    child.on("error", (error: Error) => {
+      this.handleWorkerFailure(workerState, error);
+    });
+    child.on("exit", (code) => {
+      this.handleWorkerExit(workerState, code ?? 0);
+    });
+    this.workers.set(workerId, workerState);
+    this.dispatch();
+  }
+
+  private dispatch(): void {
+    for (const workerState of this.workers.values()) {
+      if (workerState.task !== null) continue;
+      const next = this.queue.shift();
+      if (next === undefined) {
+        return;
+      }
+      const activeTask: ActiveWorkerTask = {
+        ...next,
+        workerId: workerState.workerId,
+      };
+      workerState.task = activeTask;
+      workerState.child.send({
+        requestId: activeTask.requestId,
+        kind: activeTask.kind,
+        job: activeTask.job,
+      });
+    }
+  }
+
+  private handleWorkerMessage(workerState: WorkerState, message: WorkerMessage): void {
+    const task = workerState.task;
+    if (task?.requestId !== message.requestId) {
+      return;
+    }
+    workerState.task = null;
+    if (message.ok) {
+      this.completedTasks++;
+      if (task.kind === "monitor_tick") {
+        task.resolve(message.result as MonitorTickWorkerResult);
+      } else {
+        task.resolve(message.result as RepoToolWorkerResult);
+      }
+    } else {
+      this.failedTasks++;
+      task.reject(new Error(message.error));
+    }
+    this.dispatch();
+  }
+
+  private handleWorkerFailure(workerState: WorkerState, error: Error): void {
+    const task = workerState.task;
+    workerState.task = null;
+    if (task !== null) {
+      this.failedTasks++;
+      task.reject(error);
+    }
+  }
+
+  private handleWorkerExit(workerState: WorkerState, code: number): void {
+    const task = workerState.task;
+    workerState.task = null;
+    this.workers.delete(workerState.workerId);
+    if (task !== null) {
+      this.failedTasks++;
+      task.reject(new Error(`daemon worker exited with code ${String(code)}`));
+    }
+    if (!this.closing) {
+      this.spawnWorker();
+    }
+  }
+}