@flyingrobots/graft 0.4.0 → 0.6.0

package/src/mcp/daemon-job-scheduler.ts

@@ -0,0 +1,279 @@
+import * as crypto from "node:crypto";
+
+export type DaemonJobKind = "repo_tool" | "persistent_monitor";
+export type DaemonJobPriority = "interactive" | "background";
+export type DaemonJobState = "queued" | "running";
+
+export interface DaemonJobRequest {
+  readonly sessionId: string | null;
+  readonly sliceId: string | null;
+  readonly repoId: string;
+  readonly worktreeId: string | null;
+  readonly tool: string;
+  readonly kind: DaemonJobKind;
+  readonly priority: DaemonJobPriority;
+  readonly writerId: string;
+}
+
+export interface DaemonJobView extends DaemonJobRequest {
+  readonly jobId: string;
+  readonly state: DaemonJobState;
+  readonly enqueuedAt: string;
+  readonly startedAt: string | null;
+  readonly waitMs: number;
+}
+
+export interface DaemonSchedulerCounts {
+  readonly maxConcurrentJobs: number;
+  readonly activeJobs: number;
+  readonly queuedJobs: number;
+  readonly interactiveQueuedJobs: number;
+  readonly backgroundQueuedJobs: number;
+  readonly activeWriterLanes: number;
+  readonly queuedWriterLanes: number;
+  readonly completedJobs: number;
+  readonly failedJobs: number;
+  readonly longestQueuedWaitMs: number;
+}
+
+export interface DaemonJobSchedulerOptions {
+  readonly maxConcurrentJobs?: number | undefined;
+}
+
+interface ScheduledJob extends DaemonJobRequest {
+  readonly jobId: string;
+  readonly enqueuedAtMs: number;
+  readonly enqueuedAt: string;
+  startedAtMs: number | null;
+  startedAt: string | null;
+  readonly run: () => Promise<unknown>;
+  readonly resolve: (value: unknown) => void;
+  readonly reject: (reason?: unknown) => void;
+}
+
+const DEFAULT_MAX_CONCURRENT_JOBS = 2;
+
+export const ZERO_SCHEDULER_COUNTS: DaemonSchedulerCounts = Object.freeze({
+  maxConcurrentJobs: DEFAULT_MAX_CONCURRENT_JOBS,
+  activeJobs: 0,
+  queuedJobs: 0,
+  interactiveQueuedJobs: 0,
+  backgroundQueuedJobs: 0,
+  activeWriterLanes: 0,
+  queuedWriterLanes: 0,
+  completedJobs: 0,
+  failedJobs: 0,
+  longestQueuedWaitMs: 0,
+});
+
+function normalizeMaxConcurrentJobs(value: number | undefined): number {
+  if (value === undefined) return DEFAULT_MAX_CONCURRENT_JOBS;
+  const normalized = Math.trunc(value);
+  if (!Number.isFinite(normalized) || normalized <= 0) {
+    throw new Error("maxConcurrentJobs must be a positive integer");
+  }
+  return normalized;
+}
+
+function toIso(ms: number): string {
+  return new Date(ms).toISOString();
+}
+
+function compareLaneKeys(left: string, right: string): number {
+  return left.localeCompare(right);
+}
+
+function buildWriterLaneKey(job: Pick<DaemonJobRequest, "repoId" | "writerId">): string {
+  return `${job.repoId}:${job.writerId}`;
+}
+
+function toJobView(job: ScheduledJob, nowMs: number, state: DaemonJobState): DaemonJobView {
+  return {
+    jobId: job.jobId,
+    sessionId: job.sessionId,
+    sliceId: job.sliceId,
+    repoId: job.repoId,
+    worktreeId: job.worktreeId,
+    tool: job.tool,
+    kind: job.kind,
+    priority: job.priority,
+    writerId: job.writerId,
+    state,
+    enqueuedAt: job.enqueuedAt,
+    startedAt: job.startedAt,
+    waitMs: (job.startedAtMs ?? nowMs) - job.enqueuedAtMs,
+  };
+}
+
+export class DaemonJobScheduler {
+  private readonly maxConcurrentJobs: number;
+  private readonly interactiveQueues = new Map<string, ScheduledJob[]>();
+  private readonly backgroundQueues = new Map<string, ScheduledJob[]>();
+  private readonly running = new Map<string, ScheduledJob>();
+  private completedJobs = 0;
+  private failedJobs = 0;
+  private lastInteractiveLaneKey: string | null = null;
+  private lastBackgroundLaneKey: string | null = null;
+
+  constructor(options: DaemonJobSchedulerOptions = {}) {
+    this.maxConcurrentJobs = normalizeMaxConcurrentJobs(options.maxConcurrentJobs);
+  }
+
+  enqueue<T>(request: DaemonJobRequest, run: () => Promise<T>): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      const enqueuedAtMs = Date.now();
+      const job: ScheduledJob = {
+        ...request,
+        jobId: crypto.randomUUID(),
+        enqueuedAtMs,
+        enqueuedAt: toIso(enqueuedAtMs),
+        startedAtMs: null,
+        startedAt: null,
+        run: () => run(),
+        resolve: (value) => {
+          resolve(value as T);
+        },
+        reject,
+      };
+      this.enqueueJob(job);
+      this.dispatch();
+    });
+  }
+
+  getCounts(nowMs = Date.now()): DaemonSchedulerCounts {
+    const queuedJobs = this.listQueuedJobs();
+    const interactiveQueuedJobs = queuedJobs.filter((job) => job.priority === "interactive").length;
+    const backgroundQueuedJobs = queuedJobs.length - interactiveQueuedJobs;
+    const longestQueuedWaitMs = queuedJobs.reduce((maxWait, job) => {
+      return Math.max(maxWait, nowMs - job.enqueuedAtMs);
+    }, 0);
+
+    return {
+      maxConcurrentJobs: this.maxConcurrentJobs,
+      activeJobs: this.running.size,
+      queuedJobs: queuedJobs.length,
+      interactiveQueuedJobs,
+      backgroundQueuedJobs,
+      activeWriterLanes: new Set([...this.running.values()].map((job) => buildWriterLaneKey(job))).size,
+      queuedWriterLanes: new Set(queuedJobs.map((job) => buildWriterLaneKey(job))).size,
+      completedJobs: this.completedJobs,
+      failedJobs: this.failedJobs,
+      longestQueuedWaitMs,
+    };
+  }
+
+  listJobs(nowMs = Date.now()): readonly DaemonJobView[] {
+    const runningJobs = [...this.running.values()]
+      .map((job) => toJobView(job, nowMs, "running"))
+      .sort((left, right) => {
+        return (left.startedAt ?? left.enqueuedAt).localeCompare(right.startedAt ?? right.enqueuedAt);
+      });
+    const queuedJobs = this.listQueuedJobs()
+      .map((job) => toJobView(job, nowMs, "queued"))
+      .sort((left, right) => left.enqueuedAt.localeCompare(right.enqueuedAt));
+    return [...runningJobs, ...queuedJobs];
+  }
+
+  private enqueueJob(job: ScheduledJob): void {
+    const queues = this.getQueues(job.priority);
+    const laneKey = buildWriterLaneKey(job);
+    const queue = queues.get(laneKey) ?? [];
+    queue.push(job);
+    queues.set(laneKey, queue);
+  }
+
+  private dispatch(): void {
+    while (this.running.size < this.maxConcurrentJobs) {
+      const nextJob = this.takeNextJob();
+      if (nextJob === null) {
+        return;
+      }
+      this.startJob(nextJob);
+    }
+  }
+
+  private startJob(job: ScheduledJob): void {
+    const startedAtMs = Date.now();
+    job.startedAtMs = startedAtMs;
+    job.startedAt = toIso(startedAtMs);
+    this.running.set(job.jobId, job);
+
+    void Promise.resolve()
+      .then(() => job.run())
+      .then((value) => {
+        this.running.delete(job.jobId);
+        this.completedJobs++;
+        job.resolve(value);
+        this.dispatch();
+      })
+      .catch((error: unknown) => {
+        this.running.delete(job.jobId);
+        this.failedJobs++;
+        job.reject(error);
+        this.dispatch();
+      });
+  }
+
+  private takeNextJob(): ScheduledJob | null {
+    return this.takeNextJobForPriority("interactive") ?? this.takeNextJobForPriority("background");
+  }
+
+  private takeNextJobForPriority(priority: DaemonJobPriority): ScheduledJob | null {
+    const queues = this.getQueues(priority);
+    return this.takeNextJobFromQueues(queues, priority);
+  }
+
+  private takeNextJobFromQueues(
+    queues: Map<string, ScheduledJob[]>,
+    priority: DaemonJobPriority,
+  ): ScheduledJob | null {
+    const laneKeys = [...queues.entries()]
+      .filter(([laneKey, queue]) => {
+        return queue.length > 0 && !this.hasRunningLane(laneKey);
+      })
+      .map(([laneKey]) => laneKey)
+      .sort(compareLaneKeys);
+
+    if (laneKeys.length === 0) {
+      return null;
+    }
+
+    const lastLaneKey = priority === "interactive" ? this.lastInteractiveLaneKey : this.lastBackgroundLaneKey;
+    const lastIndex = lastLaneKey === null ? -1 : laneKeys.indexOf(lastLaneKey);
+    const startIndex = lastIndex >= 0 ? (lastIndex + 1) % laneKeys.length : 0;
+
+    for (let offset = 0; offset < laneKeys.length; offset++) {
+      const laneKey = laneKeys[(startIndex + offset) % laneKeys.length];
+      if (laneKey === undefined) continue;
+      const queue = queues.get(laneKey);
+      const job = queue?.shift();
+      if (job === undefined) continue;
+      if (queue?.length === 0) {
+        queues.delete(laneKey);
+      }
+      if (priority === "interactive") {
+        this.lastInteractiveLaneKey = laneKey;
+      } else {
+        this.lastBackgroundLaneKey = laneKey;
+      }
+      return job;
+    }
+
+    return null;
+  }
+
+  private listQueuedJobs(): ScheduledJob[] {
+    return [
+      ...this.interactiveQueues.values(),
+      ...this.backgroundQueues.values(),
+    ].flatMap((queue) => [...queue]);
+  }
+
+  private hasRunningLane(laneKey: string): boolean {
+    return [...this.running.values()].some((job) => buildWriterLaneKey(job) === laneKey);
+  }
+
+  private getQueues(priority: DaemonJobPriority): Map<string, ScheduledJob[]> {
+    return priority === "interactive" ? this.interactiveQueues : this.backgroundQueues;
+  }
+}

package/src/mcp/daemon-repos.ts

@@ -0,0 +1,99 @@
+import type {
+  AuthorizedWorkspaceRecord,
+  DaemonSessionView,
+} from "./daemon-control-plane.js";
+import type { MonitorStatusView } from "./persistent-monitor-runtime.js";
+
+import { buildResult, normalizeFilter, sortAuthorizedWorkspaces } from "./repo-overview/filter.js";
+import { toRepoView } from "./repo-overview/view-projection.js";
+
+// ---------------------------------------------------------------------------
+// Barrel re-exports — keep every public type available from this path
+// ---------------------------------------------------------------------------
+
+export type {
+  DaemonRepoFilter,
+  DaemonRepoListView,
+  DaemonRepoMonitorView,
+  DaemonRepoOverviewOptions,
+  DaemonRepoView,
+  DaemonRepoWorktreeView,
+} from "./repo-overview/types.js";
+
+import type {
+  DaemonRepoFilter,
+  DaemonRepoListView,
+  DaemonRepoOverviewOptions,
+} from "./repo-overview/types.js";
+
+// ---------------------------------------------------------------------------
+// DaemonRepoOverview
+// ---------------------------------------------------------------------------
+
+export class DaemonRepoOverview {
+  constructor(private readonly options: DaemonRepoOverviewOptions) {}
+
+  async list(filter: DaemonRepoFilter = {}): Promise<DaemonRepoListView> {
+    const normalizedFilter = normalizeFilter(filter);
+    const resolvedRepoId = await this.resolveRepoIdFilter(normalizedFilter);
+    if (normalizedFilter.cwd !== undefined && resolvedRepoId === null) {
+      return buildResult(normalizedFilter, []);
+    }
+
+    const repoIdFilter = resolvedRepoId ?? normalizedFilter.repoId;
+    const [authorizedWorkspaces, monitors] = await Promise.all([
+      this.options.controlPlane.listAuthorizedWorkspaceRecords(),
+      this.options.monitorRuntime.listStatuses(),
+    ]);
+    const sessions = this.options.controlPlane.listTransports();
+    const visibleWorkspaces = authorizedWorkspaces.filter((workspace) => {
+      return repoIdFilter === undefined || workspace.repoId === repoIdFilter;
+    });
+    if (visibleWorkspaces.length === 0) {
+      return buildResult(normalizedFilter, []);
+    }
+
+    const workspacesByRepo = new Map<string, AuthorizedWorkspaceRecord[]>();
+    for (const workspace of visibleWorkspaces) {
+      const bucket = workspacesByRepo.get(workspace.repoId) ?? [];
+      bucket.push(workspace);
+      workspacesByRepo.set(workspace.repoId, bucket);
+    }
+
+    const sessionsByRepo = new Map<string, DaemonSessionView[]>();
+    for (const session of sessions) {
+      if (session.repoId === null) continue;
+      if (repoIdFilter !== undefined && session.repoId !== repoIdFilter) continue;
+      const bucket = sessionsByRepo.get(session.repoId) ?? [];
+      bucket.push(session);
+      sessionsByRepo.set(session.repoId, bucket);
+    }
+
+    const monitorsByRepo = new Map<string, MonitorStatusView>();
+    for (const status of monitors) {
+      if (repoIdFilter !== undefined && status.repoId !== repoIdFilter) continue;
+      monitorsByRepo.set(status.repoId, status);
+    }
+
+    const repos = [...workspacesByRepo.entries()]
+      .map(([repoId, repoWorkspaces]) => {
+        const sortedWorkspaces = sortAuthorizedWorkspaces(repoWorkspaces);
+        const repoSessions = sessionsByRepo.get(repoId) ?? [];
+        const monitor = monitorsByRepo.get(repoId) ?? null;
+        return toRepoView(repoId, sortedWorkspaces, repoSessions, monitor);
+      })
+      .sort((left, right) => left.gitCommonDir.localeCompare(right.gitCommonDir));
+
+    return buildResult(normalizedFilter, repos);
+  }
+
+  private async resolveRepoIdFilter(filter: DaemonRepoFilter): Promise<string | null | undefined> {
+    if (filter.cwd === undefined) return undefined;
+    const workspace = await this.options.controlPlane.getAuthorizedWorkspace({ cwd: filter.cwd });
+    if (workspace === null) return null;
+    if (filter.repoId !== undefined && workspace.repoId !== filter.repoId) {
+      return null;
+    }
+    return workspace.repoId;
+  }
+}

package/src/mcp/daemon-server.ts

@@ -0,0 +1,165 @@
+import * as fs from "node:fs/promises";
+import * as http from "node:http";
+import * as path from "node:path";
+import { CanonicalJsonCodec } from "../adapters/canonical-json.js";
+import { nodeFs } from "../adapters/node-fs.js";
+import { nodeGit } from "../adapters/node-git.js";
+import { DaemonControlPlane, type DaemonStatusView } from "./daemon-control-plane.js";
+import { DaemonJobScheduler } from "./daemon-job-scheduler.js";
+import { ChildProcessDaemonWorkerPool } from "./daemon-worker-pool.js";
+import { PersistentMonitorRuntime } from "./persistent-monitor-runtime.js";
+import { InMemoryWarpPool } from "./warp-pool.js";
+import { openWarp } from "../warp/open.js";
+import type { RunCaptureConfig } from "./run-capture-config.js";
+import type { RuntimeObservabilityState } from "./runtime-observability.js";
+import {
+  closeHttpServer,
+  defaultDaemonRoot,
+  ensurePrivateDirectory,
+  isNamedPipePath,
+  prepareSocketPath,
+  resolveSocketPath,
+  tightenSocketPermissions,
+} from "./daemon-bootstrap.js";
+import { createDaemonSessionHost } from "./daemon-session-host.js";
+
+const HEALTH_PATH = "/healthz";
+const MCP_PATH = "/mcp";
+
+export type DaemonHealthStatus = DaemonStatusView;
+
+export interface StartDaemonServerOptions {
+  readonly socketPath?: string | undefined;
+  readonly graftDir?: string | undefined;
+  readonly env?: Readonly<Record<string, string | undefined>> | undefined;
+  readonly runCapture?: Partial<RunCaptureConfig> | undefined;
+  readonly runtimeObservability?: Partial<RuntimeObservabilityState> | undefined;
+}
+
+export interface GraftDaemonServer {
+  readonly socketPath: string;
+  readonly healthPath: typeof HEALTH_PATH;
+  readonly mcpPath: typeof MCP_PATH;
+  close(): Promise<void>;
+  getHealthStatus(): DaemonHealthStatus;
+}
+
+export async function startDaemonServer(options: StartDaemonServerOptions = {}): Promise<GraftDaemonServer> {
+  const graftDir = path.resolve(options.graftDir ?? defaultDaemonRoot());
+  const socketPath = resolveSocketPath(options.socketPath, graftDir);
+  const warpPool = new InMemoryWarpPool((cwd) => openWarp({ cwd }));
+  const controlPlane = new DaemonControlPlane({
+    fs: nodeFs,
+    codec: new CanonicalJsonCodec(),
+    git: nodeGit,
+    graftDir,
+  });
+  const daemonScheduler = new DaemonJobScheduler();
+  const daemonWorkerPool = new ChildProcessDaemonWorkerPool();
+  const monitorRuntime = new PersistentMonitorRuntime({
+    fs: nodeFs,
+    codec: new CanonicalJsonCodec(),
+    git: nodeGit,
+    graftDir,
+    controlPlane,
+    scheduler: daemonScheduler,
+    workerPool: daemonWorkerPool,
+  });
+  const startedAt = new Date().toISOString();
+  const transportKind = isNamedPipePath(socketPath) ? "named_pipe" : "unix_socket";
+
+  const getHealthStatus = (): DaemonHealthStatus => {
+    return controlPlane.getStatus({
+      transport: transportKind,
+      sameUserOnly: true,
+      socketPath,
+      mcpPath: MCP_PATH,
+      healthPath: HEALTH_PATH,
+      activeWarpRepos: warpPool.size(),
+      startedAt,
+    }, monitorRuntime.getCounts(), daemonScheduler.getCounts(), daemonWorkerPool.getCounts());
+  };
+
+  await ensurePrivateDirectory(graftDir);
+  await ensurePrivateDirectory(path.join(graftDir, "sessions"));
+  await controlPlane.initialize();
+  await monitorRuntime.initialize();
+  await prepareSocketPath(socketPath);
+  const sessionHost = createDaemonSessionHost({
+    graftDir,
+    socketPath,
+    transportKind,
+    healthPath: HEALTH_PATH,
+    mcpPath: MCP_PATH,
+    startedAt,
+    warpPool,
+    controlPlane,
+    daemonScheduler,
+    daemonWorkerPool,
+    monitorRuntime,
+    getHealthStatus,
+    ...(options.env !== undefined ? { env: options.env } : {}),
+    ...(options.runCapture !== undefined ? { runCapture: options.runCapture } : {}),
+    ...(options.runtimeObservability !== undefined
+      ? { runtimeObservability: options.runtimeObservability }
+      : {}),
+  });
+
+  const httpServer = http.createServer((req, res) => {
+    void sessionHost.handleRequest(req, res);
+  });
+
+  await new Promise<void>((resolve, reject) => {
+    const onError = (error: Error): void => {
+      httpServer.off("listening", onListening);
+      reject(error);
+    };
+    const onListening = (): void => {
+      httpServer.off("error", onError);
+      resolve();
+    };
+    httpServer.once("error", onError);
+    httpServer.once("listening", onListening);
+    httpServer.listen(socketPath);
+  });
+  await tightenSocketPermissions(socketPath);
+
+  let closing: Promise<void> | null = null;
+
+  const shutdown = (): void => {
+    void daemon.close().finally(() => {
+      process.exitCode = process.exitCode ?? 0;
+    });
+  };
+
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+
+  const daemon: GraftDaemonServer = {
+    socketPath,
+    healthPath: HEALTH_PATH,
+    mcpPath: MCP_PATH,
+    getHealthStatus(): DaemonHealthStatus {
+      return getHealthStatus();
+    },
+    async close(): Promise<void> {
+      if (closing !== null) return closing;
+      closing = (async () => {
+        process.off("SIGINT", shutdown);
+        process.off("SIGTERM", shutdown);
+        await sessionHost.close();
+        await monitorRuntime.close();
+        await daemonWorkerPool.close();
+        await closeHttpServer(httpServer);
+        if (!isNamedPipePath(socketPath)) {
+          await fs.unlink(socketPath).catch(() => {
+            return undefined;
+          });
+        }
+      })();
+      return closing;
+    },
+  };
+
+  return daemon;
+}