@huynle/hub 0.6.0 → 0.7.1

package/packages/node/src/daemon.ts

@@ -0,0 +1,281 @@
+// The node daemon (design §6.3): one NodeConnection to the hub and the runtimes it offers. It serves the hub's
+// prepare → (start → stop) → collect → cleanup (and cancel) for each launch, reports progress frames, and holds a
+// slot per launch from prepare until cleanup (or a failed or cancelled prepare), sending a capacity frame whenever
+// use changes. prepare goes to the runtime the request names; every later op to the runtime the prepared workspace
+// names (workspace.runtime, absent = host). It announces the "sandbox" feature (start and stop) only when one of its
+// runtimes starts harnesses (openshell); the host runtime answers start and stop with started/stopped false.
+// It keeps launches in memory only: the later ops carry the prepared workspace, so a restarted node can still
+// serve them. There is no remote exec channel: these ops are all the hub can ask.
+import { randomUUID } from "node:crypto"
+import { arch as osArch, platform as osPlatform } from "node:os"
+import {
+  NODE_FEATURES,
+  NODE_MESSAGE_MAX,
+  preparedWorkspaceRuntime,
+  type NodeCapacity,
+  type NodeHello,
+  type NodeLaunchPhase,
+  type NodeLaunchReport,
+  type NodeRequest,
+  type NodeResultFor,
+  type NodeRuntimeInfo,
+  type PreparedWorkspace,
+} from "../../protocol/src/index.ts"
+import { RuntimeError, type Runtime } from "../../runtime-sdk/src/index.ts"
+import pkg from "../package.json" with { type: "json" }
+import type { NodeConfig, SpokeCredentials } from "./config.ts"
+import { NodeConnection, type NodeConnectionOptions, type NodeConnectionState } from "./connection.ts"
+import { HostRuntime } from "./host.ts"
+
+export const NODE_VERSION: string = pkg.version
+
+/** What the daemon needs of a connection (NodeConnection, or a fake in tests). */
+export interface ConnectionLike {
+  start(): void
+  stop(): void
+  send(frame: Parameters<NodeConnection["send"]>[0]): boolean
+}
+
+export interface NodeDaemonOptions {
+  readonly config: NodeConfig
+  readonly credentials: SpokeCredentials
+  /** Default: the host runtime over the config's roots. */
+  readonly runtimes?: readonly Runtime[]
+  readonly instanceID?: string
+  /** The os and arch labels (default this machine's). */
+  readonly platform?: { readonly os: string; readonly arch: string }
+  readonly connect?: (options: NodeConnectionOptions) => ConnectionLike
+  readonly onState?: (state: NodeConnectionState, error?: string) => void
+  /** Backends per runtime kind (e.g. openshell: the gateway's compute drivers), added to the hello's runtimes. */
+  readonly backends?: Readonly<Record<string, readonly string[]>>
+  /** Launches found at startup (an earlier run's sandboxes): reported in the hello, each holding a slot until cleanup. */
+  readonly recovered?: readonly NodeLaunchReport[]
+}
+
+interface Tracked {
+  phase: NodeLaunchPhase
+  directory?: string
+  sandbox?: string
+  abort?: AbortController
+}
+
+/** A message fit for a progress frame: one line of printable text, at most NODE_MESSAGE_MAX characters. */
+export function progressText(text: string): string | undefined {
+  const line = text.replace(/[\p{Cc}\p{Cf}\u2028\u2029]+/gu, " ").replace(/\s+/g, " ").trim()
+  return line ? line.slice(0, NODE_MESSAGE_MAX) : undefined
+}
+
+const errorText = (cause: unknown) => (cause instanceof Error ? cause.message : String(cause))
+
+export class NodeDaemon {
+  private readonly launches = new Map<string, Tracked>()
+  private readonly runtimes: readonly Runtime[]
+  private readonly instanceID: string
+  private readonly labels: Readonly<Record<string, string>>
+  private draining = false
+  private connection?: ConnectionLike
+
+  constructor(private readonly options: NodeDaemonOptions) {
+    this.runtimes = options.runtimes ?? [new HostRuntime({ roots: options.config.workspaceRoots })]
+    this.instanceID = options.instanceID ?? randomUUID()
+    const platform = options.platform ?? { os: osPlatform(), arch: osArch() }
+    this.labels = { os: platform.os, arch: platform.arch, ...options.config.labels }
+    for (const r of options.recovered ?? [])
+      this.launches.set(r.launchID, { phase: r.phase, ...(r.directory ? { directory: r.directory } : {}), ...(r.sandbox ? { sandbox: r.sandbox } : {}) })
+  }
+
+  start(): void {
+    if (this.connection) return
+    const connectionOptions: NodeConnectionOptions = {
+      server: this.options.credentials.server,
+      token: this.options.credentials.token,
+      hello: () => this.hello(),
+      handle: (req) => this.handle(req),
+      ...(this.options.onState ? { onState: this.options.onState } : {}),
+    }
+    this.connection = (this.options.connect ?? ((o) => new NodeConnection(o)))(connectionOptions)
+    this.connection.start()
+  }
+
+  stop(): void {
+    for (const launch of this.launches.values()) launch.abort?.abort()
+    this.connection?.stop()
+    this.connection = undefined
+  }
+
+  /** Takes no new launches (running ones finish); the hub learns it from a capacity frame. */
+  drain(): void {
+    this.draining = true
+    this.sendCapacity()
+  }
+
+  hello(): Omit<NodeHello, "t" | "v" | "role"> {
+    const features = NODE_FEATURES.filter((f) => f !== "sandbox" || this.servesSandbox())
+    return {
+      instanceID: this.instanceID,
+      nodeVersion: NODE_VERSION,
+      ...(features.length ? { features } : {}),
+      runtimes: this.runtimes.map((r) => this.runtimeInfo(r)),
+      labels: this.labels,
+      capacity: this.capacity(),
+      used: this.used(),
+      workspaceRoots: [...this.options.config.workspaceRoots],
+      draining: this.draining,
+      launches: [...this.launches].map(
+        ([launchID, t]): NodeLaunchReport => ({ launchID, phase: t.phase, ...(t.directory ? { directory: t.directory } : {}), ...(t.sandbox ? { sandbox: t.sandbox } : {}) }),
+      ),
+    }
+  }
+
+  async handle(req: NodeRequest): Promise<NodeResultFor[NodeRequest["op"]]> {
+    switch (req.op) {
+      case "prepare":
+        return this.prepare(req)
+      case "collect": {
+        const runtime = this.workspaceRuntime(req.workspace)
+        const before = this.launches.get(req.launchID)?.phase
+        this.phase(req.launchID, "collecting")
+        try {
+          const artifacts = await runtime.collect({ launchID: req.launchID, workspace: req.workspace })
+          this.phase(req.launchID, "collected")
+          return { artifacts }
+        } catch (cause) {
+          const tracked = this.launches.get(req.launchID)
+          if (tracked && before) tracked.phase = before
+          throw cause
+        }
+      }
+      case "cleanup": {
+        const runtime = this.workspaceRuntime(req.workspace)
+        this.phase(req.launchID, "cleaning")
+        try {
+          const result = await runtime.cleanup({ launchID: req.launchID, workspace: req.workspace }, req.policy)
+          this.progress(req.launchID, "cleaned", result.removed ? undefined : result.reason)
+          return result
+        } finally {
+          this.release(req.launchID)
+        }
+      }
+      case "cancel": {
+        const tracked = this.launches.get(req.launchID)
+        if (!tracked?.abort || tracked.phase !== "preparing") return { cancelled: false }
+        tracked.abort.abort()
+        return { cancelled: true }
+      }
+      case "start":
+        return this.startLaunch(req)
+      case "stop":
+        return this.stopLaunch(req)
+    }
+  }
+
+  private async startLaunch(req: Extract<NodeRequest, { op: "start" }>): Promise<NodeResultFor["start"]> {
+    const runtime = this.workspaceRuntime(req.workspace)
+    if (!runtime.start) return { started: false, reason: `the ${runtime.kind} runtime does not start harnesses (its sessions run in the machine's harness service)` }
+    const before = this.launches.get(req.launchID)?.phase
+    this.phase(req.launchID, "starting")
+    try {
+      const result = await runtime.start(req.workspace, { launchID: req.launchID, harness: req.harness, hub: req.hub, ...(req.env ? { env: req.env } : {}) }, {
+        progress: (message) => this.progress(req.launchID, "starting", message),
+      })
+      this.phase(req.launchID, "running")
+      return { started: result.started, ...(result.reason ? { reason: result.reason } : {}) }
+    } catch (cause) {
+      const tracked = this.launches.get(req.launchID)
+      if (tracked && before) tracked.phase = before
+      this.progress(req.launchID, "failed", errorText(cause))
+      throw cause
+    }
+  }
+
+  private async stopLaunch(req: Extract<NodeRequest, { op: "stop" }>): Promise<NodeResultFor["stop"]> {
+    const runtime = this.workspaceRuntime(req.workspace)
+    if (!runtime.stop) return { stopped: false, reason: `the ${runtime.kind} runtime runs no harness of its own to stop` }
+    const before = this.launches.get(req.launchID)?.phase
+    this.phase(req.launchID, "stopping")
+    try {
+      const result = await runtime.stop({ launchID: req.launchID, workspace: req.workspace }, req.mode ?? "graceful")
+      this.phase(req.launchID, result.stopped ? "stopped" : (before ?? "stopped"))
+      return { stopped: result.stopped, ...(result.reason ? { reason: result.reason } : {}) }
+    } catch (cause) {
+      const tracked = this.launches.get(req.launchID)
+      if (tracked && before) tracked.phase = before
+      throw cause
+    }
+  }
+
+  private async prepare(req: Extract<NodeRequest, { op: "prepare" }>): Promise<NodeResultFor["prepare"]> {
+    if (this.draining) throw new RuntimeError("this node is draining and takes no new launches")
+    if (this.launches.has(req.launchID)) throw new RuntimeError(`launch ${req.launchID} is already on this node`)
+    const runtime = this.runtimeFor(req.runtime.kind)
+    if (this.used().slots >= this.options.config.slots) throw new RuntimeError(`no free slot on this node (${this.options.config.slots} in use)`)
+    const abort = new AbortController()
+    this.launches.set(req.launchID, { phase: "preparing", abort })
+    this.sendCapacity()
+    this.progress(req.launchID, "preparing")
+    try {
+      const workspace = await runtime.prepare(
+        { launchID: req.launchID, harness: req.harness, runtime: req.runtime, workspace: req.workspace },
+        { signal: abort.signal, progress: (message) => this.progress(req.launchID, "preparing", message) },
+      )
+      if (abort.signal.aborted) throw new RuntimeError("prepare cancelled")
+      this.launches.set(req.launchID, { phase: "prepared", directory: workspace.directory, ...(workspace.runtime?.sandbox ? { sandbox: workspace.runtime.sandbox } : {}) })
+      this.progress(req.launchID, "prepared")
+      return { workspace }
+    } catch (cause) {
+      this.progress(req.launchID, abort.signal.aborted ? "cancelled" : "failed", abort.signal.aborted ? undefined : errorText(cause))
+      this.release(req.launchID)
+      throw abort.signal.aborted ? new RuntimeError("prepare cancelled") : cause
+    }
+  }
+
+  private runtimeFor(kind: string): Runtime {
+    const runtime = this.runtimes.find((r) => r.kind === kind)
+    if (!runtime) throw new RuntimeError(`runtime ${kind || "(none)"} is not available on this node`)
+    return runtime
+  }
+
+  /** The runtime that owns a prepared workspace: the one it names (absent: host). */
+  private workspaceRuntime(workspace: PreparedWorkspace): Runtime {
+    return this.runtimeFor(preparedWorkspaceRuntime(workspace).kind)
+  }
+
+  /** Whether a runtime here starts and stops harnesses itself (the "sandbox" feature). */
+  private servesSandbox(): boolean {
+    return this.runtimes.some((r) => typeof r.start === "function" && typeof r.stop === "function")
+  }
+
+  private runtimeInfo(runtime: Runtime): NodeRuntimeInfo {
+    const info = runtime.info()
+    const backends = this.options.backends?.[runtime.kind]
+    return backends?.length ? { ...info, backends: [...backends] } : info
+  }
+
+  private capacity(): NodeCapacity {
+    const { slots, memoryMB } = this.options.config
+    return { slots, ...(memoryMB !== undefined ? { memoryMB } : {}) }
+  }
+
+  private used(): NodeCapacity {
+    return { slots: this.launches.size }
+  }
+
+  private phase(launchID: string, phase: NodeLaunchPhase): void {
+    const tracked = this.launches.get(launchID)
+    if (tracked) tracked.phase = phase
+    this.progress(launchID, phase)
+  }
+
+  private release(launchID: string): void {
+    if (this.launches.delete(launchID)) this.sendCapacity()
+  }
+
+  private progress(launchID: string, phase: NodeLaunchPhase, message?: string): void {
+    const text = message === undefined ? undefined : progressText(message)
+    this.connection?.send({ t: "progress", launchID, phase, ...(text ? { message: text } : {}) })
+  }
+
+  private sendCapacity(): void {
+    this.connection?.send({ t: "capacity", capacity: this.capacity(), used: this.used(), draining: this.draining })
+  }
+}

package/packages/node/src/git.ts

@@ -0,0 +1,107 @@
+// The node's only way to run git (decision 10). Never a shell: spawn (execFile semantics, no shell) with an argv the caller builds with `--`
+// before paths and refs. The environment is fixed: no terminal or credential-manager prompts (a daemon has no one to
+// ask), only the https and ssh transports (never ext::, file:: or a local path as a remote), and none of the
+// inherited GIT_* variables that redirect a repository, its config or git's own programs. Every run has a time limit
+// and can be cancelled. node:child_process (not Bun.spawn) so the node also runs bundled for node.
+import { spawn } from "node:child_process"
+
+type Env = Readonly<Record<string, string | undefined>>
+
+/** Transports git may use (GIT_ALLOW_PROTOCOL). */
+export const ALLOWED_PROTOCOLS = "https:ssh"
+export const GIT_TIMEOUT_MS = 120_000
+const MAX_BUFFER = 16 * 1024 * 1024
+const STDERR_MAX = 600
+
+/** Inherited GIT_* names that only choose how ssh runs; every other GIT_* name is dropped. */
+const KEPT_GIT_VARS = new Set(["GIT_SSH", "GIT_SSH_COMMAND", "GIT_SSH_VARIANT"])
+
+export class GitError extends Error {
+  override readonly name = "GitError"
+}
+
+/** `base` without its GIT_* overrides, plus the hardened settings and then `overlay`. */
+export function gitEnv(base: Env = process.env, overlay: Env = {}): Record<string, string> {
+  const env: Record<string, string> = {}
+  for (const [name, value] of Object.entries(base)) {
+    if (value === undefined) continue
+    if (name.startsWith("GIT_") && !KEPT_GIT_VARS.has(name)) continue
+    env[name] = value
+  }
+  Object.assign(env, { GIT_TERMINAL_PROMPT: "0", GIT_ALLOW_PROTOCOL: ALLOWED_PROTOCOLS, GCM_INTERACTIVE: "never", LC_ALL: "C" })
+  for (const [name, value] of Object.entries(overlay)) if (value !== undefined) env[name] = value
+  return env
+}
+
+/** Removes credentials (userinfo) from URLs in git output. */
+export function redact(text: string): string {
+  return text.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, "$1***@")
+}
+
+export interface GitOptions {
+  readonly cwd: string
+  readonly timeoutMs?: number
+  readonly signal?: AbortSignal
+  /** The environment to start from (default process.env); GIT_* overrides are dropped from it. */
+  readonly baseEnv?: Env
+  /** Applied after the hardening (tests use it to allow the file transport). */
+  readonly env?: Env
+  /** The git executable (default "git" on PATH). */
+  readonly binary?: string
+}
+
+/** Runs `git -c protocol.ext.allow=never <args>` and resolves with stdout; rejects with a GitError. */
+export function runGit(args: readonly string[], options: GitOptions): Promise<string> {
+  const argv = ["-c", "protocol.ext.allow=never", ...args]
+  const what = `git ${args.find((a) => !a.startsWith("-")) ?? ""}`.trim()
+  return new Promise((resolve, reject) => {
+    if (options.signal?.aborted) return reject(new GitError(`${what} cancelled`))
+    let settled = false
+    const finish = (error?: GitError, stdout?: string) => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      options.signal?.removeEventListener("abort", onAbort)
+      if (error) reject(error)
+      else resolve(stdout ?? "")
+    }
+    // Its own process group, so a timeout or cancel also kills what git started (ssh, git-remote-https).
+    const child = spawn(options.binary ?? "git", argv, {
+      cwd: options.cwd,
+      env: gitEnv(options.baseEnv ?? process.env, options.env),
+      stdio: ["ignore", "pipe", "pipe"],
+      detached: true,
+      windowsHide: true,
+    })
+    const out: Buffer[] = []
+    const err: Buffer[] = []
+    let size = 0
+    const collect = (into: Buffer[]) => (chunk: Buffer) => {
+      size += chunk.length
+      if (size > MAX_BUFFER) return stop("printed too much output")
+      into.push(chunk)
+    }
+    child.stdout.on("data", collect(out))
+    child.stderr.on("data", collect(err))
+    child.on("error", (cause) => finish(new GitError(`${what} failed: ${cause.message}`)))
+    child.on("close", (code, signal) => {
+      if (code === 0) return finish(undefined, Buffer.concat(out).toString("utf8"))
+      const detail = redact(Buffer.concat(err).toString("utf8").trim()).slice(0, STDERR_MAX) || (signal ? `killed by ${signal}` : `exit ${code}`)
+      finish(new GitError(`${what} failed: ${detail}`))
+    })
+    // Settles at once: a killed git's children may hold its output pipes open for a while.
+    function stop(why: string): void {
+      if (settled) return
+      try {
+        if (child.pid) process.kill(-child.pid, "SIGKILL")
+      } catch {
+        child.kill("SIGKILL")
+      }
+      finish(new GitError(`${what} ${why}`))
+    }
+    const timeoutMs = options.timeoutMs ?? GIT_TIMEOUT_MS
+    const timer = setTimeout(() => stop(`timed out after ${Math.round(timeoutMs / 1000) || 1} s`), timeoutMs)
+    const onAbort = () => stop("cancelled")
+    options.signal?.addEventListener("abort", onAbort, { once: true })
+  })
+}

package/packages/node/src/harnesses.ts

@@ -0,0 +1,60 @@
+// Which harnesses the host runtime offers (the node hello's runtimes[host].harnesses, design §6.3). The host
+// runtime starts nothing itself: a host launch's session is created by the machine's service spoke of its harness
+// (placement also needs that spoke connected with "launch"), so this only says what can run here:
+// - opencode: always (its service spoke is the OpenCode plugin, which the node cannot see);
+// - claude-code: when the Claude Code CLI is found: HUB_CLAUDE_BIN (absolute, or a name on PATH), else `claude` on
+//   PATH; the Claude Code spoke daemon (`hub claude-code daemon`) creates its launch sessions;
+// - node.json `harnesses` overrides detection either way (true offers a harness, false withholds it).
+import { accessSync, constants, statSync } from "node:fs"
+import path from "node:path"
+
+type Env = Readonly<Record<string, string | undefined>>
+
+/** The order harnesses are listed in; others follow in node.json's order. */
+const KNOWN = ["opencode", "claude-code", "codex"]
+
+export interface DetectHarnessesOptions {
+  /** The environment to look in (PATH, HUB_CLAUDE_BIN); default process.env. */
+  readonly env?: Env
+  /** node.json `harnesses`. */
+  readonly overrides?: Readonly<Record<string, boolean>>
+}
+
+function executable(file: string): boolean {
+  try {
+    if (!statSync(file).isFile()) return false
+    accessSync(file, constants.X_OK)
+    return true
+  } catch {
+    return false
+  }
+}
+
+/** The executable `name` runs as: an absolute path checked as it is, else the first match on PATH. */
+export function findExecutable(name: string, env: Env): string | undefined {
+  if (!name) return undefined
+  if (path.isAbsolute(name)) return executable(name) ? name : undefined
+  if (name.includes("/")) return undefined
+  for (const dir of (env.PATH ?? "").split(path.delimiter)) {
+    if (!dir || !path.isAbsolute(dir)) continue
+    const file = path.join(dir, name)
+    if (executable(file)) return file
+  }
+  return undefined
+}
+
+/** The harnesses this machine's host runtime offers. */
+export function detectHarnesses(options: DetectHarnessesOptions = {}): string[] {
+  const env = options.env ?? process.env
+  const found = new Set<string>(["opencode"])
+  if (findExecutable(env.HUB_CLAUDE_BIN || "claude", env)) found.add("claude-code")
+  for (const [name, on] of Object.entries(options.overrides ?? {})) {
+    if (on) found.add(name)
+    else found.delete(name)
+  }
+  const rank = (name: string) => {
+    const i = KNOWN.indexOf(name)
+    return i < 0 ? KNOWN.length : i
+  }
+  return [...found].sort((a, b) => rank(a) - rank(b))
+}