@huynle/hub 0.6.1 → 0.7.2

package/packages/node/src/config.ts

@@ -0,0 +1,184 @@
+// The node's settings and credentials. node.json (decision 6) sits next to the spoke's config.json, in the same
+// directory (~/.config/hub, or wherever HUB_SPOKE_CONFIG points): the spoke's normalize() drops fields it does not
+// know, so node settings never live in config.json. The node reads config.json only for the hub URL and the
+// machine token (spoke and node share one machine record) and never writes it.
+//
+// node.json: { "workspaceRoots": ["/Users/huy/projects"], "labels": {"gpu": "no"}, "slots": 6, "memoryMB": 49152,
+//   "openshell": { "enabled": true, "gateway": "openshell", "image": "ghcr.io/huynle/hub-sandbox:0.1",
+//                  "profiles": { "untrusted-code": { "providers": ["anthropic"] } } },
+//   "harnesses": { "claude-code": false } }
+// `harnesses` overrides what the host runtime detects (harnesses.ts): true offers a harness, false withholds it.
+import { readFileSync } from "node:fs"
+import { homedir, hostname } from "node:os"
+import path from "node:path"
+import { isAbsolutePath, labelsProblem, NODE_ROOTS_MAX, NODE_SLOTS_MAX } from "../../protocol/src/index.ts"
+import { normalizeOpenShellProfiles, type OpenShellProfiles } from "../../runtime-openshell/src/profiles.ts"
+import { configPath, readConfig } from "../../../src/config.ts"
+
+type Env = Readonly<Record<string, string | undefined>>
+
+export const NODE_CONFIG_FILE = "node.json"
+/** Concurrent launches when node.json sets no `slots`. */
+export const DEFAULT_SLOTS = 4
+const MEMORY_MB_MAX = 2 ** 31
+const GATEWAY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/
+const IMAGE = /^[A-Za-z0-9][A-Za-z0-9._/:@-]{0,254}$/
+/** A harness id as hellos and launch specs spell it. */
+const HARNESS = /^[a-z0-9][a-z0-9._-]{0,63}$/
+
+export interface NodeConfig {
+  /** Every launch workspace must resolve (realpath) under one of these. Empty: the node prepares nothing. */
+  readonly workspaceRoots: readonly string[]
+  /** Labels the node adds to its own (os, arch); placement matches on them. */
+  readonly labels: Readonly<Record<string, string>>
+  readonly slots: number
+  readonly memoryMB?: number
+  /** The `openshell` runtime's settings; absent when node.json has no openshell section. */
+  readonly openshell?: OpenShellNodeConfig
+  /** Per-harness overrides of detection (true: offer it, false: withhold it); absent: detection alone. */
+  readonly harnesses?: Readonly<Record<string, boolean>>
+}
+
+/** node.json `openshell`: whether the node offers sandboxes, through which gateway, with which image and profiles. */
+export interface OpenShellNodeConfig {
+  readonly enabled: boolean
+  /** An OpenShell gateway name (`--gateway`); default: the CLI's active gateway. */
+  readonly gateway?: string
+  /** The image for profiles without their own (default: the runtime's). */
+  readonly image?: string
+  readonly profiles: OpenShellProfiles
+}
+
+/** A config and what was wrong with the input (each bad field is dropped, with one problem line). */
+export interface NodeConfigResult {
+  readonly config: NodeConfig
+  readonly problems: readonly string[]
+}
+
+/** node.json in the spoke config's directory: ~/.config/hub/node.json, or next to $HUB_SPOKE_CONFIG. */
+export function nodeConfigPath(env: Env = process.env, home: string = homedir()): string {
+  return path.join(path.dirname(configPath(env, home)), NODE_CONFIG_FILE)
+}
+
+const isObject = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null && !Array.isArray(v)
+const whole = (v: unknown, min: number, max: number): v is number => typeof v === "number" && Number.isInteger(v) && v >= min && v <= max
+const show = (v: unknown) => JSON.stringify(v)?.slice(0, 80) ?? String(v)
+
+/** Normalizes untrusted node.json content, field by field. */
+export function normalizeNodeConfig(raw: unknown): NodeConfigResult {
+  const problems: string[] = []
+  let r: Record<string, unknown> = {}
+  if (isObject(raw)) r = raw
+  else if (raw !== undefined) problems.push("node.json must hold a JSON object")
+
+  const workspaceRoots: string[] = []
+  if (r.workspaceRoots !== undefined && !Array.isArray(r.workspaceRoots)) problems.push("workspaceRoots must be a list of absolute paths")
+  for (const root of Array.isArray(r.workspaceRoots) ? r.workspaceRoots : []) {
+    if (!isAbsolutePath(root) || root === "/") {
+      problems.push(`workspaceRoots: ${show(root)} is not an absolute path (without . or .. segments, and not /)`)
+      continue
+    }
+    const clean = root.replace(/\/+$/, "")
+    if (!workspaceRoots.includes(clean)) workspaceRoots.push(clean)
+  }
+  if (workspaceRoots.length > NODE_ROOTS_MAX) {
+    problems.push(`workspaceRoots: at most ${NODE_ROOTS_MAX} roots; the rest are ignored`)
+    workspaceRoots.length = NODE_ROOTS_MAX
+  }
+
+  let labels: Record<string, string> = {}
+  if (r.labels !== undefined) {
+    const problem = labelsProblem(r.labels)
+    if (problem) problems.push(`labels: ${problem}`)
+    else labels = { ...(r.labels as Record<string, string>) }
+  }
+
+  let slots = DEFAULT_SLOTS
+  if (r.slots !== undefined) {
+    if (whole(r.slots, 1, NODE_SLOTS_MAX)) slots = r.slots
+    else problems.push(`slots must be a whole number from 1 to ${NODE_SLOTS_MAX}`)
+  }
+
+  let memoryMB: number | undefined
+  if (r.memoryMB !== undefined) {
+    if (whole(r.memoryMB, 1, MEMORY_MB_MAX)) memoryMB = r.memoryMB
+    else problems.push("memoryMB must be a positive whole number")
+  }
+
+  const openshell = r.openshell === undefined ? undefined : normalizeOpenShell(r.openshell, problems)
+  const harnesses = r.harnesses === undefined ? undefined : normalizeHarnesses(r.harnesses, problems)
+
+  return {
+    config: { workspaceRoots, labels, slots, ...(memoryMB !== undefined ? { memoryMB } : {}), ...(openshell ? { openshell } : {}), ...(harnesses ? { harnesses } : {}) },
+    problems,
+  }
+}
+
+function normalizeHarnesses(raw: unknown, problems: string[]): Record<string, boolean> | undefined {
+  if (!isObject(raw)) {
+    problems.push("harnesses must be an object of harness names to true or false")
+    return undefined
+  }
+  const out: Record<string, boolean> = {}
+  for (const [name, on] of Object.entries(raw)) {
+    if (!HARNESS.test(name)) problems.push(`harnesses: ${show(name)} is not a harness name`)
+    else if (typeof on !== "boolean") problems.push(`harnesses.${name} must be true or false`)
+    else out[name] = on
+  }
+  return out
+}
+
+function normalizeOpenShell(raw: unknown, problems: string[]): OpenShellNodeConfig | undefined {
+  if (!isObject(raw)) {
+    problems.push("openshell must be an object")
+    return undefined
+  }
+  let enabled = false
+  if (raw.enabled !== undefined) {
+    if (typeof raw.enabled === "boolean") enabled = raw.enabled
+    else problems.push("openshell.enabled must be true or false")
+  }
+  let gateway: string | undefined
+  if (raw.gateway !== undefined) {
+    if (typeof raw.gateway === "string" && GATEWAY.test(raw.gateway)) gateway = raw.gateway
+    else problems.push(`openshell.gateway: ${show(raw.gateway)} is not a gateway name`)
+  }
+  let image: string | undefined
+  if (raw.image !== undefined) {
+    if (typeof raw.image === "string" && IMAGE.test(raw.image)) image = raw.image
+    else problems.push(`openshell.image: ${show(raw.image)} is not an image reference`)
+  }
+  const profiles = normalizeOpenShellProfiles(raw.profiles)
+  problems.push(...profiles.problems)
+  return { enabled, ...(gateway ? { gateway } : {}), ...(image ? { image } : {}), profiles }
+}
+
+/** Reads node.json; a missing file is the defaults, unreadable or bad JSON the defaults with a problem. */
+export function readNodeConfig(file: string): NodeConfigResult {
+  let text: string
+  try {
+    text = readFileSync(file, "utf8")
+  } catch (cause) {
+    if ((cause as NodeJS.ErrnoException).code === "ENOENT") return normalizeNodeConfig(undefined)
+    return { config: normalizeNodeConfig(undefined).config, problems: [`${file}: cannot read (${(cause as Error).message})`] }
+  }
+  try {
+    return normalizeNodeConfig(JSON.parse(text))
+  } catch {
+    return { config: normalizeNodeConfig(undefined).config, problems: [`${file}: not valid JSON`] }
+  }
+}
+
+/** What the node connects with: the hub URL, the machine token and the machine ID, from the spoke's config.json. */
+export interface SpokeCredentials {
+  readonly server: string
+  readonly token: string
+  readonly machineID?: string
+}
+
+/** The enrolled machine's credentials from config.json (read-only), or undefined when it holds no token. */
+export function readSpokeCredentials(env: Env = process.env, home: string = homedir()): SpokeCredentials | undefined {
+  const config = readConfig(configPath(env, home), hostname())
+  if (!config.token) return undefined
+  return { server: config.server, token: config.token, ...(config.machineID ? { machineID: config.machineID } : {}) }
+}

package/packages/node/src/connection.ts

@@ -0,0 +1,283 @@
+// The node's single outbound WebSocket to the hub (design §6.3): /spoke/ws with the machine token (spoke and node
+// share one machine record), a hello with role "node", then the hub's prepare/collect/cleanup/cancel requests
+// answered in order of arrival. Reconnects with jittered exponential backoff; a hub that does not serve nodes (its
+// welcome lacks "nodes", or it rejects the node hello before any welcome, as a hub older than nodes does) is
+// retried only after a long wait, so an old hub is not hammered. A revoked machine stops for good. A bye carrying
+// retryAfterMs (a hub pod draining, design §10) is followed by one short jittered reconnect (0–retryAfterMs) instead.
+//
+// The WebSocket is the runtime's global one with an Authorization header: Bun's accepts `{ headers }`, and so does
+// Node's (undici, Node 22+), so this runs on both (bundled for node, or from source on bun).
+import {
+  hubServesNodes,
+  MAX_FRAME,
+  NODE_ROLE,
+  parseNodeServerMessage,
+  PROTOCOL_VERSION,
+  type NodeCapacityFrame,
+  type NodeHello,
+  type NodeProgress,
+  type NodeRequest,
+  type Welcome,
+} from "../../protocol/src/index.ts"
+
+export type NodeConnectionState = "off" | "connecting" | "connected" | "unsupported" | "revoked" | "error"
+
+export interface Backoff {
+  readonly minMs: number
+  readonly maxMs: number
+  readonly random?: () => number
+}
+
+/** Minimal WebSocket surface used here. */
+export interface SocketLike {
+  readonly readyState: number
+  onopen: ((ev: unknown) => void) | null
+  onmessage: ((ev: { data: unknown }) => void) | null
+  onclose: ((ev: { code?: number; reason?: string }) => void) | null
+  onerror: ((ev: unknown) => void) | null
+  send(data: string): void
+  close(code?: number, reason?: string): void
+}
+
+export interface NodeConnectionOptions {
+  readonly server: string
+  readonly token: string
+  /** Hello fields, computed per attempt so capacity, use and launches are current. */
+  readonly hello: () => Omit<NodeHello, "t" | "v" | "role">
+  /** Serves a hub request; a thrown error becomes `{ ok: false, error }`. */
+  readonly handle: (req: NodeRequest) => Promise<unknown>
+  readonly onWelcome?: (welcome: Welcome) => void
+  readonly onState?: (state: NodeConnectionState, error?: string) => void
+  readonly backoff?: Backoff
+  /** Wait before retrying a hub that does not serve nodes (default 30 minutes). */
+  readonly unsupportedRetryMs?: number
+  readonly socket?: (url: string, headers: Record<string, string>) => SocketLike
+}
+
+export const UNSUPPORTED_RETRY_MS = 30 * 60_000
+
+export function backoffDelay(attempt: number, backoff: Backoff): number {
+  const base = Math.min(backoff.maxMs, backoff.minMs * 2 ** Math.min(attempt, 30))
+  return Math.round(base * (0.5 + 0.5 * (backoff.random ?? Math.random)()))
+}
+
+/** The reconnect delay a bye's retryAfterMs asks for: uniformly jittered over 0–retryAfterMs, so peers spread out. */
+export function retryAfterDelay(retryAfterMs: number, random: () => number = Math.random): number {
+  return Math.round(retryAfterMs * random())
+}
+
+export function wsURL(server: string): string {
+  return `${server.replace(/\/+$/, "").replace(/^http/, "ws")}/spoke/ws`
+}
+
+const defaultSocket = (url: string, headers: Record<string, string>): SocketLike =>
+  new (WebSocket as unknown as new (url: string, options: { headers: Record<string, string> }) => SocketLike)(url, { headers })
+
+const message = (cause: unknown) => (cause instanceof Error ? cause.message : String(cause))
+
+function text(data: unknown): string {
+  if (typeof data === "string") return data
+  if (data instanceof ArrayBuffer) return Buffer.from(data).toString("utf8")
+  if (ArrayBuffer.isView(data)) return Buffer.from(data.buffer, data.byteOffset, data.byteLength).toString("utf8")
+  return String(data)
+}
+
+/** The id of a frame that is a request, valid or not (so an invalid request still gets an answer). */
+function requestID(raw: string): string | undefined {
+  if (raw.length > MAX_FRAME) return undefined
+  try {
+    const v = JSON.parse(raw)
+    return v && typeof v === "object" && v.t === "req" && typeof v.id === "string" && v.id.length > 0 && v.id.length <= 300 ? v.id : undefined
+  } catch {
+    return undefined
+  }
+}
+
+export class NodeConnection {
+  state: NodeConnectionState = "off"
+  lastError?: string
+  machineID?: string
+  publicURL?: string
+
+  private socket?: SocketLike
+  private welcomed = false
+  private stopped = true
+  private attempt = 0
+  private retry?: ReturnType<typeof setTimeout>
+  /** The last bye's retryAfterMs (a draining hub): the next reconnect waits about this instead of the backoff. */
+  private retryAfterMs?: number
+  private readonly backoff: Backoff
+
+  constructor(private readonly options: NodeConnectionOptions) {
+    this.backoff = options.backoff ?? { minMs: 1000, maxMs: 60_000 }
+  }
+
+  start(): void {
+    if (!this.stopped) return
+    this.stopped = false
+    this.open()
+  }
+
+  stop(): void {
+    this.stopped = true
+    clearTimeout(this.retry)
+    this.drop(1000, "stop")
+    this.setState("off")
+  }
+
+  /** Sends a capacity or progress frame once the hub has welcomed us. Returns false when it could not be sent. */
+  send(frame: NodeCapacityFrame | NodeProgress): boolean {
+    return this.write(frame)
+  }
+
+  private write(frame: unknown): boolean {
+    if (!this.socket || !this.welcomed || this.socket.readyState !== 1) return false
+    try {
+      this.socket.send(JSON.stringify(frame))
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  private open(): void {
+    if (this.stopped) return
+    this.setState(this.state === "error" || this.state === "unsupported" ? this.state : "connecting")
+    let socket: SocketLike
+    try {
+      socket = (this.options.socket ?? defaultSocket)(wsURL(this.options.server), { Authorization: `Bearer ${this.options.token}` })
+    } catch (cause) {
+      return this.fail(message(cause))
+    }
+    this.socket = socket
+    this.welcomed = false
+    let opened = false
+    let rejected = false
+    socket.onopen = () => {
+      opened = true
+      try {
+        socket.send(JSON.stringify({ t: "hello", v: PROTOCOL_VERSION, role: NODE_ROLE, ...this.options.hello() }))
+      } catch (cause) {
+        this.lastError = message(cause)
+      }
+    }
+    socket.onmessage = (ev) => {
+      const raw = text(ev.data)
+      const frame = parseNodeServerMessage(raw)
+      if (!frame) {
+        const id = this.welcomed ? requestID(raw) : undefined
+        if (id) this.write({ t: "res", id, ok: false, error: "invalid request" })
+        return
+      }
+      switch (frame.t) {
+        case "welcome":
+          return this.welcome(frame)
+        case "bye":
+          if (frame.reason === "revoked") return this.revoked()
+          // A draining hub is no hub refusing nodes: come back soon (to another pod).
+          if (frame.retryAfterMs !== undefined) this.retryAfterMs = frame.retryAfterMs
+          else if (!this.welcomed) rejected = true
+          this.lastError = frame.reason
+          return
+        case "req":
+          if (this.welcomed) void this.answer(frame.id, frame.req)
+          return
+        case "res":
+          return // the node makes no requests of the hub
+      }
+    }
+    socket.onerror = () => {
+      this.lastError = this.welcomed ? "Connection error" : "Could not connect to the hub"
+    }
+    socket.onclose = (ev) => {
+      if (this.socket !== socket) return
+      this.socket = undefined
+      const wasWelcomed = this.welcomed
+      this.welcomed = false
+      if (this.stopped || this.state === "revoked") return
+      // Opened, then closed by the hub before any welcome: the hub refused the node hello.
+      if (!wasWelcomed && opened && (rejected || ev.code === 4000) && this.retryAfterMs === undefined) return this.unsupported(`The hub does not accept nodes${this.lastError ? ` (${this.lastError})` : ""}`)
+      this.fail(wasWelcomed ? `Connection closed${ev.reason ? `: ${ev.reason}` : ""}` : (this.lastError ?? "Could not connect"))
+    }
+  }
+
+  private welcome(frame: Welcome): void {
+    if (!hubServesNodes(frame)) {
+      this.drop(1000, "hub does not serve nodes")
+      return this.unsupported("The hub does not serve nodes (upgrade the hub)")
+    }
+    this.welcomed = true
+    this.attempt = 0
+    this.machineID = frame.machineID
+    this.publicURL = frame.publicURL
+    this.lastError = undefined
+    this.setState("connected")
+    try {
+      this.options.onWelcome?.(frame)
+    } catch (cause) {
+      console.error("[hub node] welcome handler failed", cause)
+    }
+  }
+
+  private async answer(id: string, req: NodeRequest): Promise<void> {
+    let reply: unknown
+    try {
+      reply = { t: "res", id, ok: true, result: await this.options.handle(req) }
+    } catch (cause) {
+      reply = { t: "res", id, ok: false, error: message(cause) }
+    }
+    if (JSON.stringify(reply).length > MAX_FRAME) reply = { t: "res", id, ok: false, error: "result too large to send" }
+    this.write(reply)
+  }
+
+  private revoked(): void {
+    this.stopped = true
+    clearTimeout(this.retry)
+    this.lastError = "This machine was revoked on the hub"
+    this.setState("revoked")
+    this.drop(1000, "revoked")
+  }
+
+  private unsupported(error: string): void {
+    this.lastError = error
+    this.setState("unsupported")
+    this.schedule(this.options.unsupportedRetryMs ?? UNSUPPORTED_RETRY_MS)
+  }
+
+  private fail(error: string): void {
+    this.lastError = error
+    this.setState("error")
+    const retryAfterMs = this.retryAfterMs
+    this.retryAfterMs = undefined
+    this.schedule(retryAfterMs !== undefined ? retryAfterDelay(retryAfterMs, this.backoff.random) : backoffDelay(this.attempt++, this.backoff))
+  }
+
+  private schedule(ms: number): void {
+    if (this.stopped) return
+    clearTimeout(this.retry)
+    this.retry = setTimeout(() => this.open(), ms)
+  }
+
+  private drop(code: number, reason: string): void {
+    const socket = this.socket
+    this.socket = undefined
+    this.welcomed = false
+    if (!socket) return
+    socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null
+    try {
+      socket.close(code, reason)
+    } catch {
+      // already closed
+    }
+  }
+
+  private setState(state: NodeConnectionState): void {
+    if (state === this.state) return
+    this.state = state
+    try {
+      this.options.onState?.(state, this.lastError)
+    } catch {
+      // observers must not break the connection
+    }
+  }
+}