@humwick/runner 0.0.0-stage → 0.130.0

package/src/artifact-path-materialize.ts

@@ -0,0 +1,52 @@
+import { maxArtifactBytesFromEnv, maxPathUploadRawBytes, type TokenConstraints } from "@humwick/sdk";
+import { openArtifactPathSource } from "@humwick/sdk/artifact-path-roots";
+
+// The relay server reads `path` uploads from its own filesystem, so a path that
+// exists on this agent host is "not found" on a remote relay (#2857). The runner
+// proxy runs beside the agent, reads the file here, and forwards bytes as base64
+// (agent→proxy stays a small path arg; only proxy→relay carries the bytes).
+export type PathMaterializeResult = { args?: Record<string, unknown>; error?: string };
+
+type TokenClaims = { constraints?: TokenConstraints; scope?: unknown };
+
+function decodeClaims(token: string | undefined): TokenClaims | undefined {
+  const payload = token?.split(".")[1];
+  if (!payload) return undefined;
+  try {
+    const normalized = payload.replace(/-/g, "+").replace(/_/g, "/");
+    return JSON.parse(Buffer.from(normalized, "base64").toString("utf8")) as TokenClaims;
+  } catch {
+    return undefined;
+  }
+}
+
+function hasWildcardScope(claims: TokenClaims): boolean {
+  return Array.isArray(claims.scope) && claims.scope.includes("*");
+}
+
+export async function materializeArtifactPathUpload(
+  args: Record<string, unknown>,
+  getToken: () => string | undefined,
+): Promise<PathMaterializeResult> {
+  if (typeof args.path !== "string") return {};
+  if (args.content !== undefined || args.base64 !== undefined) return {};
+  const claims = decodeClaims(getToken());
+  if (!claims) return {};
+  const rawLimit = maxPathUploadRawBytes(maxArtifactBytesFromEnv());
+  let source: Awaited<ReturnType<typeof openArtifactPathSource>> | undefined;
+  try {
+    source = await openArtifactPathSource(args.path.trim(), {
+      constraints: claims.constraints,
+      unrestricted: hasWildcardScope(claims),
+      maxBytes: rawLimit,
+    });
+    const bytes = Buffer.from(await new Response(source.stream).arrayBuffer());
+    if (bytes.length > rawLimit) return { error: `artifact exceeds ${rawLimit} bytes` };
+    const { path: _dropped, ...rest } = args;
+    return { args: { ...rest, base64: bytes.toString("base64"), ...(typeof args.filename === "string" ? {} : { filename: source.filename }) } };
+  } catch (error) {
+    return { error: (error as Error).message };
+  } finally {
+    await source?.cleanup();
+  }
+}

package/src/async-guard.ts

@@ -0,0 +1,35 @@
+// #1665 — runner adapter over the SHARED async guards in `@humwick/sdk`.
+//
+// The primitives themselves live in `sdk/src/async-guard.ts` because #1676 is the same
+// defect class in the orchestrator: an unguarded `void <promise>` / async timer callback whose
+// rejection reaches the runtime and (in Bun) exits the process. One implementation, two
+// processes. Import from `@humwick/sdk` directly if you are outside this package.
+//
+// #2443 — `src/unguarded-async-ratchet.test.ts` has enforced the STRUCTURAL half of #1665 since
+// #1676 (no NEW unguarded site may appear in runner/src), but the actual process-level backstop
+// (`installUnhandledRejectionGuard`) was never wired up here — only referenced by comments (see
+// runner-core.ts's `reportBackgroundRejection` doc). `runner/src/index.ts` installs it now.
+//
+// All this file adds is the `[runner]` log prefix, so the daemon's log format is unchanged and
+// call sites pass a bare label ("opencode server exit callback", not the prefix).
+
+import {
+  fireAndForget as sdkFireAndForget,
+  installUnhandledRejectionGuard as sdkInstallUnhandledRejectionGuard,
+  type GuardLog,
+} from "@humwick/sdk";
+
+export { type GuardLog } from "@humwick/sdk";
+
+const PREFIX = "[runner]";
+
+/** See {@link import("@humwick/sdk").fireAndForget}. Replaces bare `void promise()`. */
+export function fireAndForget(label: string, work: Promise<unknown> | (() => Promise<unknown> | unknown), log?: GuardLog): Promise<void> {
+  return sdkFireAndForget(`${PREFIX} ${label}`, work, log);
+}
+
+/** Install the non-fatal unhandled-rejection backstop for the runner daemon (#1665/#2443). See
+ *  the SDK function for the "log and survive" reasoning and why `uncaughtException` stays fatal. */
+export function installUnhandledRejectionGuard(log?: GuardLog): () => void {
+  return sdkInstallUnhandledRejectionGuard(PREFIX, log);
+}

package/src/attachment-cache-sweep.ts

@@ -0,0 +1,31 @@
+import { errMessage } from "@humwick/sdk";
+import { pruneAttachmentCache } from "./attachment-cache";
+import { runnerFsOffload } from "./fs-offload";
+
+const SWEEP_INTERVAL_MS = 60 * 60 * 1000;
+
+/** Process-local maintenance: intentionally separate from message delivery. */
+export class AttachmentCacheSweep {
+  private timer?: Timer;
+  private inFlight = false;
+
+  constructor(private readonly agentId: string, private readonly log: (message: string) => void) {}
+
+  start(): void {
+    if (this.timer) return;
+    const sweep = () => {
+      if (this.inFlight) return;
+      this.inFlight = true;
+      void runnerFsOffload().run("runner-attachment-cache-sweep", () => pruneAttachmentCache(this.agentId))
+        .catch((error) => this.log(`attachment cache sweep failed: ${errMessage(error)}`))
+        .finally(() => { this.inFlight = false; });
+    };
+    sweep();
+    this.timer = setInterval(sweep, SWEEP_INTERVAL_MS);
+  }
+
+  stop(): void {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = undefined;
+  }
+}

package/src/attachment-cache.ts

@@ -0,0 +1,195 @@
+import { mkdir, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { basename, join } from "node:path";
+import type { Artifact, Message } from "@humwick/sdk";
+import { errMessage, isRecord } from "@humwick/sdk";
+import { sanitizeFsName } from "@humwick/sdk/fs-name";
+import { attachmentCacheDirFromEnv } from "./config";
+
+const DEFAULT_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+interface AttachmentCacheClient {
+  downloadArtifact(id: string): Promise<{ stream: ReadableStream<Uint8Array>; meta: Artifact }>;
+}
+
+interface AttachmentCacheOptions {
+  agentId: string;
+  rootDir?: string;
+  maxAgeMs?: number;
+  now?: number;
+  onError?: (message: string) => void;
+}
+
+interface CachedAttachment {
+  path: string;
+  filename: string;
+  mediaType: string;
+  size: number;
+  digest: string;
+}
+
+function attachmentRefs(message: Message): Record<string, unknown>[] {
+  const payloadRefs = message.payload?.attachments;
+  const topLevelRefs = (message as Message & { attachments?: unknown }).attachments;
+  const refs = Array.isArray(payloadRefs) ? payloadRefs : Array.isArray(topLevelRefs) ? topLevelRefs : [];
+  return refs.filter(isRecord);
+}
+
+function attachmentCacheRoot(agentId: string, rootDir = attachmentCacheDirFromEnv()): string {
+  return join(attachmentCacheBase(rootDir), safePathPart(agentId));
+}
+
+function attachmentCacheBase(rootDir = attachmentCacheDirFromEnv()): string {
+  return rootDir || join(homedir(), ".agent-relay", "attachments");
+}
+
+export async function messagesWithCachedAttachments(
+  messages: Message[],
+  client: AttachmentCacheClient,
+  options: AttachmentCacheOptions,
+): Promise<Message[]> {
+  const enriched: Message[] = [];
+  for (const message of messages) enriched.push(await messageWithCachedAttachments(message, client, options));
+  return enriched;
+}
+
+async function messageWithCachedAttachments(
+  message: Message,
+  client: AttachmentCacheClient,
+  options: AttachmentCacheOptions,
+): Promise<Message> {
+  const refs = attachmentRefs(message);
+  if (!refs.length) return message;
+  const nextRefs = [];
+  let changed = false;
+  for (const ref of refs) {
+    const artifactId = typeof ref.artifactId === "string" ? ref.artifactId.trim() : "";
+    if (!artifactId) {
+      nextRefs.push(ref);
+      continue;
+    }
+    try {
+      const cached = await cacheArtifact(client, artifactId, options);
+      nextRefs.push(withCacheMetadata(ref, cached));
+      changed = true;
+    } catch (error) {
+      options.onError?.(`attachment ${artifactId} cache failed: ${errMessage(error)}`);
+      nextRefs.push(withCacheError(ref, error));
+      changed = true;
+    }
+  }
+  if (!changed) return message;
+  return {
+    ...message,
+    payload: {
+      ...message.payload,
+      attachments: nextRefs,
+    },
+  };
+}
+
+async function cacheArtifact(client: AttachmentCacheClient, artifactId: string, options: AttachmentCacheOptions): Promise<CachedAttachment> {
+  const { stream, meta } = await client.downloadArtifact(artifactId);
+  const root = attachmentCacheRoot(options.agentId, options.rootDir);
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  const filename = safeFilename(meta.filename || artifactId);
+  const digestPart = safePathPart((meta.digest || meta.blobDigest || artifactId).replace(/^sha256:/, "")).slice(0, 24) || safePathPart(artifactId);
+  const path = join(root, `${digestPart}-${filename}`);
+  try {
+    const current = await stat(path);
+    if (current.isFile() && current.size === meta.size) {
+      return { path, filename, mediaType: meta.mediaType, size: meta.size, digest: meta.digest };
+    }
+  } catch {}
+  const bytes = new Uint8Array(await new Response(stream).arrayBuffer());
+  if (bytes.byteLength !== meta.size) throw new Error(`artifact size mismatch: expected ${meta.size}, got ${bytes.byteLength}`);
+  const temp = `${path}.${process.pid}.${Date.now()}.tmp`;
+  await writeFile(temp, bytes, { mode: 0o600 });
+  await rm(path, { force: true });
+  await rename(temp, path);
+  return { path, filename, mediaType: meta.mediaType, size: meta.size, digest: meta.digest };
+}
+
+function withCacheMetadata(ref: Record<string, unknown>, cached: CachedAttachment): Record<string, unknown> {
+  const metadata = isRecord(ref.metadata) ? ref.metadata : {};
+  const agentRelay = isRecord(metadata.agentRelay) ? metadata.agentRelay : {};
+  return {
+    ...ref,
+    metadata: {
+      ...metadata,
+      agentRelay: {
+        ...agentRelay,
+        localPath: cached.path,
+        filename: cached.filename,
+        mediaType: cached.mediaType,
+        size: cached.size,
+        digest: cached.digest,
+        cachedAt: new Date().toISOString(),
+      },
+    },
+  };
+}
+
+function withCacheError(ref: Record<string, unknown>, error: unknown): Record<string, unknown> {
+  const metadata = isRecord(ref.metadata) ? ref.metadata : {};
+  const agentRelay = isRecord(metadata.agentRelay) ? metadata.agentRelay : {};
+  return {
+    ...ref,
+    metadata: {
+      ...metadata,
+      agentRelay: {
+        ...agentRelay,
+        cacheError: errMessage(error),
+      },
+    },
+  };
+}
+
+export async function pruneAttachmentCache(
+  agentId: string,
+  rootDir = attachmentCacheDirFromEnv(),
+  maxAgeMs = DEFAULT_CACHE_MAX_AGE_MS,
+  now = Date.now(),
+): Promise<void> {
+  const base = attachmentCacheBase(rootDir);
+  let entries: string[];
+  try {
+    entries = await readdir(base);
+  } catch {
+    return;
+  }
+  const roots = [attachmentCacheRoot(agentId, rootDir), ...entries.map((entry) => join(base, entry))];
+  await Promise.all([...new Set(roots)].map((root) => pruneAttachmentCacheDir(root, maxAgeMs, now)));
+}
+
+async function pruneAttachmentCacheDir(root: string, maxAgeMs: number, now: number): Promise<void> {
+  let entries: string[];
+  try {
+    entries = await readdir(root);
+  } catch {
+    return;
+  }
+  await Promise.all(entries.map(async (entry) => {
+    const path = join(root, entry);
+    try {
+      const current = await stat(path);
+      if (current.isFile() && now - current.mtimeMs > maxAgeMs) await rm(path, { force: true });
+    } catch {
+      // Best-effort cache cleanup only.
+    }
+  }));
+  try {
+    if ((await readdir(root)).length === 0) await rm(root, { recursive: true, force: true });
+  } catch {
+    // Best-effort cache cleanup only.
+  }
+}
+
+function safeFilename(value: string): string {
+  const name = safePathPart(basename(value));
+  return name || "artifact";
+}
+
+function safePathPart(value: string): string {
+  return sanitizeFsName(value, { replacement: "_", collapseReplacement: true, maxLen: 180 });
+}

package/src/background-script-reconcile.ts

@@ -0,0 +1,106 @@
+// #1856 — reconcile a leftover `background-script` busy that outlived its work.
+//
+// The Claude Stop hook (`plugins/claude/hooks/relay-status.sh` `relay_post_turn_end_status`)
+// posts a `background-script:background-tasks` busy record at turn-end whenever a
+// `run_in_background` shell is still running — but Claude's `background_tasks[]` payload carries
+// only a synthetic shell id, never an OS pid, so that record is posted WITHOUT `pids`. A record
+// with no pids can never be liveness-reconciled (#650 `reconcileLiveness` only touches pid-bearing
+// records), and work records have no TTL, so the only thing that ever clears it is the NEXT Stop
+// hook — i.e. a fresh turn. If the background process finishes, or the operator kills it in the
+// Web terminal, nothing reconciles the belief: the agent reads `busy` forever, its turn-final
+// report is never flushed to the coordinator, and it holds a live-children quota slot doing nothing
+// (the #1856 wedge; #1068 is the sibling transient-reap symptom).
+//
+// This is the runner-side backstop that breaks the wedge WITHOUT depending on per-task pids the
+// hook cannot supply: once the agent is busy on background-script work ALONE (its provider turn has
+// already ended) and has made no forward progress for the stale window, the leftover record is
+// force-cleared and the agent reconciles to idle on its own. When pids ARE available (a caller that
+// can resolve them, or a future hook), the #650 pid reconcile still clears it the instant the
+// process dies — this predicate only adds the no-pid backstop and is deliberately conservative:
+//   - a live provider turn (or any non-background-script work) → NEVER stale (that work owns busy),
+//   - a background-script record with a still-alive tracked pid → NEVER stale (demonstrably live),
+//   - no forward-progress timestamp yet → NEVER stale (nothing to age against).
+
+/** Default stale window: mirror the provider-turn backstop (runner-core STALE_PROVIDER_TURN_BUSY_MS).
+ *  A background task that legitimately runs this long with no intervening turn stops PINNING the
+ *  agent busy — the task keeps running, but the worker's report is allowed to flush. */
+export const STALE_BACKGROUND_SCRIPT_BUSY_MS = 10 * 60_000;
+
+export interface StaleBackgroundScriptBusyInput {
+  /** The runner's current active-work records (kind + optional tracked pids). */
+  activeWork: Array<{ kind: string; pids?: number[] }>;
+  /** Latest forward-progress epoch-ms (the runner's lastProviderEventAt). 0/undefined ⇒ none yet. */
+  lastProgressAt: number | undefined;
+  /** Now, injectable for tests. */
+  now: number;
+  /** Liveness probe for a tracked pid. */
+  isPidAlive: (pid: number) => boolean;
+  /** Stale window; defaults to {@link STALE_BACKGROUND_SCRIPT_BUSY_MS}. */
+  staleAfterMs?: number;
+}
+
+/**
+ * True when the agent's ENTIRE busy state is a leftover `background-script` record that has aged
+ * past the stale window with no live tracked process and no forward progress — i.e. it should be
+ * force-cleared so the agent reconciles busy→idle without a human nudge. Pure so the decision is
+ * unit-testable without standing up an AgentRunner.
+ */
+export function isStaleBackgroundScriptOnlyBusy(input: StaleBackgroundScriptBusyInput): boolean {
+  const { activeWork } = input;
+  // Something must be pinning busy, and it must be background-script work exclusively — a live
+  // provider-turn, subagent, or any other work kind owns the busy state and must never be healed
+  // here (the #1656/#1838 "never clear a live turn" invariant).
+  if (activeWork.length === 0) return false;
+  if (activeWork.some((work) => work.kind !== "background-script")) return false;
+  // A tracked pid still alive is positive proof the background work is genuinely running — hold.
+  // (No pids at all falls through: the hook can't supply them, so time is the only signal left.)
+  if (activeWork.some((work) => work.pids?.some((pid) => input.isPidAlive(pid)))) return false;
+  const lastProgressAt = input.lastProgressAt ?? 0;
+  if (!lastProgressAt) return false;
+  const staleAfterMs = input.staleAfterMs ?? STALE_BACKGROUND_SCRIPT_BUSY_MS;
+  return input.now - lastProgressAt >= staleAfterMs;
+}
+
+// #2034 (#2031 L3) — build the per-task background-script LEDGER a watchdog (#2035/#2036)
+// reads to name a specific long-running script and its age, rather than just a count.
+//
+// The Claude Stop hook (`relay_background_task_list` in relay-status.sh) can only ever post
+// the CURRENT snapshot of running tasks as {id, command} — each hook invocation is a fresh
+// process with no memory of when a task first appeared. This function is where startedAt
+// actually gets tracked: called on every background-script busy report with the freshly
+// reported tasks and the PRIOR ledger (the existing work record's metadata.backgroundTasks,
+// if any), it stamps startedAt=now for a task id seen for the first time and carries the
+// original startedAt forward for a task id that was already in the prior ledger — so a task
+// that keeps running across turns keeps its true start time instead of resetting every turn.
+// A task id absent from the fresh report (completed, or killed) is simply not in the
+// returned ledger; the caller's own work-record lifecycle (finishWork on idle) clears the
+// record entirely once no task is left running at all.
+export interface BackgroundTask {
+  id: string;
+  command: string;
+}
+
+export interface BackgroundLedgerTask extends BackgroundTask {
+  startedAt: number;
+}
+
+function isBackgroundTask(value: unknown): value is BackgroundTask {
+  return typeof value === "object" && value !== null
+    && typeof (value as Record<string, unknown>).id === "string"
+    && typeof (value as Record<string, unknown>).command === "string";
+}
+
+export function buildBackgroundTaskLedger(freshTasks: unknown, priorLedger: unknown, now: number): BackgroundLedgerTask[] {
+  const priorStartedAt = new Map<string, number>();
+  if (Array.isArray(priorLedger)) {
+    for (const entry of priorLedger) {
+      if (isBackgroundTask(entry) && typeof (entry as { startedAt?: unknown }).startedAt === "number") {
+        priorStartedAt.set(entry.id, (entry as BackgroundLedgerTask).startedAt);
+      }
+    }
+  }
+  if (!Array.isArray(freshTasks)) return [];
+  return freshTasks
+    .filter(isBackgroundTask)
+    .map((task) => ({ id: task.id, command: task.command, startedAt: priorStartedAt.get(task.id) ?? now }));
+}

package/src/bounded-log.ts

@@ -0,0 +1,59 @@
+import { appendFileSync, renameSync, rmSync, statSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+
+export const RUNNER_LOG_MAX_BYTES = 5 * 1024 * 1024;
+export const RUNNER_LOG_ARCHIVES = 2;
+export const RUNNER_LOG_RECORD_MAX_BYTES = 64 * 1024;
+const TRUNCATED = " …[truncated]\n";
+
+/** Bound a record before encoding it, including pathological multi-megabyte diagnostics. */
+export function boundedLogRecord(line: string, maxBytes: number): Buffer {
+  const bytes = Buffer.from(line.slice(0, maxBytes), "utf8");
+  if (bytes.length <= maxBytes && line.length <= maxBytes) return bytes;
+  const suffix = Buffer.from(TRUNCATED);
+  let end = maxBytes - suffix.length;
+  while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end--;
+  return Buffer.concat([bytes.subarray(0, end), suffix]);
+}
+
+function size(path: string): number {
+  try { return statSync(path).size; } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return 0;
+    throw error;
+  }
+}
+
+// Keep existing oversized logs intact for operator inspection; only bounded generations
+// participate in archive pruning (#2668).
+function preserveLegacyFile(path: string, fileSize: number, maxBytes: number): void {
+  if (fileSize <= maxBytes + Math.min(maxBytes, RUNNER_LOG_RECORD_MAX_BYTES)) return;
+  try { renameSync(path, `${path}.legacy-${randomUUID()}`); } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+  }
+}
+
+/** The runner's single diagnostics sink: active file + two archives, each <= maxBytes. */
+export function appendBoundedLog(path: string, line: string, maxBytes = RUNNER_LOG_MAX_BYTES, firstWrite = true): void {
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 128 || maxBytes > RUNNER_LOG_MAX_BYTES) {
+    throw new Error("invalid runner log size limit");
+  }
+  const bytes = boundedLogRecord(line, Math.min(maxBytes, RUNNER_LOG_RECORD_MAX_BYTES));
+  const activeSize = size(path);
+  if (firstWrite || activeSize + bytes.length > maxBytes) {
+    preserveLegacyFile(path, activeSize, maxBytes);
+    for (let i = 1; i <= RUNNER_LOG_ARCHIVES; i++) {
+      const archive = `${path}.${i}`;
+      preserveLegacyFile(archive, size(archive), maxBytes);
+    }
+  }
+  if (activeSize + bytes.length > maxBytes) {
+    rmSync(`${path}.${RUNNER_LOG_ARCHIVES}`, { force: true });
+    for (let i = RUNNER_LOG_ARCHIVES - 1; i >= 0; i--) {
+      const from = i === 0 ? path : `${path}.${i}`;
+      try { renameSync(from, `${path}.${i + 1}`); } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      }
+    }
+  }
+  appendFileSync(path, bytes, { mode: 0o600 });
+}

package/src/bounded-spawn-sync.ts

@@ -0,0 +1,18 @@
+// Synchronous subprocesses run on the runner's event-loop thread. A hung helper
+// must therefore be deadline-killed rather than pinning the whole runner forever.
+// Callers deliberately receive Bun's result unchanged: several commands use a
+// non-zero exit as an ordinary answer (for example, tmux has-session when absent).
+export const SYNC_SUBPROCESS_TIMEOUT_MS = 10_000;
+
+export function spawnSyncBounded(
+  cmd: string[],
+  options = {},
+  timeoutMs = SYNC_SUBPROCESS_TIMEOUT_MS,
+) {
+  return Bun.spawnSync({
+    cmd,
+    ...options,
+    timeout: timeoutMs,
+    killSignal: "SIGKILL",
+  });
+}