benchworks 0.1.1 → 0.3.0

package/bin/benchworks.mjs

@@ -2,19 +2,27 @@
 /*
  * Benchworks' runner. `benchworks connect --code XXXX-XXXX` pairs this Mac with your Benchworks account, and keeps it
  * connected while this Terminal window is open. It checks what the Assistant needs (Claude Code, its login, git), finds
- * the git repositories in the usual folders, and reports both to benchworks.dev every few seconds. It sends folder names
- * and versions, never the contents of a file. Run it again without a code and it reconnects as the same Mac.
+ * the git repositories in the usual folders, and reports both to benchworks.dev every few seconds. When you ask the
+ * Assistant something on benchworks.dev, it runs your own Claude Code here, under Benchworks' rules (not your personal
+ * settings or MCP servers), and sends back what Claude says and does; anything that would change a file or run a
+ * command waits for your Allow on the page. Run it again without a code and it reconnects as the same Mac.
  */
-import { execFile } from "node:child_process";
+import { execFile, spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { createInterface } from "node:readline";
 import { chmod, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
 import { homedir, hostname, platform } from "node:os";
 import { join } from "node:path";
 import { promisify } from "node:util";
 
-const VERSION = "0.1.1";
+const VERSION = "0.3.0";
 const API = (process.env.BENCHWORKS_URL ?? "https://benchworks.dev").replace(/\/+$/, "");
 const HOME = homedir();
-const STATE = join(HOME, ".benchworks", "runner.json");
+// the runner's own folder: its key and the folder conversations start in (BENCHWORKS_HOME moves it, for a second pairing on one Mac)
+const OWN = process.env.BENCHWORKS_HOME ?? join(HOME, ".benchworks");
+const STATE = join(OWN, "runner.json");
 const EVERY_MS = 5_000;
 const REPOS_EVERY_MS = 60_000;
 
@@ -34,6 +42,7 @@ async function out(cmd, args, cwd) {
 }
 
 const tilde = (p) => (p.startsWith(HOME) ? `~${p.slice(HOME.length)}` : p);
+const untilde = (p) => (p.startsWith("~/") ? join(HOME, p.slice(2)) : p);
 
 /* ---------------------------------------------------------------- what the Assistant needs */
 
@@ -112,6 +121,34 @@ async function api(path, body, key) {
   return { status: res.status, body: answer };
 }
 
+/**
+ * The long poll, on a connection of its own: through fetch's shared pool, a request held open for 25 seconds made the
+ * reports and the events wait behind it.
+ */
+function get(path, key) {
+  const url = new URL(`${API}${path}`);
+  const send = url.protocol === "http:" ? httpRequest : httpsRequest;
+  return new Promise((resolve, reject) => {
+    const req = send(url, { headers: { "user-agent": `benchworks-runner/${VERSION}`, authorization: `Bearer ${key}` }, agent: false, timeout: 40_000 }, (res) => {
+      let raw = "";
+      res.setEncoding("utf8");
+      res.on("data", (d) => (raw += d));
+      res.on("end", () => {
+        let body = {};
+        try {
+          body = JSON.parse(raw);
+        } catch {
+          /* not JSON: an error page on the way */
+        }
+        resolve({ status: res.statusCode ?? 0, body });
+      });
+    });
+    req.on("timeout", () => req.destroy(new Error("timeout")));
+    req.on("error", reject);
+    req.end();
+  });
+}
+
 const WHY = {
   no_code: "That code does not work. Make a new one on benchworks.dev and run the command again.",
   used: "That code was used already. Make a new one on benchworks.dev and run the command again.",
@@ -128,7 +165,7 @@ async function readState() {
 }
 
 async function saveState(state) {
-  await mkdir(join(HOME, ".benchworks"), { recursive: true, mode: 0o700 });
+  await mkdir(OWN, { recursive: true, mode: 0o700 });
   await writeFile(STATE, JSON.stringify(state, null, 2), { mode: 0o600 });
   await chmod(STATE, 0o600);
 }
@@ -170,11 +207,14 @@ async function connect(code) {
     await saveState(state);
     say(`Connected ${name} to Benchworks as ${paired.body.email}.`);
   } else if (!state?.key) {
-    return fail("This Mac is not connected yet. Copy the command with its code from benchworks.dev and run it here.");
+    return fail("This Mac is not connected yet. On benchworks.dev, open Settings › Your Macs, choose Connect a Mac, and run the command it gives you here.");
   } else {
     say(`Reconnecting ${state.machine} as ${state.email}.`);
   }
   printChecks(c, null);
+  key = state.key;
+  known = found;
+  void jobs();
   again = state.again ?? again;
   say(dim(`Keep this window open while you work. Ctrl-C disconnects; to come back, run: ${again}`));
 
@@ -186,16 +226,18 @@ async function connect(code) {
     printChecks(c, before);
     if (Date.now() - foundAt > REPOS_EVERY_MS) {
       found = await repos();
+      known = found;
       foundAt = Date.now();
     }
     const body = { report: report(), ...(picked !== undefined ? { picked } : {}) };
     picked = undefined;
-    const answer = await api("/api/runner/report", body, state.key).catch(() => null);
-    if (!answer || answer.status >= 500) {
-      if (!away) say(warn(`Cannot reach ${API}. Trying again.`));
+    const answer = await api("/api/runner/report", body, state.key).catch((e) => ({ status: 0, body: {}, why: e.cause?.code ?? e.name ?? "error" }));
+    if (answer.status === 0 || answer.status >= 500) {
+      if (!away) say(warn(`Cannot reach ${API} (${answer.why ?? answer.status}). Trying again.`));
       away = true;
     } else if (answer.status === 401) {
-      return fail("Benchworks no longer knows this Mac. Make a new code on benchworks.dev and run the command with it.");
+      fail("Benchworks no longer knows this Mac. Make a new code on benchworks.dev and run the command with it.");
+      return quit(1);
     } else {
       if (away) say("Back in touch with Benchworks.");
       away = false;
@@ -211,6 +253,337 @@ async function connect(code) {
   }
 }
 
+/* ---------------------------------------------------------------- conversations */
+
+/*
+ * A conversation runs as one Claude Code process per conversation, kept open between its turns, the way the local
+ * workbench's daemon runs it (daemon/src/agent/driver.ts and runner.ts): stream-json in and out, permission asks over
+ * stdio. What Claude says goes to Benchworks in the daemon's own shapes (messages of blocks, text deltas, permission
+ * requests, one result per turn), so benchworks.dev shows it with the workbench's own page.
+ */
+
+const SPACE = join(OWN, "space");
+const IDLE_MS = 15 * 60_000;
+const KILL_GRACE_MS = 5_000;
+const RESULT_TEXT_MAX = 50_000;
+const INPUT_STRING_MAX = 20_000;
+const STATUS_SUBTYPES = ["status", "thinking_tokens", "post_turn_summary", "api_retry"];
+let key = null;
+let known = [];
+
+/** Benchworks' rules for a conversation: where it starts, the person's repositories, and how to ask before changing. */
+function rules(found) {
+  return [
+    "You are the Benchworks Assistant. You run on the person's own Mac, through their own Claude Code, started from benchworks.dev.",
+    `You start in Benchworks' own folder, ${SPACE}. The person's git repositories on this Mac, newest work first:`,
+    ...found.map((r) => `- ${r.name}: ${untilde(r.path)}${r.branch ? ` (on ${r.branch})` : ""}`),
+    "When they name one, work in it by its full path. Reading them needs no permission; changing a file or running a command asks the person first, on the page.",
+    "Answer plainly and briefly: the person reads you in a browser.",
+  ].join("\n");
+}
+
+const sessions = new Map();
+const outbox = [];
+const acks = [];
+let flushing = null;
+let flushTimer = null;
+
+/** Events go to Benchworks in small batches: pieces of text every 150 ms, anything else at once. */
+function emit(event, now = false) {
+  outbox.push(event);
+  if (now) void flush();
+  else if (!flushTimer) flushTimer = setTimeout(() => void flush(), 150);
+}
+
+async function flush() {
+  if (flushTimer) clearTimeout(flushTimer);
+  flushTimer = null;
+  if (flushing) return flushing.then(() => flush());
+  if (!outbox.length && !acks.length) return;
+  const events = outbox.splice(0);
+  const done = acks.splice(0);
+  flushing = api("/api/runner/events", { acks: done, events }, key)
+    .then((r) => r.status < 500, () => false)
+    .then((sent) => {
+      flushing = null;
+      if (!sent) {
+        outbox.unshift(...events);
+        acks.unshift(...done);
+        setTimeout(() => void flush(), 2000);
+      }
+    });
+  return flushing;
+}
+
+const record = (v) => (typeof v === "object" && v !== null && !Array.isArray(v) ? v : null);
+const str = (v) => (typeof v === "string" ? v : null);
+const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : null);
+const list = (v) => (Array.isArray(v) ? v : []);
+const clipText = (s, max) => (s.length > max ? `${s.slice(0, max)}\n… (${s.length - max} more characters on the Mac)` : s);
+
+/** A tool's input as the page shows it: long strings (a whole file for Write) are cut so a message stays small. */
+function clipInput(v) {
+  if (typeof v === "string") return clipText(v, INPUT_STRING_MAX);
+  if (Array.isArray(v)) return v.map(clipInput);
+  const o = record(v);
+  return o ? Object.fromEntries(Object.entries(o).map(([k, x]) => [k, clipInput(x)])) : v;
+}
+
+/** A tool result's text; pictures stay on the Mac for now, and Claude's note naming one stays as it is. */
+function resultText(v) {
+  if (typeof v === "string") return v;
+  return list(v)
+    .map((item) => (record(item)?.type === "text" ? (str(item.text) ?? "") : ""))
+    .filter(Boolean)
+    .join("\n");
+}
+
+/** Claude Code's content blocks as the workbench's AgentBlocks (driver.ts parseBlocks). */
+function parseBlocks(content) {
+  if (typeof content === "string") return content ? [{ type: "text", text: content }] : [];
+  const out = [];
+  for (const block of list(content)) {
+    const b = record(block);
+    if (!b) continue;
+    if (b.type === "text") {
+      if (str(b.text) !== null) out.push({ type: "text", text: b.text });
+    } else if (b.type === "thinking") {
+      out.push({ type: "thinking", text: str(b.thinking) ?? "" });
+    } else if (b.type === "redacted_thinking") {
+      out.push({ type: "thinking", text: "" });
+    } else if (b.type === "tool_use") {
+      if (str(b.id) !== null && str(b.name) !== null) out.push({ type: "tool_use", id: b.id, name: b.name, input: clipInput(b.input) });
+    } else if (b.type === "tool_result") {
+      if (str(b.tool_use_id) !== null) out.push({ type: "tool_result", toolUseId: b.tool_use_id, content: clipText(resultText(b.content), RESULT_TEXT_MAX), isError: b.is_error === true });
+    }
+  }
+  return out;
+}
+
+/** An AskUserQuestion result also carries the answers by question; they go on the result (driver.ts withToolAnswers). */
+function withToolAnswers(blocks, toolUseResult) {
+  const given = record(record(toolUseResult)?.answers);
+  if (!given) return blocks;
+  const answers = Object.fromEntries(Object.entries(given).filter(([, a]) => typeof a === "string"));
+  const results = blocks.filter((b) => b.type === "tool_result");
+  if (results.length !== 1 || Object.keys(answers).length === 0) return blocks;
+  return blocks.map((b) => (b.type === "tool_result" ? { ...b, answers } : b));
+}
+
+function describeSystem(m, subtype) {
+  const detail = str(m.status) ?? str(m.status_detail) ?? str(m.message) ?? str(m.error);
+  if (subtype === "api_retry") return `api retry${num(m.attempt) !== null ? ` ${m.attempt}` : ""}${detail ? `: ${detail}` : ""}`;
+  if (subtype === "thinking_tokens") return num(m.tokens) !== null ? `thinking (${m.tokens} tokens)` : "thinking";
+  return detail ?? subtype;
+}
+
+/** Claude Code's environment without the variables a Claude Code session sets, so one started from inside Claude Code still runs. */
+function cleanEnv() {
+  const env = { ...process.env };
+  for (const k of Object.keys(env)) if (k.startsWith("CLAUDECODE") || k.startsWith("CLAUDE_CODE_")) delete env[k];
+  return env;
+}
+
+function argv(spec) {
+  const args = ["-p", "--input-format", "stream-json", "--output-format", "stream-json", "--verbose", "--include-partial-messages", "--permission-prompt-tool", "stdio"];
+  args.push("--model", spec.model, "--effort", spec.effort);
+  if (spec.permissionMode && spec.permissionMode !== "default") args.push("--permission-mode", spec.permissionMode);
+  if (spec.cliSessionId) args.push("--resume", spec.cliSessionId);
+  args.push("--setting-sources", "project", "--strict-mcp-config", "--append-system-prompt", rules(known));
+  if (known.length) args.push("--add-dir", ...known.map((r) => untilde(r.path)));
+  return args;
+}
+
+function write(s, message) {
+  if (!s.proc.stdin.writable) return false;
+  s.proc.stdin.write(`${JSON.stringify(message)}\n`);
+  return true;
+}
+
+/** Claude Code for one conversation; a runner that restarted, or a process that idled out, resumes it. */
+function open(spec) {
+  const proc = spawn("claude", argv(spec), { cwd: SPACE, stdio: ["pipe", "pipe", "pipe"], env: cleanEnv() });
+  const s = { id: spec.id, spec, proc, busy: false, inputs: new Map(), lastAt: Date.now(), ending: null, interrupted: false, respawn: false, stderr: [], startedAt: 0, costBase: 0 };
+  createInterface({ input: proc.stdout }).on("line", (line) => heard(s, line));
+  createInterface({ input: proc.stderr }).on("line", (line) => {
+    if (line.trim()) s.stderr = [...s.stderr, line].slice(-20);
+  });
+  proc.on("error", (e) => {
+    sessions.delete(s.id);
+    s.ending ??= "crashed";
+    emit({ session: s.id, type: "refused", error: `Claude Code could not start on this Mac: ${e.message}` }, true);
+  });
+  proc.on("exit", (code, signal) => {
+    if (sessions.get(s.id) === s) sessions.delete(s.id);
+    if (s.ending === "crashed") return;
+    const reason = s.ending ?? (code === 0 && !s.busy ? "stopped" : "crashed");
+    emit({ session: s.id, type: "exit", reason, code, signal, stderr: s.stderr.slice(-5) }, true);
+  });
+  sessions.set(s.id, s);
+  return s;
+}
+
+/** Ends a conversation's process: on Stop, or quietly when it idled or must start again with another effort. */
+function end(s, reason) {
+  if (s.ending) return;
+  s.ending = reason;
+  try {
+    s.proc.stdin.end();
+  } catch {
+    /* stdin closed already */
+  }
+  s.proc.kill("SIGTERM");
+  setTimeout(() => s.proc.exitCode === null && s.proc.kill("SIGKILL"), KILL_GRACE_MS).unref();
+}
+
+function control(s, request) {
+  return write(s, { type: "control_request", request_id: randomUUID(), request });
+}
+
+/** One line of Claude's stream-json, sent on as the daemon's driver reads it. */
+function heard(s, line) {
+  let m;
+  try {
+    m = JSON.parse(line);
+  } catch {
+    if (line.trim()) s.stderr = [...s.stderr, line].slice(-20);
+    return;
+  }
+  s.lastAt = Date.now();
+  const type = m?.type;
+  if (type === "system") {
+    if (m.subtype === "init" && str(m.session_id)) emit({ session: s.id, type: "init", cliSessionId: m.session_id, model: str(m.model), version: str(m.claude_code_version) }, true);
+    else if (STATUS_SUBTYPES.includes(m.subtype)) emit({ session: s.id, type: "status", text: describeSystem(m, m.subtype) });
+  } else if (type === "stream_event") {
+    const e = record(m.event);
+    const delta = record(e?.delta);
+    if (e?.type !== "content_block_delta" || delta?.type !== "text_delta" || !str(delta.text)) return;
+    emit({ session: s.id, type: "delta", messageId: str(m.api_message_id) ?? str(m.uuid) ?? "", index: num(e.index) ?? 0, text: delta.text });
+  } else if (type === "assistant" || type === "user") {
+    const message = record(m.message);
+    if (!message) return;
+    const blocks = withToolAnswers(parseBlocks(message.content), m.tool_use_result);
+    if (!blocks.length) return;
+    emit({ session: s.id, type: "message", message: { uuid: str(m.uuid) ?? randomUUID(), role: type, at: str(m.timestamp), blocks, parentToolUseId: str(m.parent_tool_use_id) } }, true);
+  } else if (type === "control_request") {
+    const request = record(m.request);
+    if (request?.subtype !== "can_use_tool" || !str(m.request_id)) return;
+    s.inputs.set(m.request_id, request.input);
+    const toolName = str(request.tool_name) ?? "tool";
+    emit(
+      {
+        session: s.id,
+        type: "permission",
+        request: { requestId: m.request_id, toolName, displayName: str(request.display_name) ?? toolName, input: clipInput(request.input), description: str(request.description), toolUseId: str(request.tool_use_id) },
+      },
+      true,
+    );
+    say(dim(`  ? ${toolName} waits for your answer on benchworks.dev`));
+  } else if (type === "result") {
+    const total = num(m.total_cost_usd) ?? 0;
+    const costUsd = Math.max(0, total - s.costBase);
+    s.costBase = Math.max(s.costBase, total);
+    const queued = num(m.queued_turn_count) ?? 0;
+    s.busy = queued > 0;
+    s.inputs.clear();
+    emit(
+      {
+        session: s.id,
+        type: "result",
+        result: { subtype: str(m.subtype) ?? "unknown", isError: m.is_error === true, costUsd, durationMs: num(m.duration_ms) ?? 0, turns: num(m.num_turns) ?? 0, text: str(m.result), cliSessionId: str(m.session_id), queued, interrupted: s.interrupted },
+      },
+      true,
+    );
+    say(dim(`  ${s.interrupted ? "stopped" : "answered"} in ${((Date.now() - s.startedAt) / 1000).toFixed(1)} s`));
+    s.interrupted = false;
+    if (!s.busy && s.respawn) end(s, "idle");
+  }
+}
+
+/** A job from Benchworks: a message, an answer to an ask, an interrupt, new settings, or Stop. */
+function handle(job) {
+  const spec = job.session;
+  let s = sessions.get(spec.id);
+  if (s?.ending) s = undefined;
+  switch (job.kind) {
+    case "send": {
+      s ??= open(spec);
+      if (job.now && s.busy) {
+        s.interrupted = true;
+        control(s, { subtype: "interrupt" });
+      }
+      if (!s.busy) s.startedAt = Date.now();
+      s.busy = true;
+      say(dim(`  → ${job.text.split("\n")[0].slice(0, 70)}`));
+      if (!write(s, { type: "user", message: { role: "user", content: [{ type: "text", text: job.text }] } })) emit({ session: spec.id, type: "refused", error: "Claude Code on this Mac stopped taking messages. Send it again." }, true);
+      return;
+    }
+    case "permission": {
+      if (!s || !s.inputs.has(job.requestId)) return emit({ session: spec.id, type: "refused", error: "Claude was not waiting on that any more: its process on this Mac had ended. Send your message again." }, true);
+      const input = s.inputs.get(job.requestId);
+      s.inputs.delete(job.requestId);
+      const answers = record(job.answers);
+      const updatedInput = answers && Object.keys(answers).length ? { ...(record(input) ?? {}), answers } : input;
+      const response = job.behavior === "allow" ? { behavior: "allow", updatedInput } : { behavior: "deny", message: job.message ?? "Denied on benchworks.dev" };
+      write(s, { type: "control_response", response: { subtype: "success", request_id: job.requestId, response } });
+      return;
+    }
+    case "interrupt": {
+      if (!s?.busy) return;
+      s.interrupted = true;
+      control(s, { subtype: "interrupt" });
+      return;
+    }
+    case "set": {
+      if (!s) return;
+      if (job.model) control(s, { subtype: "set_model", model: job.model });
+      if (job.permissionMode) control(s, { subtype: "set_permission_mode", mode: job.permissionMode });
+      // the effort is fixed when Claude Code starts: the next message starts it again, resuming the conversation
+      if (job.effort && job.effort !== s.spec.effort) {
+        s.respawn = true;
+        if (!s.busy) end(s, "idle");
+      }
+      s.spec = { ...s.spec, ...(job.model ? { model: job.model } : {}), ...(job.permissionMode ? { permissionMode: job.permissionMode } : {}) };
+      return;
+    }
+    case "stop": {
+      if (s) end(s, "stopped");
+      return;
+    }
+  }
+}
+
+/** The long poll: Benchworks hands over jobs the moment the page sends them. Each is said to be had, and run once. */
+async function jobs() {
+  await mkdir(SPACE, { recursive: true });
+  const seen = new Set();
+  for (;;) {
+    const r = await get("/api/runner/jobs?wait=25", key).catch(() => null);
+    if (!r || r.status >= 500) {
+      await new Promise((done) => setTimeout(done, 3000));
+      continue;
+    }
+    if (r.status === 401) return;
+    for (const job of r.body.jobs ?? []) {
+      acks.push(job.id);
+      if (seen.has(job.id)) continue;
+      seen.add(job.id);
+      handle(job);
+    }
+    if (acks.length) void flush();
+  }
+}
+
+// a conversation nobody spoke to for a while lets its Claude Code go; the next message resumes it
+setInterval(() => {
+  for (const s of sessions.values()) if (!s.busy && Date.now() - s.lastAt > IDLE_MS) end(s, "idle");
+}, 60_000).unref();
+
+function quit(code = 0) {
+  for (const s of sessions.values()) s.proc.kill();
+  process.exit(code);
+}
+
 function fail(line) {
   say(warn(line));
   process.exitCode = 1;
@@ -226,7 +599,7 @@ let again = "npx benchworks connect";
 
 process.on("SIGINT", () => {
   say(`\nDisconnected. To come back, run: ${again}`);
-  process.exit(0);
+  quit(0);
 });
 
 const [command, ...rest] = process.argv.slice(2);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "benchworks",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Connects your Mac to Benchworks, so the Assistant at benchworks.dev runs your own Claude Code with your logins and your files.",
   "type": "module",
   "bin": {