@cat-factory/executor-harness 1.31.0

package/dist/process.js

@@ -0,0 +1,25 @@
+import { log } from './logger.js';
+// Shared child-process lifecycle helpers. Every CLI the harness spawns (Pi and the
+// subscription harnesses) must die the same way when the watchdog or a no-progress
+// guard aborts, so the SIGTERM→SIGKILL escalation lives here rather than being
+// re-implemented per runner.
+// How long to wait after SIGTERM before escalating to SIGKILL.
+const KILL_GRACE_MS = 5_000;
+/**
+ * Terminate a child process: SIGTERM first, then SIGKILL after a grace period if it
+ * hasn't exited (ignored an ordinary terminate). The escalation timer is `unref()`d
+ * so it never by itself keeps the event loop alive. Safe to call more than once.
+ *
+ * An actual escalation to SIGKILL is logged at warn level: a process that ignores
+ * SIGTERM and has to be force-killed is a signal worth seeing (a wedged Pi/CLI), and
+ * was previously invisible. Pass a child logger to carry the run's `jobId`.
+ */
+export function killChildProcess(child, graceMs = KILL_GRACE_MS, logger = log) {
+    child.kill('SIGTERM');
+    setTimeout(() => {
+        if (child.exitCode === null && child.signalCode === null) {
+            logger.warn('killChildProcess: process ignored SIGTERM, escalating to SIGKILL', { graceMs });
+            child.kill('SIGKILL');
+        }
+    }, graceMs).unref();
+}

package/dist/redact.js

@@ -0,0 +1,109 @@
+// Single source of truth for credential redaction. Two complementary rules run on
+// EVERY redaction so no error path can scrub one class of secret and leak the other:
+//
+//  - PATTERN-based: scrubs credential SHAPES (URL userinfo, `x-access-token:<tok>`,
+//    bare GitHub token prefixes, and `KEY=value` / `KEY: value` assignments whose key
+//    names a credential) even when the exact value isn't known ahead of time — this is
+//    what catches a freshly-minted installation token in a git error, or a plaintext
+//    `POSTGRES_PASSWORD=…` echoed by a docker-compose dependency stand-up.
+//  - VALUE-based: scrubs a list of KNOWN secret strings (the leased subscription
+//    token + any token-like JSON leaf harvested from a credential blob).
+//
+// Historically these lived in two modules (git.ts pattern-only, agent-runner.ts
+// value-only) and ran on disjoint paths, so a secret only one rule covered could leak
+// on the other. They are unified here.
+// Below this length a "known secret" is too short to scrub without mangling
+// legitimate output (it would replace common substrings).
+const MIN_REDACT_LEN = 6;
+// Only harvest token-like JSON leaves: real OAuth access/refresh tokens and ids are
+// long, while short values (`auth_mode: "chatgpt"`, `type: "oauth"`, …) are non-secret
+// words that would over-redact legitimate error text if scrubbed. 12 chars is a safe
+// floor below which a value is not a credential.
+const MIN_HARVEST_LEN = 12;
+// `KEY=value` / `KEY: value` assignments whose key NAMES a credential. Catches plaintext
+// secrets the shape rules above miss — e.g. a docker-compose dependency echoing
+// `POSTGRES_PASSWORD=hunter2` or `DATABASE_PASSWORD: hunter2` on a failed stand-up, which
+// is not a token shape and is not in the known-value list (the harness never sees the
+// service's own secrets). The key token is matched within a surrounding identifier so
+// `DB_ACCESS_KEY`/`api_key` are covered; `auth` is deliberately excluded so it can't
+// clobber a git `Author:` line. The value is the first whitespace-delimited run.
+const CREDENTIAL_ASSIGNMENT = /\b([A-Za-z0-9_]*(?:password|passwd|pwd|secret|token|key|credential)[A-Za-z0-9_]*\s*[:=]\s*)\S+/gi;
+/**
+ * Strip credentials out of any string before it is logged or stored. Applies the
+ * pattern rules (URL userinfo `https://user:pass@host`, `x-access-token:<token>`, bare
+ * `ghs_`/`ghp_`/`gho_`/`github_pat_` shapes, and credential-named `KEY=value` / `KEY:
+ * value` assignments) and then scrubs every supplied known-secret value. Idempotent —
+ * safe to call on already-redacted text.
+ */
+export function redact(input, knownSecrets = []) {
+    let out = input
+        .replace(/(https?:\/\/)[^@\s/]*@/gi, '$1***@')
+        .replace(/x-access-token:[^@\s]+/gi, 'x-access-token:***')
+        .replace(/\b(gh[pso]_|github_pat_)[A-Za-z0-9_]+/g, '$1***')
+        .replace(CREDENTIAL_ASSIGNMENT, '$1***');
+    for (const secret of knownSecrets) {
+        // Guard against scrubbing trivially-short values that would mangle output.
+        if (secret.length >= MIN_REDACT_LEN)
+            out = out.split(secret).join('***');
+    }
+    return out;
+}
+/** Pattern-only redaction (no known values). Kept for callers without a secret list. */
+export function redactSecrets(input) {
+    return redact(input);
+}
+/** Cap on captured command output kept on an infra record (tail-biased — failures show last). */
+export const MAX_CAPTURED_OUTPUT_CHARS = 16_000;
+/**
+ * Combine, redact and tail-bound captured stdout+stderr into a single stored string. Keeps
+ * the LAST {@link MAX_CAPTURED_OUTPUT_CHARS} (where a failure's error lives), prefixed with a
+ * truncation marker when trimmed. Returns undefined for empty output so a record stays sparse.
+ * Shared by the docker-compose and the frontend UI-test stand-ups.
+ */
+export function captureRedactedOutput(stdout, stderr) {
+    const merged = [String(stdout ?? ''), String(stderr ?? '')]
+        .map((s) => s.trim())
+        .filter(Boolean)
+        .join('\n');
+    if (!merged)
+        return undefined;
+    const redacted = redactSecrets(merged);
+    if (redacted.length <= MAX_CAPTURED_OUTPUT_CHARS)
+        return redacted;
+    return `…(${redacted.length - MAX_CAPTURED_OUTPUT_CHARS} earlier chars trimmed)\n${redacted.slice(-MAX_CAPTURED_OUTPUT_CHARS)}`;
+}
+/** Recursively harvest token-like string leaves from a parsed JSON value. */
+function collectStrings(value, out) {
+    if (typeof value === 'string') {
+        if (value.length >= MIN_HARVEST_LEN)
+            out.add(value);
+    }
+    else if (Array.isArray(value)) {
+        for (const v of value)
+            collectStrings(v, out);
+    }
+    else if (value && typeof value === 'object') {
+        for (const v of Object.values(value))
+            collectStrings(v, out);
+    }
+}
+/**
+ * The set of secret strings to scrub from a run's stderr/output. For Claude (and the
+ * Anthropic-compatible vendors GLM/Kimi/DeepSeek) the credential IS the token string,
+ * so the whole-string entry covers it. For Codex the credential is a whole `auth.json`
+ * blob, so we ALSO scrub every string value parsed out of it (access/refresh tokens,
+ * ids): a token echoed on its OWN — not as part of the whole blob — would otherwise
+ * slip past a whole-blob-only match and leak into an error message.
+ */
+export function secretsToRedact(subscriptionToken) {
+    const secrets = new Set();
+    if (subscriptionToken)
+        secrets.add(subscriptionToken);
+    try {
+        collectStrings(JSON.parse(subscriptionToken), secrets);
+    }
+    catch {
+        // Not JSON (a Claude OAuth token / API key) — the whole-string entry covers it.
+    }
+    return [...secrets];
+}

package/dist/runner.js

@@ -0,0 +1,228 @@
+import { redactSecrets } from './redact.js';
+import { log } from './logger.js';
+import { failureCauseOf, inactivityAbortMessage, maxDurationAbortMessage, } from './failure.js';
+function intEnv(value, fallback) {
+    const n = value ? Number(value) : NaN;
+    return Number.isFinite(n) && n > 0 ? n : fallback;
+}
+export function loadRunnerLimits(env = process.env) {
+    return {
+        // 60 minutes: generous headroom for serious multi-file coding tasks while
+        // still bounding a runaway container.
+        maxDurationMs: intEnv(env.JOB_MAX_DURATION_MS, 60 * 60_000),
+        // 10 minutes of zero output is treated as hung (a single long LLM/tool call
+        // is far shorter; Pi streams events as it works). The per-git command ceiling
+        // (`GIT_TIMEOUT_MS` in git.ts) is DERIVED from this value — a fixed margin below
+        // it — so a slow clone/push (which emits no activity events) always times out
+        // with git's own clear reason rather than this watchdog's "likely hung" message,
+        // for any configured window. See the invariant note in git.ts.
+        inactivityMs: intEnv(env.JOB_INACTIVITY_MS, 10 * 60_000),
+    };
+}
+function toView(entry) {
+    const { promise: _promise, spanBuffer: _spanBuffer, followUpBuffer: _followUpBuffer, ...view } = entry;
+    return { ...view };
+}
+/**
+ * Tracks background jobs by id. Keyed by the backend-supplied job id (the per-step
+ * job id) so a re-dispatched start re-attaches to the running job rather than starting
+ * a duplicate — which keeps the durable driver's retries idempotent and avoids redoing
+ * already-running work. Generic over the job/result shape so the same lifecycle +
+ * inactivity/max-duration watchdogs drive every agent run.
+ */
+export class JobRegistry {
+    limits;
+    run;
+    describe;
+    jobs = new Map();
+    constructor(limits, 
+    // The unit of work (the `agent` handler). Injectable so tests can drive the
+    // registry's lifecycle/watchdog logic with a different runner.
+    run, 
+    // Non-secret correlation fields to bind on the per-job logger (repo, branch, agentKind).
+    // The registry is generic over the job shape, so the kind supplies this extractor; the
+    // job id is always bound. Defaults to no extra fields.
+    describe = () => ({})) {
+        this.limits = limits;
+        this.run = run;
+        this.describe = describe;
+    }
+    /** Start the job for `id`, or return the existing one (idempotent re-attach). */
+    start(id, job) {
+        const existing = this.jobs.get(id);
+        if (existing)
+            return toView(existing);
+        const now = Date.now();
+        const entry = {
+            id,
+            state: 'running',
+            startedAt: now,
+            // Seed the live phase so a poll BEFORE the handler enters its first phase still
+            // shows "starting" (the container is up; the agent hasn't begun cloning yet)
+            // rather than an absent/blank phase.
+            phase: 'starting',
+            heartbeatAt: now,
+            promise: Promise.resolve(),
+            spanBuffer: [],
+            followUpBuffer: [],
+        };
+        this.jobs.set(id, entry);
+        entry.promise = this.drive(entry, job);
+        return toView(entry);
+    }
+    /**
+     * Poll the job — and DRAIN its tool-span buffer (drain-on-read). The GET /jobs/{id}
+     * handler is the sole caller, so each poll returns the spans accumulated since the
+     * previous poll and clears them, bounding the harness buffer to one poll interval.
+     */
+    get(id) {
+        const entry = this.jobs.get(id);
+        if (!entry)
+            return undefined;
+        const view = toView(entry);
+        if (entry.spanBuffer.length > 0) {
+            view.spans = entry.spanBuffer;
+            entry.spanBuffer = [];
+        }
+        if (entry.followUpBuffer.length > 0) {
+            view.followUps = entry.followUpBuffer;
+            entry.followUpBuffer = [];
+        }
+        return view;
+    }
+    async drive(entry, job) {
+        const controller = new AbortController();
+        let killReason;
+        const jobLog = log.child({ jobId: entry.id, ...this.describe(job) });
+        // Stuck-run breadcrumb: the coarse phase the handler is in, per-phase wall-clock, and
+        // the last completed tool — so an inactivity kill can say WHERE it hung instead of a
+        // bare "likely hung", and the finish/fail log carries the phase-timing breakdown.
+        let phase = 'starting';
+        let phaseEnteredAt = Date.now();
+        const phaseTimingsMs = {};
+        const markPhase = (next) => {
+            const now = Date.now();
+            phaseTimingsMs[phase] = (phaseTimingsMs[phase] ?? 0) + (now - phaseEnteredAt);
+            phase = next;
+            phaseEnteredAt = now;
+            // Surface the live phase on the view so a poll shows WHAT the container is doing
+            // (cloning / running the agent / pushing) — the same marker drives the failure
+            // breadcrumb. A terminal `done`/`failed` is set by the caller below.
+            entry.phase = next;
+        };
+        let lastTool;
+        let inactivity;
+        const resetInactivity = () => {
+            clearTimeout(inactivity);
+            inactivity = setTimeout(() => {
+                // First watchdog to fire wins the reason (a later timer firing in the
+                // teardown window must not relabel why the job was killed).
+                killReason ??= 'inactivity';
+                controller.abort(new Error('inactivity timeout'));
+            }, this.limits.inactivityMs);
+        };
+        const cap = setTimeout(() => {
+            killReason ??= 'max-duration';
+            controller.abort(new Error('max duration exceeded'));
+        }, this.limits.maxDurationMs);
+        const heartbeat = () => {
+            entry.heartbeatAt = Date.now();
+            resetInactivity();
+        };
+        resetInactivity();
+        jobLog.info('job started', {});
+        try {
+            const result = await this.run(job, {
+                signal: controller.signal,
+                onActivity: heartbeat,
+                onProgress: (progress) => {
+                    entry.progress = progress;
+                },
+                onSpan: (span) => {
+                    entry.spanBuffer.push(span);
+                    lastTool = { name: span.tool, at: span.endedAt };
+                },
+                onFollowUp: (items) => {
+                    entry.followUpBuffer.push(...items);
+                },
+                onPhase: (next) => markPhase(next),
+                log: jobLog,
+            });
+            markPhase('done');
+            entry.state = 'done';
+            entry.result = result;
+            // A clean-exit result can still be a failure (e.g. no usable output): carry its
+            // structured cause onto the view so the backend classifies it without regex.
+            if (result.error && result.failureCause)
+                entry.failureCause = result.failureCause;
+            jobLog.info('job finished', {
+                durationMs: Date.now() - entry.startedAt,
+                jobError: result.error ?? null,
+                phaseTimingsMs,
+            });
+        }
+        catch (error) {
+            // Capture the phase the job was IN before recording the 'failed' transition, so the
+            // breadcrumb names where it hung (markPhase below would otherwise overwrite it).
+            const failedInPhase = phase;
+            markPhase('failed');
+            const { message, cause, detail } = this.describeFailure(killReason, error, failedInPhase, lastTool, phaseTimingsMs);
+            entry.state = 'failed';
+            entry.error = message;
+            entry.failureCause = cause;
+            entry.detail = detail;
+            jobLog.error('job failed', {
+                durationMs: Date.now() - entry.startedAt,
+                reason: killReason ?? 'error',
+                failureCause: cause,
+                error: message,
+                phaseTimingsMs,
+            });
+        }
+        finally {
+            clearTimeout(inactivity);
+            clearTimeout(cap);
+            entry.heartbeatAt = Date.now();
+        }
+    }
+    /**
+     * Build the redacted one-line `error`, the structured {@link FailureCause}, and the extended
+     * `detail` for a failed job. Watchdog kills keep their regex-stable phrase (so the backend's
+     * `classifyBootstrapFailure` fallback still works) and gain a breadcrumb of where they hung;
+     * a thrown error keeps its own message and its structured cause when tagged (a git op → `git`,
+     * an upstream API call → `api`), else `agent`. All strings are credential-scrubbed.
+     */
+    describeFailure(killReason, error, phase, lastTool, phaseTimingsMs) {
+        // `lastTool` is the last tool that COMPLETED (a span is emitted on tool end), so when the
+        // hang is inside a still-running tool the breadcrumb points at the prior one — worded
+        // "last completed tool" so the reader knows the stuck call may be the next, unfinished one.
+        const breadcrumb = lastTool
+            ? `last completed tool ${lastTool.name} ${Math.round((Date.now() - lastTool.at) / 1000)}s ago`
+            : 'no tool had completed yet';
+        const phaseBreakdown = Object.entries(phaseTimingsMs)
+            .map(([p, ms]) => `${p}=${Math.round(ms / 1000)}s`)
+            .join(', ');
+        if (killReason === 'inactivity') {
+            return {
+                message: redactSecrets(`${inactivityAbortMessage(this.limits.inactivityMs)} (likely hung in ${phase} phase; ${breadcrumb})`),
+                cause: 'inactivity-timeout',
+                detail: redactSecrets(`Phase timings: ${phaseBreakdown || '(none)'}. ${breadcrumb}.`),
+            };
+        }
+        if (killReason === 'max-duration') {
+            return {
+                message: redactSecrets(maxDurationAbortMessage(this.limits.maxDurationMs)),
+                cause: 'max-duration',
+                detail: redactSecrets(`Phase timings: ${phaseBreakdown || '(none)'}. ${breadcrumb}.`),
+            };
+        }
+        const raw = error instanceof Error ? error.message : String(error);
+        // A thrown error tagged with a structured cause (a git op / an upstream API call) keeps
+        // it; an untagged throw is a generic agent failure.
+        return {
+            message: redactSecrets(raw),
+            cause: failureCauseOf(error) ?? 'agent',
+            detail: redactSecrets(`${phaseBreakdown ? `Phase timings: ${phaseBreakdown}. ` : ''}Failed in ${phase} phase; ${breadcrumb}.`),
+        };
+    }
+}

package/dist/server.js

@@ -0,0 +1,135 @@
+import { timingSafeEqual } from 'node:crypto';
+import { createServer } from 'node:http';
+import { parseAgentJob } from './job.js';
+import { handleAgent } from './agent.js';
+import { redactSecrets } from './git.js';
+import { JobRegistry, loadRunnerLimits } from './runner.js';
+import { log } from './logger.js';
+// The container's HTTP entry point. The Worker addresses one instance per run and
+// POSTs a job to /jobs (the body's `kind` selects which agent runs); the harness
+// starts that job in the background (bounded by an inactivity + max-duration
+// watchdog) and returns a job id, which the Worker then polls via GET /jobs/{id}.
+// Nothing here holds long-lived secrets: the per-job GitHub + proxy tokens arrive
+// in the request body and live only for the duration of the job in an ephemeral
+// workspace.
+const PORT = Number(process.env.PORT ?? 8080);
+// Optional inbound auth. When HARNESS_SHARED_SECRET is set, every non-health
+// request must present a matching `x-harness-secret` header (constant-time
+// compared). When it is unset the harness behaves as before (open), so local/dev
+// and the existing acceptance flow keep working without configuration.
+// The direct callers send the matching header when the secret is configured: the
+// local Docker transport (LocalContainerRunnerTransport) and the Cloudflare
+// transport (CloudflareContainerTransport, which also injects the secret into the
+// container env). A self-hosted runner pool reaches the harness through its own
+// control plane, so its operator configures the secret pool-side.
+const SHARED_SECRET = process.env.HARNESS_SHARED_SECRET;
+const HEADER = 'x-harness-secret';
+/** Constant-time check of the shared-secret header; true when auth is disabled. */
+function authorized(req) {
+    if (!SHARED_SECRET)
+        return true;
+    const provided = req.headers[HEADER];
+    const got = Buffer.from(Array.isArray(provided) ? (provided[0] ?? '') : (provided ?? ''));
+    const want = Buffer.from(SHARED_SECRET);
+    // Length check first; timingSafeEqual requires equal-length buffers.
+    return got.length === want.length && timingSafeEqual(got, want);
+}
+// One registry per kind per container process. A run addresses its own container
+// instance (one Durable Object id per execution / bootstrap job) and dispatches its
+// sequence of step jobs to it; every kind shares the same watchdog/lifecycle but
+// produces a different result, so each gets its own registry keyed by the job id.
+const limits = loadRunnerLimits();
+/** Pair a body validator with a registry running its handler under the shared limits. */
+function defineKind(parse, handler, 
+// Non-secret correlation fields bound on the per-job logger (see JobRegistry.describe).
+describe) {
+    return {
+        parse,
+        registry: new JobRegistry(limits, handler, describe),
+    };
+}
+// The dispatch table. The harness now serves a SINGLE, manifest-driven kind: the
+// generic `agent` (the job body's `mode` — explore | coding — and its data select the
+// flow; WHAT the agent does is decided entirely by the backend). The per-kind bespoke
+// handlers (run/blueprint/spec/explore/merge/test/…) were strangled onto this one kind
+// and removed. A `POST /jobs` reads the body's `kind` to pick the entry; `GET /jobs/{id}`
+// checks every registry (job ids never collide across kinds). `kind` mirrors kernel's
+// `RunnerDispatchKind` (now also just `'agent'`); the harness keeps its own copy so the
+// image carries no runtime deps.
+const KINDS = {
+    agent: defineKind(parseAgentJob, handleAgent, (job) => ({
+        mode: job.mode,
+        repo: `${job.repo.owner}/${job.repo.name}`,
+        branch: job.branch,
+    })),
+};
+async function readBody(req) {
+    const chunks = [];
+    for await (const chunk of req)
+        chunks.push(chunk);
+    return Buffer.concat(chunks).toString('utf8');
+}
+function send(res, status, body) {
+    const payload = JSON.stringify(body);
+    res.writeHead(status, { 'content-type': 'application/json' });
+    res.end(payload);
+}
+const server = createServer((req, res) => {
+    void (async () => {
+        if (req.method === 'GET' && req.url === '/health') {
+            return send(res, 200, { status: 'ok' });
+        }
+        // All non-health endpoints are gated by the optional shared secret.
+        if (!authorized(req)) {
+            return send(res, 401, { error: 'unauthorized' });
+        }
+        // Poll a running/finished job: GET /jobs/{id}. Job ids are unique per kind, so
+        // check each registry in turn; the first hit wins.
+        if (req.method === 'GET' && req.url?.startsWith('/jobs/')) {
+            const id = decodeURIComponent(req.url.slice('/jobs/'.length));
+            for (const { registry } of Object.values(KINDS)) {
+                const view = registry.get(id);
+                if (view)
+                    return send(res, 200, view);
+            }
+            return send(res, 404, { error: 'job not found' });
+        }
+        // Start (or re-attach to) a job: POST /jobs with the kind in the body. The body's
+        // `kind` selects the validator + registry; the rest is that kind's job spec.
+        // Returns immediately with the job id; the caller polls GET /jobs/{id} for live
+        // subtask progress and the final result. Idempotent: a re-dispatched POST
+        // (a durable-driver replay) re-attaches to the job already running for the id
+        // rather than starting a duplicate.
+        if (req.method === 'POST' && req.url === '/jobs') {
+            let kind;
+            try {
+                const raw = JSON.parse(await readBody(req));
+                kind = raw.kind;
+                const entry = typeof kind === 'string' ? KINDS[kind] : undefined;
+                if (!entry) {
+                    return send(res, 404, { error: `unknown job kind '${String(kind)}'` });
+                }
+                const job = entry.parse(raw);
+                const view = entry.registry.start(job.jobId, job);
+                return send(res, 202, { jobId: view.id, state: view.state });
+            }
+            catch (error) {
+                // Parse failures (incl. host-allowlist rejection) are client errors → 400.
+                const message = redactSecrets(error instanceof Error ? error.message : String(error));
+                log.error('failed to start job', {
+                    kind: typeof kind === 'string' ? kind : undefined,
+                    error: message,
+                });
+                return send(res, 400, { error: message });
+            }
+        }
+        return send(res, 404, { error: 'not found' });
+    })();
+});
+// Only auto-listen when run as the entry point (tests import handleRun directly).
+if (process.env.NODE_ENV !== 'test') {
+    server.listen(PORT, () => {
+        console.log(`executor-harness listening on :${PORT}`);
+    });
+}
+export { server };