@pome-sh/cli 0.20.0 → 0.21.0

package/dist/src/runner/runTaskHosted.js

@@ -1,660 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-import { createHash, randomUUID } from "node:crypto";
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { dirname, join } from "node:path";
-import { runAgentCommand } from "./agentRunner.js";
-import { toTwinHttpEvent, writeRunArtifactsCore } from "../recorder/artifacts.js";
-import { VERDICT_ARTIFACT_VERSION, writeVerdictArtifact, } from "../hosted/evalResultCache.js";
-import { redactEvent, redactSecrets } from "../recorder/redaction.js";
-import { parseTaskFile } from "../task/parseTask.js";
-import { createHostedClient, perTwinReturnedByCloud, } from "../hosted/client.js";
-import { ensureMcpSuffix } from "../cli/session.js";
-import { redactJsonl, scoreFromFinalizeResponse, uploadRunBlobs, } from "../hosted/uploadAndFinalize.js";
-import { resolveRunAgentIdentity } from "../cli/agent-identity.js";
-import { HostedOrchError, HostedTrialError } from "../hosted/errors.js";
-import { scoreStatus, } from "../hosted/evalResultView.js";
-/** Build the agent subprocess env for a hosted run.
- *
- * Two layers, in this order:
- *
- *  1. The LEGACY block (origin/main parity). Every hosted run — regardless of
- *     which twins the session mounts — ALWAYS injected github + stripe vars
- *     pointing at the primary/bare `twin_url`, plus POME_TWIN_BASE_URL and the
- *     OTEL/task/run scaffolding. Byte-identical single-twin env is the bar, so
- *     this block reproduces main's exact keys, values, AND insertion order.
- *
- *  2. The per-twin fan-out OVERLAY (M3). For each twin actually in the session,
- *     set POME_<T>_REST_URL / POME_<T>_MCP_URL from `per_twin[T]` (distinct URLs
- *     per twin) plus its provider bearer. For github/stripe this overwrites the
- *     legacy values IN PLACE (same key → insertion order preserved); other twins
- *     append new keys.
- *
- * MCP URL rule (regression guard): only trust `per_twin[T].mcp_url` when the
- * cloud ACTUALLY RETURNED `per_twin` (`perTwinFromCloud`). A single-twin OLD
- * cloud returns no `per_twin`; the schema synthesizes one whose `mcp_url`
- * host-rewrites api.pome.sh→mcp.pome.sh with NO `/mcp` suffix. Trusting it would
- * drift POME_*_MCP_URL off main's `${twin_url}/mcp`. So for a synthesized entry
- * we construct `${api_url}/mcp`; only a cloud-returned value gets ensureMcpSuffix
- * (the same normalization `pome session create` applies). */
-export function buildAgentEnv(params) {
-    const { session, twins, perTwinFromCloud, prompt, otlpEndpoint, apiKey, agentId, runId, artifactsDir, slug, signalsPath, } = params;
-    const env = {
-        POME_TASK: prompt,
-        POME_TWIN_NAMES: twins.join(","),
-        POME_OTEL_EXPORTER_OTLP_ENDPOINT: otlpEndpoint,
-        POME_OTEL_EXPORTER_OTLP_HEADERS: `x-api-key=${apiKey}`,
-        OTEL_SERVICE_NAME: agentId ?? "pome-agent",
-        OTEL_RESOURCE_ATTRIBUTES: `pome.session_id=${session.session_id},pome.run_id=${runId}${agentId ? `,pome.agent_id=${agentId}` : ""}`,
-        // The twin base URL, same value the self-host docs teach agents to fall
-        // back on. Agent code written against the standalone docker twin
-        // reads POME_TWIN_BASE_URL with a 127.0.0.1:3333 fallback; without this
-        // injection that fallback fires on hosted runs and the agent probes a
-        // loopback port nothing listens on. Legacy = the primary twin's bare url.
-        POME_TWIN_BASE_URL: session.twin_url,
-        // Legacy unconditional block (origin/main parity). Injected on EVERY hosted
-        // run regardless of twin; the per-twin overlay below overwrites these in
-        // place for github/stripe when those twins are in the session.
-        POME_GITHUB_REST_URL: session.twin_url,
-        POME_GITHUB_MCP_URL: `${session.twin_url}/mcp`,
-        POME_GITHUB_TOKEN: session.provider_credentials.github?.token ?? session.agent_token,
-        POME_STRIPE_API_BASE: session.twin_url,
-        // Always use the JWT agent_token as the Stripe SDK's Bearer in hosted mode.
-        // The cloud also returns `provider_credentials.stripe.api_key` (an
-        // HMAC-signed `sk_test_pome_*`), but the pome-cloud proxy only accepts
-        // JWTs — it routes by verifying the bearer against `agent_token`. A
-        // stripe-style key short-circuits to the proxy's opaque 404 before the
-        // request reaches the twin pod. The Stripe SDK sends any opaque string as
-        // `Authorization: Bearer …`, and twin-stripe's dual-auth accepts JWTs
-        // natively, so the JWT round-trips cleanly end-to-end.
-        POME_STRIPE_API_KEY: session.agent_token,
-        POME_AUTH_TOKEN: session.agent_token,
-        POME_RUN_ID: runId,
-        POME_ARTIFACTS_DIR: join(artifactsDir, slug, runId),
-        POME_ADAPTER_SIGNALS_PATH: signalsPath,
-    };
-    for (const twin of twins) {
-        const upper = twin.toUpperCase();
-        const pt = session.per_twin?.[twin];
-        const apiUrl = pt?.api_url ?? session.twin_url;
-        const mcpUrl = perTwinFromCloud && pt?.mcp_url
-            ? ensureMcpSuffix(pt.mcp_url)
-            : `${apiUrl.replace(/\/$/, "")}/mcp`;
-        env[`POME_${upper}_REST_URL`] = apiUrl;
-        env[`POME_${upper}_MCP_URL`] = mcpUrl;
-        if (twin === "github") {
-            env.POME_GITHUB_TOKEN =
-                session.provider_credentials.github?.token ?? session.agent_token;
-        }
-        else if (twin === "stripe") {
-            env.POME_STRIPE_API_BASE = apiUrl;
-            env.POME_STRIPE_API_KEY = session.agent_token;
-        }
-        else if (twin === "slack") {
-            // Same rationale as the Stripe api-key line: the twin proxy verifies only
-            // the session JWT as bearer, so the agent's Slack bearer is the
-            // agent_token — NOT provider_credentials.slack.token, which the proxy
-            // would reject.
-            env.POME_SLACK_TOKEN = session.agent_token;
-        }
-        else if (twin === "gmail") {
-            // Gmail uses the Pome session JWT directly. This is an alias for agent
-            // SDKs that conventionally read a provider-specific bearer variable;
-            // there is deliberately no provider_credentials.gmail shape.
-            env.POME_GMAIL_TOKEN = session.agent_token;
-        }
-    }
-    return env;
-}
-export async function runTaskHosted(options) {
-    const scenario = await parseTaskFile(options.taskPath);
-    const taskSource = await readFile(options.taskPath, "utf8");
-    const taskHash = createHash("sha256").update(taskSource).digest("hex");
-    const artifactsDir = options.artifactsDir ?? "runs";
-    const startedAt = new Date().toISOString();
-    const client = options.client ??
-        createHostedClient({ baseUrl: options.hosted.baseUrl, apiKey: options.hosted.apiKey });
-    // F-819 — resolve the agent id + version from the manifest (`agent.slug` →
-    // `.pome/link.json` cache, team-gated, re-resolved by slug on miss) and the
-    // `agent.framework` (the dashboard "SDK badge", formerly `agent.sdk`). The
-    // `agent_version` stamps the session; `--agent-version` overrides it. All are
-    // optional during rollout — a manifest-less repo runs unattributed.
-    const identity = await resolveRunAgentIdentity({
-        startDir: dirname(options.taskPath),
-        apiBaseUrl: options.hosted.baseUrl,
-        agentVersionOverride: options.agentVersion,
-    });
-    const agentId = identity.agentId;
-    const agentVersion = identity.agentVersion;
-    const agentSdk = identity.framework ?? null;
-    // Per-run signals file. POME_ADAPTER_SIGNALS_PATH is injected into the agent
-    // env so adapter-rich correlator gets step/tool_call signals when the agent
-    // emits them; otherwise the heuristic correlator path runs over events alone.
-    const signalsDir = await mkdtemp(join(tmpdir(), "pome-signals-"));
-    const signalsPath = join(signalsDir, `${randomUUID()}.jsonl`);
-    // Touch the file so adapters that `appendFileSync` to it on first signal
-    // don't race the read in the empty case.
-    await writeFile(signalsPath, "");
-    // 1. Spawn cloud session — fails fast on auth/quota/orch. FDRS-636 trial
-    // groups mint all k sessions upfront (one shared group_id) and pass each
-    // one in as `premintedSession`; the single-run path mints its own here.
-    // Forward the resolved seed (sidecar -> inline JSON -> twin defaults, in that
-    // precedence; see parseTask.resolveSeedState) so the cloud doesn't need
-    // to re-extract it from the markdown. Prose `## Seed State` sections have no
-    // fenced JSON to extract — cloud would 422 with "no fenced code block".
-    const session = await createSessionOrExplain(client, options.premintedSession, {
-        taskSource,
-        twins: scenario.config.twins,
-        agentId,
-        agentVersion,
-        seed: scenario.seedState,
-    });
-    // Multi-twin (M3) locals. `twins[0]` is the primary twin: the source of the
-    // legacy top-level state blobs / URLs / criterion attribution.
-    const twins = scenario.config.twins;
-    const primaryTwin = twins[0];
-    const isMultiTwin = twins.length > 1;
-    // Each twin's own api base. Prefer the per-twin URL; fall back to the legacy
-    // bare `twin_url` (the primary) when the cloud didn't disambiguate per twin
-    // (older cloud, or the empty `per_twin` in tests). Keeps single-twin runs
-    // fetching from the same URL they always did.
-    const twinApiUrl = (twin) => session.per_twin?.[twin]?.api_url ?? session.twin_url;
-    // Use session_id as the local artifact-dir id. The cloud's run_id (assigned
-    // at /result time) is reported separately to the caller.
-    const runId = session.session_id;
-    try {
-        // 2. Capture initial twin state via bearer-protected /_pome/state — one
-        //    fetch per twin (each from its own api base). The primary twin's state
-        //    stays the legacy `stateInitial`.
-        const stateInitialByTwin = await fetchStateForTwins(client, twins, twinApiUrl, session.agent_token);
-        const stateInitial = stateInitialByTwin[primaryTwin];
-        // Agent telemetry → pome-cloud. The agent runs as a LOCAL subprocess even on
-        // hosted runs, and its LLM calls happen inside the SDK (they never traverse
-        // pome's capture-server proxy), so the agent emits its OWN `gen_ai` OTLP
-        // spans (via @pome-sh/adapter-claude-sdk). They post to the SESSION-SCOPED
-        // traces endpoint, which ingests them into THIS sim session's
-        // `otlp-spans.jsonl` blob; finalize rolls them up onto the run for the
-        // dashboard's "Agent telemetry" panel. Wire contract (sim-traces.ts):
-        //   - OTLP/HTTP JSON only (the adapter exporter is http/json).
-        //   - Auth: the team API key via `X-API-KEY`. sim-traces documents a
-        //     preferred `Bearer <agent_token>` path too, but every /v1/sessions
-        //     sub-router shares a `use("*", requireApiKey)` middleware that runs
-        //     before sim-traces' own auth and rejects a JWT bearer — so the
-        //     team-key fallback is the path that actually reaches ingest. The key
-        //     is the caller's own and the agent runs locally, so this is the same
-        //     trust boundary as POME_GITHUB_TOKEN below.
-        // POME_OTEL_EXPORTER_OTLP_ENDPOINT / _HEADERS are the pome-namespaced env the
-        // bundled adapter reads; with no endpoint the adapter emission is inert.
-        // POME_OTEL_COLLECTOR_URL fully overrides the endpoint for non-standard deploys.
-        const otlpEndpoint = process.env.POME_OTEL_COLLECTOR_URL?.trim() ||
-            new URL(`/v1/sessions/${session.session_id}/traces`, options.hosted.baseUrl).toString();
-        // 3. Run the agent. Env mirrors self-host so a customer's agent code
-        //    is twin-mode-agnostic. See `buildAgentEnv` for the origin/main legacy
-        //    block + per-twin fan-out overlay.
-        const env = buildAgentEnv({
-            session,
-            twins,
-            perTwinFromCloud: perTwinReturnedByCloud(session),
-            prompt: scenario.prompt,
-            otlpEndpoint,
-            apiKey: options.hosted.apiKey,
-            agentId,
-            runId,
-            artifactsDir,
-            slug: scenario.slug,
-            signalsPath,
-        });
-        const preflightTimeoutSeconds = Math.min(10, scenario.config.timeout);
-        const preflight = await runAgentCommand({
-            command: options.agentCommand,
-            env,
-            timeoutSeconds: preflightTimeoutSeconds,
-            preflight: true,
-        });
-        let agentResult = preflight;
-        if (preflight.exitCode === 0) {
-            // F-771 — the preflight probe appended its own signals (turns/steps/
-            // tool_calls) to the shared POME_ADAPTER_SIGNALS_PATH file. Those must
-            // NOT reach the uploaded signals.jsonl / usage ledger. Truncate before
-            // the real run so the blob counts real-run telemetry only. runAgentCommand
-            // awaits child exit, so the preflight's writes are already flushed here.
-            // Gated on preflight success: the preflight-failed → single-run path keeps
-            // its current behavior (it uploads the preflight's signals).
-            await writeFile(signalsPath, "");
-            agentResult = await runAgentCommand({
-                command: options.agentCommand,
-                env,
-                timeoutSeconds: scenario.config.timeout,
-            });
-        }
-        // FDRS-636 — group-trial failure semantics. An agent failure on a group
-        // trial ABANDONS the session with a machine error_code and throws,
-        // instead of finalizing: verdicts come from cloud evaluation only, and a
-        // crashed/timed-out trial must render as an errored row EXCLUDED from
-        // the fraction — never as a judged score over a garbage trace. Default
-        // (single-run) behavior below is untouched: it still finalizes so the
-        // run is visible on the dashboard.
-        if (options.abandonOnFailure &&
-            (agentResult.timedOut || agentResult.exitCode !== 0)) {
-            // Classify PREFLIGHT failures first: when the preflight never passed
-            // (non-zero exit OR killed at its own min(10, timeout)s cap), the real
-            // run never happened, so the code is preflight_failed — and a hung
-            // preflight quotes the cap that actually killed it, never the full
-            // scenario timeout. Only a real-run failure classifies as
-            // agent_timeout / agent_exit_nonzero.
-            //
-            // FDRS-667 — name the cause. The errored trial row renders the reason
-            // verbatim, so a bare "agent preflight failed" leaves the user with
-            // zero forensics (the k=5 first-publish e2e died five times on the
-            // same swallowed stderr). Append the agent's last stderr line to the
-            // exit-shaped failures; timeouts already name what killed them.
-            const failed = preflight.exitCode !== 0 ? preflight : agentResult;
-            const cause = stderrTail(failed.stderr);
-            const failure = preflight.exitCode !== 0
-                ? {
-                    errorCode: "preflight_failed",
-                    reason: preflight.timedOut
-                        ? `agent preflight timed out after ${preflightTimeoutSeconds}s`
-                        : `agent preflight failed${cause ? ` — ${cause}` : ""}`,
-                }
-                : agentResult.timedOut
-                    ? {
-                        errorCode: "agent_timeout",
-                        reason: `agent timed out after ${scenario.config.timeout}s`,
-                    }
-                    : {
-                        errorCode: "agent_exit_nonzero",
-                        reason: `agent exited non-zero (exit ${agentResult.exitCode})${cause ? ` — ${cause}` : ""}`,
-                    };
-            // FDRS-667 — an abandoned trial never reaches the artifact write in
-            // step 5, so land the agent's output where a completed trial's would
-            // go (runs/<slug>/<session>/): stdout.txt + stderr.log, redacted like
-            // writeRunArtifactsCore's copies. Best effort — forensics must never
-            // mask the trial's own error.
-            try {
-                const runDir = join(artifactsDir, scenario.slug, runId);
-                await mkdir(runDir, { recursive: true });
-                await writeFile(join(runDir, "stdout.txt"), redactSecrets(failed.stdout));
-                await writeFile(join(runDir, "stderr.log"), redactSecrets(failed.stderr));
-            }
-            catch {
-                // ignore — the thrown HostedTrialError below is the signal that matters
-            }
-            await abandonBestEffort(client, session.session_id, failure.errorCode);
-            throw new HostedTrialError(failure.reason, failure.errorCode);
-        }
-        // 4. Capture final state + recorder events — one fetch per twin, all in
-        //    parallel for latency. If any rejects (e.g., a twin pod restarted
-        //    between preflight and now), the run fails before finalize fires — the
-        //    cloud has no record. Recovery is "user reruns"; V1 doesn't retry
-        //    transient twin failures. Events from every twin are merged into one
-        //    ts-ordered stream (each line already carries its twin id from the
-        //    recorder; we only stamp it when a line lacks one).
-        const stateFinalByTwin = {};
-        const eventsByTwin = {};
-        await Promise.all(twins.map(async (twin) => {
-            const [sf, ev] = await Promise.all([
-                client.fetchState({ twinUrl: twinApiUrl(twin), agentToken: session.agent_token }),
-                client.fetchEvents({ twinUrl: twinApiUrl(twin), agentToken: session.agent_token }),
-            ]);
-            stateFinalByTwin[twin] = sf;
-            // Single-twin keeps the twin's events byte-identical to origin/main —
-            // no `twin` field is stamped. Only the multi-twin merge needs each line
-            // tagged so the interleaved stream stays attributable.
-            eventsByTwin[twin] = isMultiTwin
-                ? ev.map((e) => tagEventTwin(e, twin))
-                : ev;
-        }));
-        const stateFinal = stateFinalByTwin[primaryTwin];
-        // Single-twin keeps the twin's own event order untouched (byte-identical);
-        // multi-twin interleaves all twins' events by timestamp.
-        const events = isMultiTwin
-            ? mergeEventsByTimestamp(twins.flatMap((t) => eventsByTwin[t] ?? []))
-            : eventsByTwin[primaryTwin] ?? [];
-        // artifacts.ts still types `events` as the legacy twin/github RecorderEvent
-        // (no step_id / tool_call_id / state_delta fields). Runtime data is the
-        // same shape — cast to keep writeRunArtifactsCore happy without churning
-        // that signature here.
-        const legacyEvents = events;
-        // 5. Write local artifacts — RAW TRACE + state. ADR-013 / FDRS-657:
-        //    the OSS CLI never scores locally (no score.json is ever written).
-        //    Cloud is the authoritative judge; what /finalize returns is printed
-        //    to the terminal and — since FDRS-644 — also cached next to the
-        //    trace as a provenance-labeled verdict.json (step 11) so
-        //    `pome fix-prompt` can read the cloud's verdict offline.
-        const completedAt = new Date().toISOString();
-        const durationMs = Date.parse(completedAt) - Date.parse(startedAt);
-        const artifacts = await writeRunArtifactsCore({
-            artifactsDir,
-            runId,
-            scenario,
-            startedAt,
-            completedAt,
-            stdout: agentResult.stdout,
-            stderr: agentResult.stderr,
-            exitCode: agentResult.exitCode,
-            events: legacyEvents,
-            stateInitial,
-            stateFinal,
-        });
-        // Multi-twin (M3): writeRunArtifactsCore writes the primary twin's final
-        // state as the legacy `state_final.json`. Write each NON-primary twin's
-        // final state alongside it as `state_final.<twin>.json` so local inspection
-        // sees every twin's world. Best-effort — a disk failure here must not fail
-        // an otherwise-complete run.
-        if (isMultiTwin) {
-            for (const twin of twins) {
-                if (twin === primaryTwin)
-                    continue;
-                try {
-                    await writeFile(join(artifacts.runDir, `state_final.${twin}.json`), `${JSON.stringify(redactSecrets(stateFinalByTwin[twin]), null, 2)}\n`);
-                }
-                catch (err) {
-                    console.warn(`[pome] state_final.${twin}.json not written (${err instanceof Error ? err.message : String(err)})`);
-                }
-            }
-        }
-        // 6. FDRS-657 — NO local correlation. Correlation (like scoring/judging)
-        //    is a cloud responsibility; the OSS CLI only captures the raw trace.
-        //    Cloud correlates the uploaded events/signals server-side. The events
-        //    are uploaded exactly as captured (no local step_id back-fill).
-        // 7. Upload events.jsonl + state blobs + adapter signals.jsonl to cloud
-        //    storage in parallel. /finalize defaults the trace storage key to
-        //    the conventional `team-<>/session-<>/events.jsonl` path, so cloud's
-        //    judge finds the trace without an explicit override. State blobs
-        //    and signals have no conventional fallback today — without an
-        //    explicit override the judge sees "{}" for state files (FDRS-395)
-        //    and skips the adapter-rich correlator (F0-4 / L7). All four
-        //    uploads are best-effort: any failure leaves the corresponding
-        //    blob missing, the judge still runs against whatever it does have,
-        //    and the dashboard handles nulls gracefully.
-        // Wrap legacy RecorderEvents (no `kind` discriminator) into the unified
-        // FDRS-398 shape before upload — cloud's schema gate rejects raw legacy
-        // rows. Matches the wrap that `writeRunArtifactsCore` applies for the
-        // local events.jsonl file. Use `legacyEvents` (already cast) because
-        // `toTwinHttpEvent`'s input is the legacy twin/github RecorderEvent type.
-        const eventsJsonl = legacyEvents.map((e) => JSON.stringify(redactEvent(toTwinHttpEvent(e)))).join("\n") + "\n";
-        const stateInitialJson = JSON.stringify(redactSecrets(stateInitial));
-        const stateFinalJson = JSON.stringify(redactSecrets(stateFinal));
-        // Upload orchestration lives in ../hosted/uploadAndFinalize.ts (FDRS-656)
-        // so `pome eval` shares the exact best-effort semantics. Signals are
-        // read + redacted here (the tmp file is runner-owned); empty payloads
-        // skip the upload inside uploadRunBlobs. Read + redaction stay inside a
-        // guard so a redaction failure degrades to "signals skipped" — the
-        // pre-extraction contract — instead of aborting the whole hosted run.
-        let signalsJsonl = "";
-        try {
-            signalsJsonl = redactJsonl(await readFile(signalsPath, "utf8").catch(() => ""));
-        }
-        catch (err) {
-            console.warn(`[pome] signals.jsonl upload skipped (${err instanceof Error ? err.message : String(err)}); continuing with signals_storage_key=null`);
-        }
-        // D18.1 — meta.json is already redacted on disk (writeRunArtifactsCore's
-        // writeJson applies redactSecrets); read it back rather than re-deriving
-        // it so what's uploaded is byte-identical to the local artifact. A read
-        // failure degrades to an empty-object upload rather than aborting the
-        // run — meta.json is a best-effort sidecar, never load-bearing for the
-        // score.
-        const metaJson = await readFile(join(artifacts.runDir, "meta.json"), "utf8").catch(() => "{}");
-        // Multi-twin (M3): per-twin state blobs, keyed by twin id (includes the
-        // primary, whose blobs also land at the legacy top-level path). Undefined
-        // for single-twin, which keeps the flat state upload path unchanged.
-        const perTwinState = isMultiTwin
-            ? Object.fromEntries(twins.map((twin) => [
-                twin,
-                {
-                    initialJson: JSON.stringify(redactSecrets(stateInitialByTwin[twin])),
-                    finalJson: JSON.stringify(redactSecrets(stateFinalByTwin[twin])),
-                },
-            ]))
-            : undefined;
-        const uploaded = await uploadRunBlobs(client, session.session_id, {
-            eventsJsonl,
-            stateInitialJson,
-            stateFinalJson,
-            signalsJsonl,
-            metaJson,
-            twins: isMultiTwin ? twins : undefined,
-            perTwinState,
-        });
-        const eventsJsonlUrl = uploaded.eventsKey;
-        const stateKeys = {
-            initialKey: uploaded.stateInitialKey,
-            finalKey: uploaded.stateFinalKey,
-        };
-        const signalsKey = uploaded.signalsKey;
-        // 8. Finalize the run on cloud (ADR-013). Cloud loads the trace from
-        //    storage, calls the managed judge via AI Gateway, persists the run
-        //    row, and returns the authoritative score the dashboard records.
-        //    The CLI prints this score on the `score:` line.
-        // The finalize wire carries the canonical `code`/`model` kinds (F-778;
-        // needs shared-types >= 0.10.0 cloud-side, whose tolerant reader also
-        // still accepts the legacy `D`/`P` spellings from older released CLIs).
-        // Multi-twin (M3): carry the per-criterion twin attribution so the cloud
-        // judge checks each criterion against its twin's state. Absent = the
-        // primary twin (single-twin scenarios omit it entirely).
-        const criteriaDefs = scenario.criteria.map((c, idx) => ({
-            id: `crit_${idx}`,
-            text: c.text,
-            kind: c.type,
-            ...(c.twin ? { twin: c.twin } : {}),
-        }));
-        const stopReason = agentResult.timedOut
-            ? "agent_timeout"
-            : agentResult.exitCode === 0
-                ? "agent_exit_0"
-                : "agent_exit_nonzero";
-        const finalized = await client.finalize(session.session_id, {
-            stopReason,
-            exitCode: agentResult.exitCode ?? 0,
-            durationMs,
-            agentModel: options.agentModel ?? "unknown",
-            agentSdk,
-            criteria: criteriaDefs,
-            taskName: scenario.slug,
-            taskHash,
-            taskPrompt: scenario.prompt,
-            expectedBehavior: scenario.expectedBehavior,
-            // Explicit overrides when uploads succeeded; otherwise let cloud fall
-            // back. For events.jsonl the conventional `team-<>/session-<>/events.jsonl`
-            // path matches what /result-upload-url signs, so omitting traceStorageKey
-            // still resolves correctly. For state blobs there is no conventional
-            // fallback today (FDRS-395) — omitting them means cloud's judge sees
-            // "{}", which we accept as best-effort degradation.
-            traceStorageKey: eventsJsonlUrl ?? undefined,
-            stateInitialStorageKey: stateKeys.initialKey ?? undefined,
-            stateFinalStorageKey: stateKeys.finalKey ?? undefined,
-            signalsStorageKey: signalsKey ?? undefined,
-            // Multi-twin (M3): per-twin state storage keys (>=1 key per entry).
-            // Undefined for single-twin / an older cloud that returned no per_twin
-            // upload block — the cloud then scores the primary twin from the flat keys.
-            perTwinStateKeys: uploaded.perTwinStateKeys,
-        });
-        // 9. Synthesize the cloud verdict for terminal display + the exit code.
-        //    FDRS-657: the CLI never computes a verdict — this is the CLOUD's,
-        //    reshaped. Synthesis (incl. the F0-3 / L5 criteria_results handling
-        //    and the A1/FDRS-618 caveat) lives in scoreFromFinalizeResponse —
-        //    shared with `pome eval` (FDRS-656). Step 11 caches it to
-        //    verdict.json (FDRS-644); score.json stays never-written.
-        const score = scoreFromFinalizeResponse(finalized);
-        // 10. Map cloud score → exit code. F18 / F0-5: the old policy
-        //     ("agent failure trumps a passing score") collapsed an agent
-        //     non-zero exit AND an agent timeout to exit 3 ("auth error"),
-        //     which is wrong on two counts — it stole the auth-error slot
-        //     and it overrode the cloud judge's authoritative verdict on
-        //     sub-threshold runs (the F18 reproducer was a 50/100 hosted run
-        //     where the agent had an unhandled rejection; the test plan
-        //     expected exit 1).
-        //
-        //     Once /finalize has returned, the judge has seen the trace and
-        //     produced an authoritative score; the score is canonical. Use
-        //     the documented threshold logic: pass → 0, sub-threshold → 1.
-        //     Pre-finalize agent failures (auth, quota, twin spawn, exec
-        //     errors) take other code paths via thrown HostedAuthError /
-        //     HostedQuotaError / HostedOrchError and never reach this line.
-        //     F-925 — the score alone is not the verdict. A 100/100 run with a
-        //     criterion that never ran is `incomplete`, and exiting 0 on it is how
-        //     "the fix passed" came to mean "the check never ran". `scoreStatus`
-        //     applies the A5 guard `pome eval` has always applied; the FDRS-618
-        //     compat this used to preserve is already preserved inside
-        //     `scoreFromFinalizeResponse`, which sets can_pass true when the
-        //     response carries no criteria_results at all.
-        const verdict = scoreStatus(score, scenario.config.passThreshold);
-        const exitCode = verdict === "pass" ? 0 : 1;
-        // 11. FDRS-644 — cache the CLOUD verdict payload next to the raw trace
-        //     (verdict.json, provenance-labeled `source: "cloud-finalize"`).
-        //     Not a local score: evaluation stayed in the cloud; this records
-        //     what /finalize returned so `pome fix-prompt` can hand grouped
-        //     failure signatures to the user's coding agent offline. Best
-        //     effort: a disk failure degrades to "fix-prompt won't see this
-        //     trial", never fails a finalized run.
-        try {
-            await writeVerdictArtifact(artifacts.runDir, {
-                version: VERDICT_ARTIFACT_VERSION,
-                source: "cloud-finalize",
-                task_name: scenario.slug,
-                task_path: options.taskPath,
-                group_id: options.groupId ?? null,
-                session_id: session.session_id,
-                cloud_run_id: finalized.run_id,
-                cloud_dashboard_url: finalized.dashboard_url,
-                judge_model: score.judge_model,
-                score: finalized.score,
-                pass_threshold: scenario.config.passThreshold,
-                passed: exitCode === 0,
-                criteria_results: score.results,
-                duration_ms: durationMs,
-                finalized_at: new Date().toISOString(),
-            });
-        }
-        catch (err) {
-            console.warn(`[pome] verdict.json not written (${err instanceof Error ? err.message : String(err)}); pome fix-prompt won't see this trial`);
-        }
-        return {
-            scenario,
-            runId,
-            cloudRunId: finalized.run_id,
-            cloudDashboardUrl: finalized.dashboard_url,
-            artifacts,
-            score,
-            verdict,
-            exitCode,
-            durationMs,
-        };
-    }
-    catch (err) {
-        // FDRS-636 — a machinery crash on a group trial (twin state fetch,
-        // upload orchestration, finalize itself) also abandons the session so
-        // the reliability page shows an errored slot with a cause instead of a
-        // session stuck open until the sweeper. Must run BEFORE the `finally`
-        // DELETE below: abandon only transitions OPEN sessions, and error_code
-        // is lost once the row is terminal. Agent failures threw
-        // HostedTrialError above and already abandoned with a sharper code.
-        if (options.abandonOnFailure && !(err instanceof HostedTrialError)) {
-            await abandonBestEffort(client, session.session_id, "trial_crashed");
-        }
-        throw err;
-    }
-    finally {
-        // Best-effort teardown. TTL would reap anyway; explicit delete keeps
-        // the dashboard sessions list tidy.
-        // F-983: this teardown deliberately discards. On the success path the
-        // session is already `done` (finalize closed it) and the DELETE is a
-        // no-op; on the failure path we accept losing an ungraded tape rather
-        // than leaving open sessions to linger to TTL and pollute the
-        // reliability view. Flagged as known residue in the F-983 spec.
-        await client
-            .deleteSession(session.session_id, true, { discard: true })
-            .catch(() => undefined);
-        await rm(signalsDir, { recursive: true, force: true }).catch(() => undefined);
-    }
-}
-/** Mint the session unless one was pre-minted, mapping an old-cloud multi-twin
- *  rejection (422 `multi_twin_unsupported`) to a friendly hint. Every other
- *  error propagates unchanged so the existing auth/quota/orch mapping stands. */
-async function createSessionOrExplain(client, preminted, input) {
-    if (preminted)
-        return preminted;
-    try {
-        return await client.createSession(input);
-    }
-    catch (err) {
-        if (err instanceof HostedOrchError &&
-            err.status === 422 &&
-            (err.type === "multi_twin_unsupported" ||
-                /multi_twin_unsupported/i.test(err.message))) {
-            throw new HostedOrchError("this pome cloud does not support multi-twin sessions yet", err.requestId, err.status, err.type);
-        }
-        throw err;
-    }
-}
-/** Fetch `/_pome/state` for every twin (each from its own api base), keyed by
- *  twin id. */
-async function fetchStateForTwins(client, twins, twinApiUrl, agentToken) {
-    const byTwin = {};
-    await Promise.all(twins.map(async (twin) => {
-        byTwin[twin] = await client.fetchState({
-            twinUrl: twinApiUrl(twin),
-            agentToken,
-        });
-    }));
-    return byTwin;
-}
-/** Stamp a recorder event with its twin id ONLY when the line doesn't already
- *  carry one (the twin recorders set `twin` themselves; this is the merge-time
- *  backfill for any that don't). Returns the input untouched when it already
- *  carries a non-empty twin. */
-function tagEventTwin(event, twin) {
-    if (event && typeof event === "object") {
-        const existing = event.twin;
-        if (existing === undefined || existing === null || existing === "") {
-            return { ...event, twin };
-        }
-    }
-    return event;
-}
-/** Merge multiple twins' event streams into one timestamp-ordered stream.
- *  Stable: events with equal (or unparseable) `ts` keep their concatenation
- *  order, so per-twin ordering is preserved within a timestamp. */
-function mergeEventsByTimestamp(events) {
-    const tsOf = (e) => {
-        const ts = e.ts;
-        return typeof ts === "string" ? ts : "";
-    };
-    return events
-        .map((event, index) => ({ event, index }))
-        .sort((a, b) => {
-        const ta = tsOf(a.event);
-        const tb = tsOf(b.event);
-        if (ta < tb)
-            return -1;
-        if (ta > tb)
-            return 1;
-        return a.index - b.index;
-    })
-        .map(({ event }) => event);
-}
-/** FDRS-667 — the last non-empty stderr line, flattened and bounded, as the
- *  one named cause an errored trial row can render inline. Redacted with the
- *  same rules as the on-disk stderr.log copy. */
-function stderrTail(stderr) {
-    const last = stderr
-        .split("\n")
-        .map((line) => line.trim())
-        .filter(Boolean)
-        .at(-1);
-    if (!last)
-        return null;
-    const flat = redactSecrets(last.replace(/\s+/g, " ")).trim();
-    return flat.length > 160 ? `${flat.slice(0, 157)}…` : flat;
-}
-/** Abandon is a bookkeeping signal — a failure to deliver it must never
- *  mask the trial's own error. */
-async function abandonBestEffort(client, sessionId, errorCode) {
-    try {
-        await client.abandonSession(sessionId, { errorCode });
-    }
-    catch (err) {
-        console.warn(`[pome] session abandon skipped (${err instanceof Error ? err.message : String(err)}); the expiry sweeper will close ${sessionId}`);
-    }
-}