crysk 0.0.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +176 -0
- package/README.md +609 -3
- package/bin/pr-hero.js +166 -0
- package/config/models/anthropic.json +5 -0
- package/dist/SKILL-7e3725zn.md +496 -0
- package/dist/SKILL-7q35wcaz.md +66 -0
- package/dist/SKILL-9at22381.md +74 -0
- package/dist/SKILL-ex2hxxh3.md +134 -0
- package/dist/adjudicator-dtg2axnh.md +86 -0
- package/dist/admission-config.example-a9vqmmad.json +9 -0
- package/dist/ci-admission-camfbx2b.md +122 -0
- package/dist/cli.js +42292 -0
- package/dist/consolidate-ac0mnfyj.md +24 -0
- package/dist/deep-review-lifecycle-m698ybjp.md +362 -0
- package/dist/deep-review-logic-a6b1gczf.md +156 -0
- package/dist/deep-review-parity-0ec904db.md +120 -0
- package/dist/deep-review-reliability-qyakbjb9.md +125 -0
- package/dist/deep-review-resilience-3tz8y26e.md +118 -0
- package/dist/index.js +20190 -0
- package/dist/opencode-ci-7b0be64p.md +141 -0
- package/dist/review-refuter-j4e9gezy.md +71 -0
- package/dist/scout-yvp153q7.md +140 -0
- package/dist/summarizer-zca44nxa.md +18 -0
- package/dist/workflow-w1g9szna.yml +151 -0
- package/package.json +81 -4
- package/prompts/consolidate.md +24 -0
- package/prompts/default/PROVENANCE.md +18 -0
- package/prompts/default/deep-review-lifecycle.md +362 -0
- package/prompts/default/deep-review-logic.md +156 -0
- package/prompts/default/deep-review-parity.md +120 -0
- package/prompts/default/deep-review-reliability.md +125 -0
- package/prompts/default/deep-review-resilience.md +118 -0
- package/prompts/default/review-refuter.md +71 -0
- package/prompts/scout.md +140 -0
- package/prompts/summarizer.md +18 -0
- package/skills/crysk/SKILL.md +74 -0
- package/skills/crysk-ci-setup/SKILL.md +134 -0
- package/skills/crysk-ci-setup/assets/admission-config.example.json +9 -0
- package/skills/crysk-ci-setup/assets/workflow.yml +151 -0
- package/skills/crysk-ci-setup/references/ci-admission.md +122 -0
- package/skills/crysk-ci-setup/references/opencode-ci.md +141 -0
- package/skills/crysk-loop/SKILL.md +496 -0
- package/skills/crysk-loop/adjudicator.md +86 -0
- package/skills/crysk-pr/SKILL.md +66 -0
- package/src/agent-env.ts +1084 -0
- package/src/asset-manifest.ts +83 -0
- package/src/assets.d.ts +6 -0
- package/src/assets.ts +183 -0
- package/src/brand-paths.ts +155 -0
- package/src/brand.ts +255 -0
- package/src/ci/admission-ledger.ts +350 -0
- package/src/ci/classify-error.ts +73 -0
- package/src/ci/cli-record.ts +271 -0
- package/src/ci/force-comment.ts +39 -0
- package/src/ci/gates.ts +923 -0
- package/src/ci/outcome.ts +2181 -0
- package/src/ci/override.ts +187 -0
- package/src/ci/publish.ts +1639 -0
- package/src/ci/reporter.ts +626 -0
- package/src/ci/review-admission.ts +901 -0
- package/src/ci/review-levels.ts +310 -0
- package/src/ci/review-policy.ts +18 -0
- package/src/ci/review-risk.ts +184 -0
- package/src/ci/setup.ts +844 -0
- package/src/ci/trigger.ts +1353 -0
- package/src/cli.ts +348 -0
- package/src/commands/activity.ts +67 -0
- package/src/commands/ci-setup-questions.ts +397 -0
- package/src/commands/ci-setup.ts +381 -0
- package/src/commands/config.ts +99 -0
- package/src/commands/doctor.ts +108 -0
- package/src/commands/init.ts +139 -0
- package/src/commands/mcp.ts +10 -0
- package/src/commands/menu.ts +129 -0
- package/src/commands/post.ts +302 -0
- package/src/commands/skills.ts +83 -0
- package/src/commands/status.ts +173 -0
- package/src/commands/triage.ts +342 -0
- package/src/commands/uninstall.ts +37 -0
- package/src/commands/upgrade.ts +214 -0
- package/src/commands/usage.ts +98 -0
- package/src/compare/compare.ts +28 -0
- package/src/compare/floor-test.ts +471 -0
- package/src/compare/martian-adapter.ts +80 -0
- package/src/config/config.ts +267 -0
- package/src/corpus/corpus.ts +1094 -0
- package/src/corpus/preflight.ts +1789 -0
- package/src/diversity/accounting.ts +90 -0
- package/src/diversity/adjudication.ts +145 -0
- package/src/diversity/admission.ts +60 -0
- package/src/diversity/benchmark.ts +195 -0
- package/src/diversity/clustering.ts +285 -0
- package/src/diversity/errors.ts +7 -0
- package/src/diversity/identity.ts +285 -0
- package/src/diversity/index.ts +11 -0
- package/src/diversity/pipeline-integration.ts +448 -0
- package/src/diversity/projection.ts +57 -0
- package/src/diversity/self-contained.ts +68 -0
- package/src/diversity/target-validation.ts +179 -0
- package/src/doctor-migration.ts +209 -0
- package/src/doctor.ts +723 -0
- package/src/errors.ts +83 -0
- package/src/execution/admission.ts +75 -0
- package/src/execution/atomic-write.ts +35 -0
- package/src/execution/attempt-evidence.ts +415 -0
- package/src/execution/bucket-id.ts +136 -0
- package/src/execution/concurrency-limiter.ts +175 -0
- package/src/execution/contracts.ts +443 -0
- package/src/execution/failure-classification.ts +42 -0
- package/src/execution/failure-policy.ts +240 -0
- package/src/execution/harness.ts +2548 -0
- package/src/execution/settlement.ts +348 -0
- package/src/execution/spawned-process.ts +66 -0
- package/src/execution/spend-limiter.ts +424 -0
- package/src/execution/step-artifacts.ts +55 -0
- package/src/execution/usage-normalized.ts +469 -0
- package/src/git/git.ts +396 -0
- package/src/git/identity.ts +64 -0
- package/src/git/refs.ts +140 -0
- package/src/home-preflight.ts +443 -0
- package/src/home.ts +243 -0
- package/src/ignore-file.ts +336 -0
- package/src/ignore-read.ts +118 -0
- package/src/index.ts +34 -0
- package/src/legacy-launchd.ts +29 -0
- package/src/mcp/mcp.ts +150 -0
- package/src/mcp/preflight.ts +407 -0
- package/src/menu-context.ts +192 -0
- package/src/model/catalog.ts +251 -0
- package/src/model/exec-snapshots.ts +390 -0
- package/src/model/free-discovery.ts +302 -0
- package/src/model/provider-capabilities.ts +814 -0
- package/src/model/routing.ts +600 -0
- package/src/pr/admission.ts +847 -0
- package/src/pr/ci-admission-gate.ts +665 -0
- package/src/pr/ci-publish.ts +76 -0
- package/src/pr/discovery.ts +432 -0
- package/src/pr/inline.ts +507 -0
- package/src/pr/plan.ts +366 -0
- package/src/pr/posting.ts +115 -0
- package/src/pr/pr-status.ts +787 -0
- package/src/pr/pr.ts +3642 -0
- package/src/pr/preflight.ts +1031 -0
- package/src/pr/publish-outcome.ts +361 -0
- package/src/pr/range.ts +119 -0
- package/src/pr/reverts-preflight.ts +606 -0
- package/src/pr/reverts.ts +377 -0
- package/src/pr/review-pr.ts +927 -0
- package/src/pr/status.ts +154 -0
- package/src/pr/target.ts +388 -0
- package/src/pr/teardown.ts +437 -0
- package/src/pr/worktree-setup.ts +60 -0
- package/src/production-runtime.ts +1877 -0
- package/src/progress.ts +487 -0
- package/src/rereview/adjudicate.ts +89 -0
- package/src/rereview/classify.ts +362 -0
- package/src/rereview/identity.ts +211 -0
- package/src/rereview/plan.ts +308 -0
- package/src/rereview/prepare.ts +1025 -0
- package/src/rereview/state.ts +767 -0
- package/src/rereview/verify.ts +241 -0
- package/src/review/boundary.ts +105 -0
- package/src/review/consolidate.ts +246 -0
- package/src/review/dedupe.ts +368 -0
- package/src/review/drafts.ts +448 -0
- package/src/review/findings-conformance.ts +120 -0
- package/src/review/findings.ts +714 -0
- package/src/review/pipeline.ts +3977 -0
- package/src/review/preflight.ts +3575 -0
- package/src/review/prompt-set.ts +177 -0
- package/src/review/proof-refs.ts +187 -0
- package/src/review/report.ts +1821 -0
- package/src/review/review.ts +490 -0
- package/src/review/root-cause.ts +120 -0
- package/src/review/route-preflight.ts +289 -0
- package/src/review/run.ts +601 -0
- package/src/review/scout.ts +323 -0
- package/src/review/size-gate.ts +595 -0
- package/src/review/spec.ts +235 -0
- package/src/review/step-runner.ts +233 -0
- package/src/runner-authority.ts +596 -0
- package/src/security/binding-policy.ts +115 -0
- package/src/security/credential-broker.ts +773 -0
- package/src/security/evidence-redaction.ts +62 -0
- package/src/security/execution-authority.ts +27 -0
- package/src/security/redact.ts +71 -0
- package/src/security/workspace-read-broker.ts +466 -0
- package/src/server/client.ts +190 -0
- package/src/server/preflight.ts +217 -0
- package/src/server/server.ts +327 -0
- package/src/signal-handler.ts +50 -0
- package/src/store/activity.ts +413 -0
- package/src/store/backfill-preflight.ts +49 -0
- package/src/store/backfill.ts +141 -0
- package/src/store/gc-preflight.ts +206 -0
- package/src/store/gc.ts +424 -0
- package/src/store/metrics-preflight.ts +229 -0
- package/src/store/metrics.ts +236 -0
- package/src/store/preflight.ts +748 -0
- package/src/store/store.ts +693 -0
- package/src/system-tools.ts +355 -0
- package/src/transport-registry.ts +1124 -0
- package/src/transports/claude-code-cli.ts +1348 -0
- package/src/transports/opencode-admission.ts +515 -0
- package/src/transports/opencode-client.ts +3328 -0
- package/src/transports/opencode-evidence.ts +557 -0
- package/src/transports/opencode-mcp.ts +248 -0
- package/src/transports/opencode-sdk.ts +2294 -0
- package/src/transports/opencode-server.ts +535 -0
- package/src/triage/adjudication.ts +113 -0
- package/src/triage/reply.ts +125 -0
- package/src/triage/triage.ts +255 -0
- package/src/ui/activity.ts +173 -0
- package/src/ui/config-edit.ts +816 -0
- package/src/ui/config.ts +462 -0
- package/src/ui/menu.ts +1325 -0
- package/src/ui/plan.ts +1039 -0
- package/src/ui/primitives.ts +300 -0
- package/src/ui/progress.ts +298 -0
- package/src/ui/result.ts +552 -0
- package/src/ui/review-menu.ts +312 -0
- package/src/ui/select.ts +528 -0
- package/src/ui/tree.ts +178 -0
- package/src/uninstaller.ts +479 -0
- package/src/updater.ts +836 -0
- package/src/usage.ts +77 -0
- package/src/watch/preflight.ts +1389 -0
- package/src/watch/watch.ts +1235 -0
- package/src/wizard.ts +877 -0
|
@@ -0,0 +1,2548 @@
|
|
|
1
|
+
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { releaseExecutionSnapshot } from "#model/exec-snapshots";
|
|
4
|
+
import {
|
|
5
|
+
type ExecutableAllowlistEntry,
|
|
6
|
+
verifyExecutableAuthority,
|
|
7
|
+
} from "#model/provider-capabilities";
|
|
8
|
+
// Lifecycle ownership (§2 docs/multi-runtime-model-diversity-design.md):
|
|
9
|
+
// HARNESS — StepSpec.timeoutMs watchdog, cancellation coordinator, retry/
|
|
10
|
+
// parse, write leases + settlement receipts, event sink, spend reservations,
|
|
11
|
+
// concurrency admission, credential projection destroy AFTER transport teardown.
|
|
12
|
+
// TRANSPORT — provider/process mechanics only: honor AbortSignal, emit bounded
|
|
13
|
+
// protocol events, return TransportOutcome, classify provider/transport causes.
|
|
14
|
+
// TransportRequest deliberately omits timeoutMs, parser, retry, and artifacts.
|
|
15
|
+
import { isVacuousHunt } from "#review/drafts";
|
|
16
|
+
import type {
|
|
17
|
+
RetryInfo,
|
|
18
|
+
StepResult,
|
|
19
|
+
StepRunner,
|
|
20
|
+
StepSpec,
|
|
21
|
+
} from "#review/step-runner";
|
|
22
|
+
import type {
|
|
23
|
+
CredentialBroker,
|
|
24
|
+
CredentialProjection,
|
|
25
|
+
} from "../security/credential-broker";
|
|
26
|
+
import { CredentialProjectionError } from "../security/credential-broker";
|
|
27
|
+
import { redactEvidenceText } from "../security/evidence-redaction";
|
|
28
|
+
import { redactDiagnostic } from "../security/redact";
|
|
29
|
+
import { WorkspaceReadBroker } from "../security/workspace-read-broker";
|
|
30
|
+
import type { TransportRegistry } from "../transport-registry";
|
|
31
|
+
import { DefaultTransportRegistry } from "../transport-registry";
|
|
32
|
+
import { ClaudeCodeCliTransport } from "../transports/claude-code-cli";
|
|
33
|
+
import { zeroUsage } from "../usage";
|
|
34
|
+
import type { AttemptAdmissionGate, AttemptLease } from "./admission";
|
|
35
|
+
import { writeJsonAtomically } from "./atomic-write";
|
|
36
|
+
import {
|
|
37
|
+
freezeAttemptEvidenceIdentity,
|
|
38
|
+
persistAttemptEvidence,
|
|
39
|
+
} from "./attempt-evidence";
|
|
40
|
+
import {
|
|
41
|
+
BucketBreakerTrippedError,
|
|
42
|
+
ConcurrencyAdmissionAbortedError,
|
|
43
|
+
} from "./concurrency-limiter";
|
|
44
|
+
import {
|
|
45
|
+
type AsyncEventSink,
|
|
46
|
+
type AuthEvent,
|
|
47
|
+
isTransportFailureCause,
|
|
48
|
+
type ProviderTransport,
|
|
49
|
+
type StepAdmissionGate,
|
|
50
|
+
type TransportFailureCause,
|
|
51
|
+
type TransportOutcome,
|
|
52
|
+
type TransportRequest,
|
|
53
|
+
} from "./contracts";
|
|
54
|
+
import type { FailureClass } from "./failure-classification";
|
|
55
|
+
import {
|
|
56
|
+
type CauseResolution,
|
|
57
|
+
decideRetryDisposition,
|
|
58
|
+
type FailureCause,
|
|
59
|
+
legacyClassificationFromCause,
|
|
60
|
+
type RetryState,
|
|
61
|
+
resolveFailureCause,
|
|
62
|
+
} from "./failure-policy";
|
|
63
|
+
import {
|
|
64
|
+
type ActiveSession,
|
|
65
|
+
createSettlement,
|
|
66
|
+
DEFAULT_CANCELLATION_DEADLINE_MS,
|
|
67
|
+
HARNESS_GRACE_MARGIN_MS,
|
|
68
|
+
type SettlementReceipt,
|
|
69
|
+
type SettlementSession,
|
|
70
|
+
synthesizeInternalFailure,
|
|
71
|
+
synthesizeUnconfirmed,
|
|
72
|
+
} from "./settlement";
|
|
73
|
+
import {
|
|
74
|
+
beginStep,
|
|
75
|
+
nextCycle,
|
|
76
|
+
type ReserveToken,
|
|
77
|
+
type SettlementDecision,
|
|
78
|
+
type SpendLedger,
|
|
79
|
+
type SpendReservation,
|
|
80
|
+
SpendReservationFencedError,
|
|
81
|
+
settlementFromUsage,
|
|
82
|
+
} from "./spend-limiter";
|
|
83
|
+
import {
|
|
84
|
+
attemptLogPath,
|
|
85
|
+
FORMAT_RETRY_REMINDER,
|
|
86
|
+
settlementReceiptPath,
|
|
87
|
+
} from "./step-artifacts";
|
|
88
|
+
import type { NormalizedUsage } from "./usage-normalized";
|
|
89
|
+
import {
|
|
90
|
+
envBillsMetered,
|
|
91
|
+
normalizeUnavailableUsage,
|
|
92
|
+
projectLegacyUsage,
|
|
93
|
+
sumNormalizedUsage,
|
|
94
|
+
} from "./usage-normalized";
|
|
95
|
+
|
|
96
|
+
// §7 line 416: injected sleep for the `rate_limit` capped-exponential
|
|
97
|
+
// backoff — a real timer in production, a recording stub in tests (§13 line
|
|
98
|
+
// 746: no offline test may actually wait out a delay, let alone the 60s cap).
|
|
99
|
+
const DEFAULT_SLEEP = (ms: number): Promise<void> =>
|
|
100
|
+
new Promise((resolve) => setTimeout(resolve, ms));
|
|
101
|
+
|
|
102
|
+
const DEFAULT_ATTEMPT_ADMISSION_TIMEOUT_MS = 30 * 60 * 1000;
|
|
103
|
+
// Credential projection must not hang the step indefinitely when a broker
|
|
104
|
+
// blocks on interactive Keychain prompts or a stalled reader.
|
|
105
|
+
const DEFAULT_CREDENTIAL_PROJECTION_TIMEOUT_MS = 60_000;
|
|
106
|
+
|
|
107
|
+
async function projectCredentialWithBudget(
|
|
108
|
+
broker: CredentialBroker,
|
|
109
|
+
input: Parameters<CredentialBroker["project"]>[0],
|
|
110
|
+
options: { readonly timeoutMs: number; readonly signal?: AbortSignal },
|
|
111
|
+
): Promise<CredentialProjection> {
|
|
112
|
+
let outcome: "pending" | "budget" | undefined;
|
|
113
|
+
const guarded = broker.project(input).then(async (projection) => {
|
|
114
|
+
if (outcome === "budget") {
|
|
115
|
+
await projection.destroy().catch(() => {});
|
|
116
|
+
throw new Error("credential_projection_superseded");
|
|
117
|
+
}
|
|
118
|
+
outcome = "pending";
|
|
119
|
+
return projection;
|
|
120
|
+
});
|
|
121
|
+
guarded.catch(() => {});
|
|
122
|
+
|
|
123
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
124
|
+
let onAbort: (() => void) | undefined;
|
|
125
|
+
const signal = options.signal;
|
|
126
|
+
|
|
127
|
+
const cleanup = (): void => {
|
|
128
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
129
|
+
if (signal !== undefined && onAbort !== undefined) {
|
|
130
|
+
signal.removeEventListener("abort", onAbort);
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
try {
|
|
135
|
+
if (signal?.aborted) {
|
|
136
|
+
throw new Error("credential_projection_aborted");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
140
|
+
timer = setTimeout(
|
|
141
|
+
() => reject(new Error("credential_projection_timed_out")),
|
|
142
|
+
options.timeoutMs,
|
|
143
|
+
);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const racers: Promise<CredentialProjection>[] = [guarded, timeoutPromise];
|
|
147
|
+
|
|
148
|
+
if (signal !== undefined) {
|
|
149
|
+
const abortPromise = new Promise<never>((_, reject) => {
|
|
150
|
+
onAbort = () => reject(new Error("credential_projection_aborted"));
|
|
151
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
152
|
+
});
|
|
153
|
+
racers.push(abortPromise);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return await Promise.race(racers);
|
|
157
|
+
} catch (error) {
|
|
158
|
+
if (outcome === undefined) {
|
|
159
|
+
outcome = "budget";
|
|
160
|
+
}
|
|
161
|
+
throw error;
|
|
162
|
+
} finally {
|
|
163
|
+
cleanup();
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function cancellableSleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
168
|
+
if (ms <= 0) return Promise.resolve();
|
|
169
|
+
if (signal?.aborted) return Promise.resolve();
|
|
170
|
+
return new Promise((resolve) => {
|
|
171
|
+
const onDone = () => {
|
|
172
|
+
signal?.removeEventListener("abort", onAbort);
|
|
173
|
+
clearTimeout(timer);
|
|
174
|
+
resolve();
|
|
175
|
+
};
|
|
176
|
+
const timer = setTimeout(onDone, ms);
|
|
177
|
+
const onAbort = () => onDone();
|
|
178
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// D1-08 PR5a (§9.2 Open Question): deriving a real bucket id from a
|
|
183
|
+
// resolved CredentialProjection's bucketScope is a caller concern (PR3's
|
|
184
|
+
// deriveBucketId is available to any caller that has one), not this
|
|
185
|
+
// harness's. This sentinel is what a configured AttemptAdmissionGate has to
|
|
186
|
+
// key on when no caller has derived a real one yet — every step run by one
|
|
187
|
+
// harness instance still coarsens onto ONE bucket, matching the "unknown
|
|
188
|
+
// scope coarsens" rule at the harness boundary too.
|
|
189
|
+
const DEFAULT_RATE_LIMIT_BUCKET_ID = "default";
|
|
190
|
+
|
|
191
|
+
export interface StepExecutionHarnessOptions {
|
|
192
|
+
readonly workspaceRoot?: string;
|
|
193
|
+
readonly executableAllowlist?: readonly ExecutableAllowlistEntry[];
|
|
194
|
+
readonly binaryPath?: string;
|
|
195
|
+
// Where verified execution snapshots are written. Tests point it at a temp
|
|
196
|
+
// dir so they never touch the real $TMPDIR; production leaves it unset.
|
|
197
|
+
readonly executableSnapshotDir?: string;
|
|
198
|
+
readonly admissionGate?: StepAdmissionGate;
|
|
199
|
+
readonly registry?: TransportRegistry;
|
|
200
|
+
readonly transport?: ProviderTransport;
|
|
201
|
+
readonly onAuthEvent?: (event: AuthEvent) => void;
|
|
202
|
+
readonly spawnFn?: typeof Bun.spawn;
|
|
203
|
+
// §6.1 D1-05: when set, credentials are projected per run and the child's
|
|
204
|
+
// HOME/TMPDIR/config identity come from the ephemeral projection.
|
|
205
|
+
readonly credentialBroker?: CredentialBroker;
|
|
206
|
+
// Source for child-env projection; injectable so tests never touch the
|
|
207
|
+
// real process environment.
|
|
208
|
+
readonly childEnv?: Readonly<Record<string, string | undefined>>;
|
|
209
|
+
// §5.3: user/pipeline cancellation (SIGINT/SIGTERM role) enters the same
|
|
210
|
+
// settlement coordinator as the watchdog.
|
|
211
|
+
readonly signal?: AbortSignal;
|
|
212
|
+
// Injectable ISO-timestamp source for §5.1 receipt timestamps (offline tests).
|
|
213
|
+
readonly nowIso?: () => string;
|
|
214
|
+
// Test override for the fixed §5.3 step-4 margin; production default is the
|
|
215
|
+
// spec value HARNESS_GRACE_MARGIN_MS = 1000 ms.
|
|
216
|
+
readonly graceMarginMs?: number;
|
|
217
|
+
// Observer for every settled ActiveSession (ledger/tests); observation only,
|
|
218
|
+
// never allowed to alter settlement.
|
|
219
|
+
readonly onSessionSettled?: (info: {
|
|
220
|
+
readonly session: ActiveSession;
|
|
221
|
+
readonly settlement: SettlementSession;
|
|
222
|
+
readonly receipt: SettlementReceipt;
|
|
223
|
+
}) => void;
|
|
224
|
+
// D1-08 PR0 (§7 line 416): backoff delay for a `rate_limit` retry_after
|
|
225
|
+
// disposition. Defaults to a real timer; tests inject a recording stub.
|
|
226
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
227
|
+
// D1-08 PR5a (§9.2): the attempt-scoped, lease-returning admission gate —
|
|
228
|
+
// admitted per ATTEMPT inside the retry loop, distinct from
|
|
229
|
+
// `admissionGate` (StepAdmissionGate, admitted once per STEP before the
|
|
230
|
+
// loop even starts and left untouched, D4). Left unconfigured, attempts
|
|
231
|
+
// are gated exactly as before this slice — the concurrency limiter stays
|
|
232
|
+
// a dormant module (D1-08 PR3's own framing) until a caller opts in. NO
|
|
233
|
+
// spend-ledger call is made here — that's PR5b.
|
|
234
|
+
readonly attemptAdmissionGate?: AttemptAdmissionGate;
|
|
235
|
+
// The rate-limit bucket every attempt this harness instance runs admits
|
|
236
|
+
// into. See DEFAULT_RATE_LIMIT_BUCKET_ID for why an explicit id is
|
|
237
|
+
// optional.
|
|
238
|
+
readonly rateLimitBucketId?: string;
|
|
239
|
+
// D1-08 PR5b (§9.1): the transactional spend reservation ledger. Left
|
|
240
|
+
// unconfigured, attempts reserve nothing — the ledger stays a dormant
|
|
241
|
+
// module exactly like `attemptAdmissionGate` before PR5a wired it, and
|
|
242
|
+
// every existing caller (PR5a's own tests included) keeps its ledger-free
|
|
243
|
+
// shape unmodified.
|
|
244
|
+
readonly spendLedger?: SpendLedger;
|
|
245
|
+
// What each attempt reserves BEFORE its actual cost is known. Estimating a
|
|
246
|
+
// real per-step dollar figure from prompt/model/provider pricing is a
|
|
247
|
+
// caller concern this slice does not attempt (same framing as PR5a's own
|
|
248
|
+
// `rateLimitBucketId` default: deriving the real number belongs to
|
|
249
|
+
// whoever composes the run, not to this harness). Defaults to 0 so an
|
|
250
|
+
// unconfigured caller reserves nothing rather than an invented ceiling.
|
|
251
|
+
readonly reservedUsdPerAttempt?: number;
|
|
252
|
+
// #279: the BINDING's own reserve decision — true for a metered or
|
|
253
|
+
// free-credential route (production-runtime.ts's `reservesSpend`),
|
|
254
|
+
// independent of whether `spendLedger` is configured at all. Defaults to
|
|
255
|
+
// `true` so every EXISTING caller that configures a `spendLedger`
|
|
256
|
+
// directly (every harness-level spend test predating this option) keeps
|
|
257
|
+
// reserving on every attempt exactly as it did when the only gate was
|
|
258
|
+
// "is a ledger configured" — this option adds a way to say "reserve a
|
|
259
|
+
// ledger-backed run's attempts SELECTIVELY", it does not change the
|
|
260
|
+
// ledger-free-by-default meaning `spendLedger` itself already carries.
|
|
261
|
+
// production-runtime.ts is the one caller that now passes `spendLedger`
|
|
262
|
+
// UNCONDITIONALLY (so a subscription binding's harness can still open the
|
|
263
|
+
// #279 degraded-fence reservation below) and sets this explicitly to
|
|
264
|
+
// `false` for a subscription binding, `true` for metered/free.
|
|
265
|
+
readonly reservesSpend?: boolean;
|
|
266
|
+
// #279: the bucket a METERED claude-code binding of this SAME
|
|
267
|
+
// provider+credential would compute (production-runtime.ts's
|
|
268
|
+
// `bindingBucketId`, mirrored with the metered kind) — used ONLY for the
|
|
269
|
+
// one attempt whose Keychain projection degraded onto an ambient
|
|
270
|
+
// key/token that bills metered (never for a `reservesSpend` reservation,
|
|
271
|
+
// which always uses `rateLimitBucketId`). Left unconfigured, that one
|
|
272
|
+
// reservation falls back to `rateLimitBucketId` — the binding's own
|
|
273
|
+
// bucket — rather than opening no reservation at all; production always
|
|
274
|
+
// configures this, so the fallback only ever applies to a caller (a test)
|
|
275
|
+
// that does not care which bucket a degraded reservation lands on.
|
|
276
|
+
readonly degradedProjectionBucketId?: string;
|
|
277
|
+
// When `attemptAdmissionGate` is configured but no pipeline cancel
|
|
278
|
+
// `signal` is wired, acquire() uses AbortSignal.timeout(ms) so a stuck
|
|
279
|
+
// FIFO queue cannot hang the run forever.
|
|
280
|
+
readonly attemptAdmissionTimeoutMs?: number;
|
|
281
|
+
readonly credentialProjectionTimeoutMs?: number;
|
|
282
|
+
// Production leased transports: tear down the shared client/server before
|
|
283
|
+
// credential projection destroy when this step is the last active lease.
|
|
284
|
+
readonly onBeforeCredentialProjectionDestroy?: () => Promise<void>;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// WHY an enumerated passthrough instead of `process.env` verbatim and instead
|
|
288
|
+
// of `{}`: `{}` is what Phase 1 shipped and it broke every real run — a claude
|
|
289
|
+
// CLI spawned without HOME cannot find its OAuth credentials ("Not logged in")
|
|
290
|
+
// and dies in seconds. The full ambient environment is the opposite failure:
|
|
291
|
+
// it leaks operator config into the child. Until credential projection (D1-05)
|
|
292
|
+
// lands, the isolation projection carries exactly the keys auth, network and
|
|
293
|
+
// binary execution need. GIT_* is deliberately absent — repo redirection must
|
|
294
|
+
// not reach the child (the workspace broker enforces the same rule for its own
|
|
295
|
+
// git spawns).
|
|
296
|
+
const ENV_PASSTHROUGH = [
|
|
297
|
+
"HOME",
|
|
298
|
+
// WHY USER/LOGNAME: on macOS the CLI reads OAuth credentials from the
|
|
299
|
+
// Keychain under an account named after the user; spawned without USER it
|
|
300
|
+
// cannot resolve that entry and reports "Not logged in" even with HOME set.
|
|
301
|
+
// Bisected live against the real CLI (2026-08-26): HOME+PATH+TERM fails,
|
|
302
|
+
// adding USER alone succeeds.
|
|
303
|
+
"USER",
|
|
304
|
+
"LOGNAME",
|
|
305
|
+
"CLAUDE_CONFIG_DIR",
|
|
306
|
+
"PATH",
|
|
307
|
+
"LANG",
|
|
308
|
+
"LC_ALL",
|
|
309
|
+
"TERM",
|
|
310
|
+
"TMPDIR",
|
|
311
|
+
"HTTP_PROXY",
|
|
312
|
+
"HTTPS_PROXY",
|
|
313
|
+
"NO_PROXY",
|
|
314
|
+
"SSL_CERT_FILE",
|
|
315
|
+
"SSL_CERT_DIR",
|
|
316
|
+
"ANTHROPIC_API_KEY",
|
|
317
|
+
"ANTHROPIC_AUTH_TOKEN",
|
|
318
|
+
"ANTHROPIC_BASE_URL",
|
|
319
|
+
// The subscription credential, and it was MISSING here while the capability
|
|
320
|
+
// gate (§11) already accepted it as proof of auth — the gate and the spawn
|
|
321
|
+
// disagreeing about the same variable. pr-hero's first real CI self-review
|
|
322
|
+
// (2026-08-27) passed the gate, passed the size gate, then lost all three
|
|
323
|
+
// hunters and the summarizer in three seconds to "Not logged in · Please run
|
|
324
|
+
// /login", $0.00 spent, because this projection dropped the token the action
|
|
325
|
+
// had just handed it. It is the same credential class as ANTHROPIC_API_KEY
|
|
326
|
+
// directly above, and it is the ONLY route on Linux: credential projection
|
|
327
|
+
// needs darwin + /usr/bin/security, so every CI runner falls back to exactly
|
|
328
|
+
// this enumerated passthrough. When a projection IS active it is stripped
|
|
329
|
+
// again by PROJECTION_OWNED_KEYS below — that path holds the same
|
|
330
|
+
// credential as a file and must not also see an ambient copy.
|
|
331
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
332
|
+
] as const;
|
|
333
|
+
|
|
334
|
+
// WHY a projection strips more than HOME/TMPDIR/CLAUDE_CONFIG_DIR: a
|
|
335
|
+
// credential projection's promise is that the child runs on ONE freshly
|
|
336
|
+
// projected credential and nothing else, and until 2026-09-02 that promise
|
|
337
|
+
// covered only the config identity. The enumerated passthrough above carries
|
|
338
|
+
// the credential variables — it must, because CI has no projection — so an
|
|
339
|
+
// ambient ANTHROPIC_API_KEY in the operator's shell survived the overlay
|
|
340
|
+
// untouched on exactly the path whose purpose is isolating the credential.
|
|
341
|
+
//
|
|
342
|
+
// The leak was inert while nothing read it back. #177 ended that: the Claude
|
|
343
|
+
// CLI transport now derives `billingMode`/`costSource` from this very env
|
|
344
|
+
// (`claudeCliCostBasis` → `envBillsMetered`), so a developer running on the
|
|
345
|
+
// projected subscription credential got the review filed as metered cash on
|
|
346
|
+
// the strength of a key the projection was supposed to have replaced. Both
|
|
347
|
+
// readings of the leak are defects — either the CLI honours the leaked key
|
|
348
|
+
// and the isolation promise is false, or it prefers the projected OAuth and
|
|
349
|
+
// the billing label is wrong — and one strip answers both.
|
|
350
|
+
//
|
|
351
|
+
// The rule the list encodes: a projection owns the CREDENTIAL IDENTITY, not
|
|
352
|
+
// just the home. ANTHROPIC_BASE_URL is deliberately NOT here — it is network
|
|
353
|
+
// routing, sitting in the passthrough beside HTTP(S)_PROXY and SSL_CERT_*,
|
|
354
|
+
// and the projection supplies no endpoint of its own, so stripping it would
|
|
355
|
+
// silently send the projection path somewhere the no-projection path does
|
|
356
|
+
// not go, with no defect behind the change.
|
|
357
|
+
//
|
|
358
|
+
// ONE list, not two: the three original `delete` statements and every key
|
|
359
|
+
// added since are the same array, so a future addition cannot land on one
|
|
360
|
+
// side of a drift. `satisfies` ties it to the passthrough — a key stripped
|
|
361
|
+
// here that is not projected there is dead code, and a typo fails tsc.
|
|
362
|
+
const PROJECTION_OWNED_KEYS = [
|
|
363
|
+
"HOME",
|
|
364
|
+
"TMPDIR",
|
|
365
|
+
"CLAUDE_CONFIG_DIR",
|
|
366
|
+
"ANTHROPIC_API_KEY",
|
|
367
|
+
"ANTHROPIC_AUTH_TOKEN",
|
|
368
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
369
|
+
] as const satisfies readonly (typeof ENV_PASSTHROUGH)[number][];
|
|
370
|
+
|
|
371
|
+
export function projectChildEnv(
|
|
372
|
+
source: Readonly<Record<string, string | undefined>>,
|
|
373
|
+
): Record<string, string> {
|
|
374
|
+
const projected: Record<string, string> = {};
|
|
375
|
+
for (const key of ENV_PASSTHROUGH) {
|
|
376
|
+
const value = source[key];
|
|
377
|
+
if (value !== undefined) projected[key] = value;
|
|
378
|
+
}
|
|
379
|
+
return projected;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// §6.3: user and system prompt payloads each carry a predeclared 2 MiB byte
|
|
383
|
+
// bound; larger steps fail before provider admission.
|
|
384
|
+
export const MAX_SYSTEM_PROMPT_BYTES = 2 * 1024 * 1024;
|
|
385
|
+
|
|
386
|
+
function notifyRetry(step: StepSpec, info: RetryInfo): void {
|
|
387
|
+
if (!step.onRetry) return;
|
|
388
|
+
try {
|
|
389
|
+
step.onRetry(info);
|
|
390
|
+
} catch {
|
|
391
|
+
// Swallowed
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// The harness's own bookkeeping lines, appended to a step's stderrTail AFTER
|
|
396
|
+
// the transport's tail (see run()). They report facts ABOUT the attempt —
|
|
397
|
+
// a degraded credential projection, metered accounting, a failed cleanup —
|
|
398
|
+
// and none of them is why a step failed. PR #315 review F001: the progress
|
|
399
|
+
// reason (pipeline.ts stepFailureReason) reads the LAST line of the tail, so
|
|
400
|
+
// without a way to recognize these it printed "failed — ambient credential
|
|
401
|
+
// bills metered ..." instead of the E2BIG above it. Spelled once here and
|
|
402
|
+
// used at every append site, so the producer and the predicate cannot drift.
|
|
403
|
+
// The transport's own diagnostics share the "[pr-hero] " tag ("spawn failed:
|
|
404
|
+
// ...; no child started"), which is why the predicate matches whole lines
|
|
405
|
+
// and never the tag alone.
|
|
406
|
+
const DESTROY_FAILED_ANNOTATION =
|
|
407
|
+
"[pr-hero] credential projection destroy failed";
|
|
408
|
+
const AMBIENT_METERED_ANNOTATION =
|
|
409
|
+
"[pr-hero] ambient credential bills metered — this attempt reserves against the spend ledger and fences its bucket if the cost cannot be confirmed";
|
|
410
|
+
// The class is variable in the shape even though only
|
|
411
|
+
// missing_subscription_record reaches the append today: the shape is what
|
|
412
|
+
// the harness writes, the class list is a policy that may grow.
|
|
413
|
+
const PROJECTION_WARNING_ANNOTATION =
|
|
414
|
+
/^\[pr-hero\] credential projection unavailable \([a-z_]+\); child runs with operator environment$/;
|
|
415
|
+
|
|
416
|
+
function projectionWarningAnnotation(failureClass: string): string {
|
|
417
|
+
return `[pr-hero] credential projection unavailable (${failureClass}); child runs with operator environment`;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
export function isHarnessStderrAnnotation(line: string): boolean {
|
|
421
|
+
const trimmed = line.trim();
|
|
422
|
+
return (
|
|
423
|
+
trimmed === DESTROY_FAILED_ANNOTATION ||
|
|
424
|
+
trimmed === AMBIENT_METERED_ANNOTATION ||
|
|
425
|
+
PROJECTION_WARNING_ANNOTATION.test(trimmed)
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// Settlement receipts are named on BOTH axes — step name and attempt — and the
|
|
430
|
+
// name is derived from the runner contract's single builder so the writer and
|
|
431
|
+
// every message that points a human at the file cannot drift apart (a hardcoded
|
|
432
|
+
// "settlement.json" in a cancellation message pointed at a file that never
|
|
433
|
+
// existed for as long as it shipped). D1-10c added a second reader of that
|
|
434
|
+
// shape — pipeline.json's per-step pointers — which is why the builder now
|
|
435
|
+
// lives in step-runner.ts rather than here.
|
|
436
|
+
function receiptPathFor(step: StepSpec, attempt: number): string {
|
|
437
|
+
return settlementReceiptPath(step.outPath, step.name, attempt);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// The generic reminder ("no prose, no fences") cannot help a draft whose JSON
|
|
441
|
+
// was fine and broke a validation rule instead: on Martian keycloak 36880 the
|
|
442
|
+
// model resent the same unresolved proof refs and lost the step twice. So the
|
|
443
|
+
// validator's own message rides along, capped so a long one cannot crowd out
|
|
444
|
+
// the prompt. Every retry is a fresh session that never saw the refused
|
|
445
|
+
// draft, so the wording names an earlier attempt, not "your" message.
|
|
446
|
+
const MAX_REJECTION_CHARS = 2000;
|
|
447
|
+
function formatRetryReminder(parseError: string | undefined): string {
|
|
448
|
+
if (parseError === undefined) return FORMAT_RETRY_REMINDER;
|
|
449
|
+
const reason =
|
|
450
|
+
parseError.length > MAX_REJECTION_CHARS
|
|
451
|
+
? `${parseError.slice(0, MAX_REJECTION_CHARS)}…`
|
|
452
|
+
: parseError;
|
|
453
|
+
return `${FORMAT_RETRY_REMINDER} An earlier attempt at this same task was rejected by the output validator, so avoid this: ${reason}`;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
async function writeAttemptLog(
|
|
457
|
+
step: StepSpec,
|
|
458
|
+
attempt: number,
|
|
459
|
+
kind: "attempt" | "format-retry",
|
|
460
|
+
outcome: TransportOutcome,
|
|
461
|
+
classification: "ok" | FailureClass,
|
|
462
|
+
// D1-08 PR0 (design row 13, additive): the coarse legacy `classification`
|
|
463
|
+
// above collapses watchdog_timeout into "transient", same as a plain
|
|
464
|
+
// network failure — this line is what lets incident triage tell them
|
|
465
|
+
// apart. Absent on a delivered ("ok") attempt.
|
|
466
|
+
cause?: FailureCause | "legacy_terminal",
|
|
467
|
+
// Why the attempt was refused. `parse_error`: `step.parse` rejected the
|
|
468
|
+
// final text. Without it a format_violation log says only THAT the draft
|
|
469
|
+
// was refused, and the result tail is capped, so the rule that fired (an
|
|
470
|
+
// unresolved proof ref, say) had to be rebuilt by hand from the validator
|
|
471
|
+
// source (Martian keycloak 36880, 2026-10-25). `persist_error`: the draft
|
|
472
|
+
// parsed and writing its artifacts failed.
|
|
473
|
+
rejection?: {
|
|
474
|
+
readonly label: "parse_error" | "persist_error";
|
|
475
|
+
readonly message: string;
|
|
476
|
+
},
|
|
477
|
+
): Promise<void> {
|
|
478
|
+
const logPath = attemptLogPath(step.outPath, step.name, attempt);
|
|
479
|
+
await mkdir(path.dirname(logPath), { recursive: true });
|
|
480
|
+
await Bun.write(
|
|
481
|
+
logPath,
|
|
482
|
+
[
|
|
483
|
+
`step: ${step.name}`,
|
|
484
|
+
`attempt: ${attempt}`,
|
|
485
|
+
`kind: ${kind}`,
|
|
486
|
+
`exit_code: ${outcome.exitCode ?? (outcome.completion === "success" ? 0 : 1)}`,
|
|
487
|
+
`timed_out: ${outcome.timedOut ?? false}`,
|
|
488
|
+
`classification: ${classification}`,
|
|
489
|
+
...(cause !== undefined ? [`cause: ${cause}`] : []),
|
|
490
|
+
...(rejection !== undefined
|
|
491
|
+
? [
|
|
492
|
+
`${rejection.label}: ${redactEvidenceText(redactDiagnostic(rejection.message)).replace(/\s+/g, " ")}`,
|
|
493
|
+
]
|
|
494
|
+
: []),
|
|
495
|
+
...(outcome.toolInvocations !== undefined
|
|
496
|
+
? [`tool_invocations: ${outcome.toolInvocations}`]
|
|
497
|
+
: []),
|
|
498
|
+
"--- stderr tail (4096) ---",
|
|
499
|
+
// §6.3: redaction before persistence — nothing unredacted hits disk.
|
|
500
|
+
redactEvidenceText(redactDiagnostic(outcome.stderrTail)),
|
|
501
|
+
// #126: the transport's own tallies, on their own line-item section so
|
|
502
|
+
// triage still reads them while no classifier can. The section is always
|
|
503
|
+
// written, empty included, so the log format stays fixed rather than
|
|
504
|
+
// varying with what a given attempt happened to observe.
|
|
505
|
+
"--- transport diagnostics ---",
|
|
506
|
+
redactEvidenceText(redactDiagnostic(outcome.diagnosticsTail ?? "")),
|
|
507
|
+
// #175 half 2: the models the PROVIDER says it ran, which is the only
|
|
508
|
+
// trustworthy answer to "what did this attempt actually use" — the
|
|
509
|
+
// route's own `modelSnapshot` records what we ASKED for. Always
|
|
510
|
+
// written, empty included, for the same reason as the section above:
|
|
511
|
+
// a log format that varies with what an attempt happened to observe is
|
|
512
|
+
// one a reader cannot tell apart from a log that lost a section.
|
|
513
|
+
"--- observed models ---",
|
|
514
|
+
...(outcome.observedModels ?? []).map((observed) =>
|
|
515
|
+
redactDiagnostic(
|
|
516
|
+
observed.canonicalModel === undefined
|
|
517
|
+
? observed.model
|
|
518
|
+
: `${observed.model} (canonical: ${observed.canonicalModel})`,
|
|
519
|
+
),
|
|
520
|
+
),
|
|
521
|
+
"--- result tail (8192) ---",
|
|
522
|
+
redactEvidenceText(redactDiagnostic(outcome.finalText.slice(-8192))),
|
|
523
|
+
"",
|
|
524
|
+
].join("\n"),
|
|
525
|
+
);
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
export class StepExecutionHarness implements StepRunner {
|
|
529
|
+
private readonly workspaceRoot?: string;
|
|
530
|
+
private readonly allowlist?: readonly ExecutableAllowlistEntry[];
|
|
531
|
+
private readonly binaryPath?: string;
|
|
532
|
+
private readonly executableSnapshotDir?: string;
|
|
533
|
+
private readonly admissionGate?: StepAdmissionGate;
|
|
534
|
+
private readonly registry?: TransportRegistry;
|
|
535
|
+
private readonly explicitTransport?: ProviderTransport;
|
|
536
|
+
private readonly transport: ProviderTransport;
|
|
537
|
+
private readonly onAuthEvent?: (event: AuthEvent) => void;
|
|
538
|
+
private readonly isTestFake: boolean;
|
|
539
|
+
private readonly projectedEnv: Record<string, string>;
|
|
540
|
+
private readonly credentialBroker?: CredentialBroker;
|
|
541
|
+
private readonly cancelSignal?: AbortSignal;
|
|
542
|
+
private readonly nowIso: () => string;
|
|
543
|
+
private readonly graceMarginMs: number;
|
|
544
|
+
private readonly onSessionSettled?: (info: {
|
|
545
|
+
readonly session: ActiveSession;
|
|
546
|
+
readonly settlement: SettlementSession;
|
|
547
|
+
readonly receipt: SettlementReceipt;
|
|
548
|
+
}) => void;
|
|
549
|
+
private readonly sleep: (ms: number) => Promise<void>;
|
|
550
|
+
private readonly attemptAdmissionGate?: AttemptAdmissionGate;
|
|
551
|
+
private readonly rateLimitBucketId: string;
|
|
552
|
+
private readonly spendLedger?: SpendLedger;
|
|
553
|
+
private readonly reservedUsdPerAttempt: number;
|
|
554
|
+
private readonly reservesSpend: boolean;
|
|
555
|
+
private readonly degradedProjectionBucketId?: string;
|
|
556
|
+
private readonly attemptAdmissionTimeoutMs: number;
|
|
557
|
+
private readonly credentialProjectionTimeoutMs: number;
|
|
558
|
+
private readonly onBeforeCredentialProjectionDestroy?: () => Promise<void>;
|
|
559
|
+
|
|
560
|
+
constructor(options: StepExecutionHarnessOptions = {}) {
|
|
561
|
+
this.workspaceRoot = options.workspaceRoot;
|
|
562
|
+
this.allowlist = options.executableAllowlist;
|
|
563
|
+
this.binaryPath = options.binaryPath;
|
|
564
|
+
this.executableSnapshotDir = options.executableSnapshotDir;
|
|
565
|
+
this.admissionGate = options.admissionGate;
|
|
566
|
+
this.registry = options.registry;
|
|
567
|
+
if (options.transport !== undefined) {
|
|
568
|
+
this.explicitTransport = options.transport;
|
|
569
|
+
this.transport = options.transport;
|
|
570
|
+
} else {
|
|
571
|
+
this.transport = new ClaudeCodeCliTransport({ spawnFn: options.spawnFn });
|
|
572
|
+
}
|
|
573
|
+
this.onAuthEvent = options.onAuthEvent;
|
|
574
|
+
this.isTestFake = Boolean(options.spawnFn);
|
|
575
|
+
this.projectedEnv = projectChildEnv(options.childEnv ?? process.env);
|
|
576
|
+
this.credentialBroker = options.credentialBroker;
|
|
577
|
+
this.cancelSignal = options.signal;
|
|
578
|
+
this.nowIso = options.nowIso ?? (() => new Date().toISOString());
|
|
579
|
+
this.graceMarginMs = options.graceMarginMs ?? HARNESS_GRACE_MARGIN_MS;
|
|
580
|
+
this.onSessionSettled = options.onSessionSettled;
|
|
581
|
+
this.sleep = options.sleep ?? DEFAULT_SLEEP;
|
|
582
|
+
this.attemptAdmissionGate = options.attemptAdmissionGate;
|
|
583
|
+
this.rateLimitBucketId =
|
|
584
|
+
options.rateLimitBucketId ?? DEFAULT_RATE_LIMIT_BUCKET_ID;
|
|
585
|
+
this.spendLedger = options.spendLedger;
|
|
586
|
+
this.reservedUsdPerAttempt = options.reservedUsdPerAttempt ?? 0;
|
|
587
|
+
// #279: default `true` — see the WHY on `StepExecutionHarnessOptions.
|
|
588
|
+
// reservesSpend` for why an unset flag must keep every pre-existing
|
|
589
|
+
// `spendLedger`-configuring caller reserving unconditionally.
|
|
590
|
+
this.reservesSpend = options.reservesSpend ?? true;
|
|
591
|
+
this.degradedProjectionBucketId = options.degradedProjectionBucketId;
|
|
592
|
+
// #279 (pr-hero review follow-up): a `reservesSpend: false` binding
|
|
593
|
+
// with a `spendLedger` but NO `degradedProjectionBucketId` used to fall
|
|
594
|
+
// back, SILENTLY, to `rateLimitBucketId` when a degraded reservation
|
|
595
|
+
// opened — exactly the binding's OWN (e.g. subscription) bucket the
|
|
596
|
+
// whole design exists to avoid fencing (see the WHY on the reserve call
|
|
597
|
+
// in `runAttempt`). Dead in production today (production-runtime.ts
|
|
598
|
+
// always computes and passes `degradedMeteredBucketId`), but a landmine
|
|
599
|
+
// for any other caller. Fail LOUD at construction instead of ever
|
|
600
|
+
// reaching that fallback at runtime: this shape cannot be built.
|
|
601
|
+
if (
|
|
602
|
+
this.spendLedger !== undefined &&
|
|
603
|
+
this.reservesSpend === false &&
|
|
604
|
+
this.degradedProjectionBucketId === undefined
|
|
605
|
+
) {
|
|
606
|
+
throw new Error(
|
|
607
|
+
"StepExecutionHarness: reservesSpend:false with a spendLedger configured requires degradedProjectionBucketId — without it, a degraded-projection reservation has no safe bucket to fence (#279)",
|
|
608
|
+
);
|
|
609
|
+
}
|
|
610
|
+
this.attemptAdmissionTimeoutMs =
|
|
611
|
+
options.attemptAdmissionTimeoutMs ?? DEFAULT_ATTEMPT_ADMISSION_TIMEOUT_MS;
|
|
612
|
+
this.credentialProjectionTimeoutMs =
|
|
613
|
+
options.credentialProjectionTimeoutMs ??
|
|
614
|
+
DEFAULT_CREDENTIAL_PROJECTION_TIMEOUT_MS;
|
|
615
|
+
this.onBeforeCredentialProjectionDestroy =
|
|
616
|
+
options.onBeforeCredentialProjectionDestroy;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
async run(step: StepSpec): Promise<StepResult> {
|
|
620
|
+
let canonicalCwd = step.cwd;
|
|
621
|
+
|
|
622
|
+
// Resolve transport for this step
|
|
623
|
+
const effectiveBackend =
|
|
624
|
+
step.route?.backend ?? step.backend ?? "claude-code";
|
|
625
|
+
let transport: ProviderTransport;
|
|
626
|
+
if (this.explicitTransport !== undefined) {
|
|
627
|
+
const declaredBackend = step.route?.backend ?? step.backend;
|
|
628
|
+
if (
|
|
629
|
+
declaredBackend !== undefined &&
|
|
630
|
+
this.explicitTransport.backend !== declaredBackend
|
|
631
|
+
) {
|
|
632
|
+
return {
|
|
633
|
+
name: step.name,
|
|
634
|
+
status: "failed",
|
|
635
|
+
usage: zeroUsage(),
|
|
636
|
+
attempts: 0,
|
|
637
|
+
stderrTail: `No transport registry configured to handle backend "${declaredBackend}" for step "${step.name}"`,
|
|
638
|
+
resultText: "",
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
transport = this.explicitTransport;
|
|
642
|
+
} else if (this.registry) {
|
|
643
|
+
try {
|
|
644
|
+
transport = this.registry.get(effectiveBackend, {
|
|
645
|
+
routeFingerprint: step.routeKey,
|
|
646
|
+
route: step.route,
|
|
647
|
+
});
|
|
648
|
+
} catch (err) {
|
|
649
|
+
return {
|
|
650
|
+
name: step.name,
|
|
651
|
+
status: "failed",
|
|
652
|
+
usage: zeroUsage(),
|
|
653
|
+
attempts: 0,
|
|
654
|
+
stderrTail: `Transport resolution failed: ${(err as Error).message}`,
|
|
655
|
+
resultText: "",
|
|
656
|
+
};
|
|
657
|
+
}
|
|
658
|
+
} else {
|
|
659
|
+
if (effectiveBackend !== "claude-code") {
|
|
660
|
+
return {
|
|
661
|
+
name: step.name,
|
|
662
|
+
status: "failed",
|
|
663
|
+
usage: zeroUsage(),
|
|
664
|
+
attempts: 0,
|
|
665
|
+
stderrTail: `No transport registry configured to handle backend "${effectiveBackend}" for step "${step.name}"`,
|
|
666
|
+
resultText: "",
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
transport = this.transport;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
if (
|
|
673
|
+
effectiveBackend === "opencode" &&
|
|
674
|
+
this.registry instanceof DefaultTransportRegistry &&
|
|
675
|
+
this.registry.needsOpenCodeSdkProbe()
|
|
676
|
+
) {
|
|
677
|
+
try {
|
|
678
|
+
await this.registry.probeOpenCodeSdk();
|
|
679
|
+
} catch (error) {
|
|
680
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
681
|
+
return {
|
|
682
|
+
name: step.name,
|
|
683
|
+
status: "failed",
|
|
684
|
+
usage: zeroUsage(),
|
|
685
|
+
attempts: 0,
|
|
686
|
+
stderrTail: `OpenCode SDK pre-confirm failed: ${message}`,
|
|
687
|
+
resultText: "",
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
// Generic execution facts resolution & validation
|
|
693
|
+
const admissionIdentity =
|
|
694
|
+
transport.admissionIdentity ??
|
|
695
|
+
(transport.backend === "claude-code"
|
|
696
|
+
? { executable: "claude", provider: "anthropic" }
|
|
697
|
+
: this.isTestFake
|
|
698
|
+
? {
|
|
699
|
+
executable:
|
|
700
|
+
transport.backend === "opencode" ? "opencode" : "claude",
|
|
701
|
+
provider:
|
|
702
|
+
transport.backend === "opencode" ? "opencode" : "anthropic",
|
|
703
|
+
}
|
|
704
|
+
: undefined);
|
|
705
|
+
|
|
706
|
+
const cancellationSemantics =
|
|
707
|
+
transport.cancellationSemantics ??
|
|
708
|
+
(transport.backend === "claude-code"
|
|
709
|
+
? "process-exit"
|
|
710
|
+
: this.isTestFake
|
|
711
|
+
? transport.backend === "opencode"
|
|
712
|
+
? "provider-proof"
|
|
713
|
+
: "process-exit"
|
|
714
|
+
: undefined);
|
|
715
|
+
|
|
716
|
+
if (
|
|
717
|
+
admissionIdentity === undefined ||
|
|
718
|
+
cancellationSemantics === undefined
|
|
719
|
+
) {
|
|
720
|
+
const missing = [
|
|
721
|
+
...(admissionIdentity === undefined ? ["admissionIdentity"] : []),
|
|
722
|
+
...(cancellationSemantics === undefined
|
|
723
|
+
? ["cancellationSemantics"]
|
|
724
|
+
: []),
|
|
725
|
+
].join(", ");
|
|
726
|
+
this.onAuthEvent?.({
|
|
727
|
+
kind: "executable",
|
|
728
|
+
status: "denied",
|
|
729
|
+
reason: `Missing required generic execution facts for backend "${transport.backend}" (${missing})`,
|
|
730
|
+
});
|
|
731
|
+
return {
|
|
732
|
+
name: step.name,
|
|
733
|
+
status: "failed",
|
|
734
|
+
denialCode: "executable_not_approved",
|
|
735
|
+
usage: zeroUsage(),
|
|
736
|
+
attempts: 0,
|
|
737
|
+
stderrTail: `Missing required generic execution facts for backend "${transport.backend}" (${missing})`,
|
|
738
|
+
resultText: "",
|
|
739
|
+
};
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// Provider mismatch / credential isolation check:
|
|
743
|
+
// When step.route specifies a provider, it must match the transport's configured provider,
|
|
744
|
+
// unless transport has a generic opencode provider that matches opencode routes.
|
|
745
|
+
if (
|
|
746
|
+
step.route?.provider !== undefined &&
|
|
747
|
+
admissionIdentity.provider !== undefined &&
|
|
748
|
+
step.route.provider !== admissionIdentity.provider &&
|
|
749
|
+
!(
|
|
750
|
+
admissionIdentity.provider === "opencode" &&
|
|
751
|
+
step.route.backend === "opencode"
|
|
752
|
+
)
|
|
753
|
+
) {
|
|
754
|
+
this.onAuthEvent?.({
|
|
755
|
+
kind: "executable",
|
|
756
|
+
status: "denied",
|
|
757
|
+
reason: `Provider mismatch: step specifies provider "${step.route.provider}" but transport is configured for "${admissionIdentity.provider}"`,
|
|
758
|
+
});
|
|
759
|
+
return {
|
|
760
|
+
name: step.name,
|
|
761
|
+
status: "failed",
|
|
762
|
+
denialCode: "executable_not_approved",
|
|
763
|
+
usage: zeroUsage(),
|
|
764
|
+
attempts: 0,
|
|
765
|
+
stderrTail: `Provider mismatch: step specifies provider "${step.route.provider}" but transport is configured for "${admissionIdentity.provider}"`,
|
|
766
|
+
resultText: "",
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// 1. Workspace authorization
|
|
771
|
+
if (this.workspaceRoot !== undefined) {
|
|
772
|
+
const workspaceBroker = new WorkspaceReadBroker({
|
|
773
|
+
workspaceRoot: this.workspaceRoot,
|
|
774
|
+
});
|
|
775
|
+
const wsResult = workspaceBroker.authorizePath(step.cwd);
|
|
776
|
+
|
|
777
|
+
if (!wsResult.approved) {
|
|
778
|
+
this.onAuthEvent?.({
|
|
779
|
+
kind: "workspace",
|
|
780
|
+
status: "denied",
|
|
781
|
+
reason: wsResult.reason,
|
|
782
|
+
});
|
|
783
|
+
return {
|
|
784
|
+
name: step.name,
|
|
785
|
+
status: "failed",
|
|
786
|
+
denialCode: "path_not_approved",
|
|
787
|
+
usage: zeroUsage(),
|
|
788
|
+
attempts: 0,
|
|
789
|
+
stderrTail: `Workspace denied: ${wsResult.reason}`,
|
|
790
|
+
resultText: "",
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
// The step must execute against the broker-approved canonical path, not
|
|
794
|
+
// the caller-supplied alias: a symlinked cwd would silently re-widen the
|
|
795
|
+
// sandbox boundary the broker just verified.
|
|
796
|
+
canonicalCwd = wsResult.canonicalPath;
|
|
797
|
+
this.onAuthEvent?.({ kind: "workspace", status: "approved" });
|
|
798
|
+
} else {
|
|
799
|
+
this.onAuthEvent?.({ kind: "workspace", status: "approved" });
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
// 2. Executable authorization
|
|
803
|
+
let verifiedBinaryPath: string;
|
|
804
|
+
|
|
805
|
+
if (this.allowlist !== undefined) {
|
|
806
|
+
const candidate = this.binaryPath ?? admissionIdentity.executable;
|
|
807
|
+
const execResult = await verifyExecutableAuthority({
|
|
808
|
+
candidatePath: candidate,
|
|
809
|
+
allowlist: this.allowlist,
|
|
810
|
+
snapshotDir: this.executableSnapshotDir,
|
|
811
|
+
});
|
|
812
|
+
|
|
813
|
+
if (!execResult.approved) {
|
|
814
|
+
this.onAuthEvent?.({
|
|
815
|
+
kind: "executable",
|
|
816
|
+
status: "denied",
|
|
817
|
+
reason: execResult.reason,
|
|
818
|
+
});
|
|
819
|
+
return {
|
|
820
|
+
name: step.name,
|
|
821
|
+
status: "failed",
|
|
822
|
+
denialCode: "executable_not_approved",
|
|
823
|
+
usage: zeroUsage(),
|
|
824
|
+
attempts: 0,
|
|
825
|
+
stderrTail: `Executable denied: ${execResult.reason}`,
|
|
826
|
+
resultText: "",
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
verifiedBinaryPath = execResult.executable.verifiedExecutionPath;
|
|
830
|
+
} else if (this.isTestFake) {
|
|
831
|
+
// Offline unit test runner with fake spawn
|
|
832
|
+
verifiedBinaryPath = this.binaryPath ?? admissionIdentity.executable;
|
|
833
|
+
} else {
|
|
834
|
+
// Production without configured allowlist -> fail closed
|
|
835
|
+
this.onAuthEvent?.({
|
|
836
|
+
kind: "executable",
|
|
837
|
+
status: "denied",
|
|
838
|
+
reason: "Missing execution authority: no allowlist configured",
|
|
839
|
+
});
|
|
840
|
+
return {
|
|
841
|
+
name: step.name,
|
|
842
|
+
status: "failed",
|
|
843
|
+
denialCode: "executable_not_approved",
|
|
844
|
+
usage: zeroUsage(),
|
|
845
|
+
attempts: 0,
|
|
846
|
+
stderrTail: "Missing execution authority: no allowlist configured",
|
|
847
|
+
resultText: "",
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
// #303: the verified execution snapshot lives exactly as long as the
|
|
852
|
+
// attempts that spawn from it. The try opens on the line after the path
|
|
853
|
+
// is assigned — even the approval event sits inside it — so every return
|
|
854
|
+
// and throw below, including the credential-projection early return,
|
|
855
|
+
// releases it exactly once, after the LAST attempt (all attempts reuse
|
|
856
|
+
// one snapshot). After a watchdog or cancel the child may still be alive;
|
|
857
|
+
// unlinking a running executable is fine on POSIX, and on Windows the
|
|
858
|
+
// failed removal stays registered for the process-exit release. The
|
|
859
|
+
// release is gated on the snapshot registry, so the test-fake and `#!`
|
|
860
|
+
// launcher paths (never snapshotted) are untouched.
|
|
861
|
+
try {
|
|
862
|
+
this.onAuthEvent?.({ kind: "executable", status: "approved" });
|
|
863
|
+
return await this.runAuthorizedStep({
|
|
864
|
+
step,
|
|
865
|
+
transport,
|
|
866
|
+
admissionIdentity,
|
|
867
|
+
canonicalCwd,
|
|
868
|
+
verifiedBinaryPath,
|
|
869
|
+
});
|
|
870
|
+
} finally {
|
|
871
|
+
releaseExecutionSnapshot(verifiedBinaryPath);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
private async runAuthorizedStep(args: {
|
|
876
|
+
readonly step: StepSpec;
|
|
877
|
+
readonly transport: ProviderTransport;
|
|
878
|
+
readonly admissionIdentity: {
|
|
879
|
+
readonly executable: string;
|
|
880
|
+
readonly provider: string;
|
|
881
|
+
};
|
|
882
|
+
readonly canonicalCwd: string;
|
|
883
|
+
readonly verifiedBinaryPath: string;
|
|
884
|
+
}): Promise<StepResult> {
|
|
885
|
+
const {
|
|
886
|
+
step,
|
|
887
|
+
transport,
|
|
888
|
+
admissionIdentity,
|
|
889
|
+
canonicalCwd,
|
|
890
|
+
verifiedBinaryPath,
|
|
891
|
+
} = args;
|
|
892
|
+
|
|
893
|
+
// #150: a transport that declares "server-lifetime" owns its OWN
|
|
894
|
+
// credential protection outside this per-step loop (the OpenCode SDK
|
|
895
|
+
// transport's server, brokered once for the server's whole run — #149)
|
|
896
|
+
// and never reads `isolation` at all. Projecting, writing an auth file
|
|
897
|
+
// to disk, and destroying it for such a transport protects nothing —
|
|
898
|
+
// it is pure disk churn plus a misleading artifact. Reading
|
|
899
|
+
// `transport.credentialProjection` HERE (not `this.transport`) matters:
|
|
900
|
+
// this is the same transport variable `admitAndExecute` below is about
|
|
901
|
+
// to run the attempt against, resolved per-step just above (registry
|
|
902
|
+
// lookup / explicit override) — gating on any other reference could
|
|
903
|
+
// diverge from what actually executes.
|
|
904
|
+
//
|
|
905
|
+
// Undeclared/"per-step" is the fail-safe default (see
|
|
906
|
+
// `ProviderTransport.credentialProjection`, contracts.ts): every
|
|
907
|
+
// existing transport, and any new one nobody has updated yet, keeps
|
|
908
|
+
// projecting exactly as before.
|
|
909
|
+
const consumesPerStepProjection =
|
|
910
|
+
transport.credentialProjection !== "server-lifetime";
|
|
911
|
+
|
|
912
|
+
// §6.1 D1-05: project credentials ONCE per run, BEFORE admission or
|
|
913
|
+
// spawn — a projection failure must never reach a provider or leak into
|
|
914
|
+
// an attempt count.
|
|
915
|
+
let projection: CredentialProjection | undefined;
|
|
916
|
+
let projectionWarning: string | undefined;
|
|
917
|
+
if (this.credentialBroker && consumesPerStepProjection) {
|
|
918
|
+
try {
|
|
919
|
+
projection = await projectCredentialWithBudget(
|
|
920
|
+
this.credentialBroker,
|
|
921
|
+
{
|
|
922
|
+
sessionId: step.name,
|
|
923
|
+
credentialRef: step.credentialRef ?? "claude-code-credentials",
|
|
924
|
+
kind: step.credentialKind ?? "claude_subscription_oauth",
|
|
925
|
+
verifiedBinaryPath,
|
|
926
|
+
},
|
|
927
|
+
{
|
|
928
|
+
timeoutMs: this.credentialProjectionTimeoutMs,
|
|
929
|
+
signal: this.cancelSignal,
|
|
930
|
+
},
|
|
931
|
+
);
|
|
932
|
+
} catch (error) {
|
|
933
|
+
// Class names only the failure class; broker error text is never
|
|
934
|
+
// echoed because third-party brokers may embed arbitrary content.
|
|
935
|
+
const failureClass =
|
|
936
|
+
error instanceof CredentialProjectionError
|
|
937
|
+
? error.failureClass
|
|
938
|
+
: "broker_error";
|
|
939
|
+
// Deliberate degradation, scoped to exactly ONE failure class
|
|
940
|
+
// (operator decision 2026-08-26): Claude desktop 2.1.246 moved the
|
|
941
|
+
// subscription OAuth record out of the keychain item into its own
|
|
942
|
+
// encrypted store, so missing_subscription_record now means "the CLI
|
|
943
|
+
// keeps its credential somewhere we cannot read" — killing every
|
|
944
|
+
// step made reviews impossible. Every OTHER class is an attack or
|
|
945
|
+
// integrity signal (symlinked layout, tampered payload, unreadable
|
|
946
|
+
// source) and still fails closed: degrading on those would run an
|
|
947
|
+
// adversarial-diff agent with operator credentials precisely when
|
|
948
|
+
// something suspicious happened. The degradation is stated on the
|
|
949
|
+
// result, never silent.
|
|
950
|
+
if (failureClass !== "missing_subscription_record") {
|
|
951
|
+
return {
|
|
952
|
+
name: step.name,
|
|
953
|
+
status: "failed",
|
|
954
|
+
usage: zeroUsage(),
|
|
955
|
+
attempts: 0,
|
|
956
|
+
stderrTail: `Credential projection failed (${failureClass}); step failed before any spawn`,
|
|
957
|
+
resultText: "",
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
projection = undefined;
|
|
961
|
+
projectionWarning = projectionWarningAnnotation(failureClass);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
const childEnv = this.buildChildEnv(projection);
|
|
966
|
+
// #279: the one attempt-shape fact admission cannot see. Admission
|
|
967
|
+
// (`resolveBindingAuthority`, runner-authority.ts) decides a claude-code
|
|
968
|
+
// route's credential KIND once, before any projection ever runs, and
|
|
969
|
+
// cannot know THIS attempt's projection is about to degrade.
|
|
970
|
+
// `projectionWarning` is set on exactly one failure class
|
|
971
|
+
// (`missing_subscription_record`, above) and nothing else, so it
|
|
972
|
+
// doubles as that signal here — every OTHER projection failure returns
|
|
973
|
+
// early above and never reaches this line.
|
|
974
|
+
const projectionDegraded = projectionWarning !== undefined;
|
|
975
|
+
|
|
976
|
+
try {
|
|
977
|
+
const result = await this.admitAndExecute({
|
|
978
|
+
step,
|
|
979
|
+
canonicalCwd,
|
|
980
|
+
verifiedBinaryPath,
|
|
981
|
+
childEnv,
|
|
982
|
+
projection,
|
|
983
|
+
projectionDegraded,
|
|
984
|
+
transport,
|
|
985
|
+
admissionIdentity,
|
|
986
|
+
// #150: what `isolation.credentialProjectionId` says when `projection`
|
|
987
|
+
// stays undefined. "operator-env-fallback" means what it always did —
|
|
988
|
+
// no broker configured, or the missing_subscription_record degrade
|
|
989
|
+
// just above — a synthetic identity was expected and did not happen.
|
|
990
|
+
// A declared server-lifetime skip is neither: no projection was ever
|
|
991
|
+
// ATTEMPTED, so labelling it a "fallback" would claim a degrade that
|
|
992
|
+
// never occurred. "server-lifetime" says the honest thing instead —
|
|
993
|
+
// this step's credentials are protected elsewhere, by the transport's
|
|
994
|
+
// own server-scoped projection.
|
|
995
|
+
fallbackCredentialProjectionId: consumesPerStepProjection
|
|
996
|
+
? "operator-env-fallback"
|
|
997
|
+
: "server-lifetime",
|
|
998
|
+
});
|
|
999
|
+
// §6.1: destroy() runs after settlement on EVERY return path; its
|
|
1000
|
+
// failure is a warning appended to stderrTail, never a thrown error
|
|
1001
|
+
// and never a replacement for the step's own outcome.
|
|
1002
|
+
await this.onBeforeCredentialProjectionDestroy?.().catch(() => {});
|
|
1003
|
+
this.appendDestroyFailure(
|
|
1004
|
+
await this.destroyProjection(projection),
|
|
1005
|
+
result,
|
|
1006
|
+
);
|
|
1007
|
+
if (projectionWarning !== undefined) {
|
|
1008
|
+
result.stderrTail = `${result.stderrTail}\n${projectionWarning}`;
|
|
1009
|
+
}
|
|
1010
|
+
// #279 (pr-hero review follow-up): a metered-accounting CLAIM, so it
|
|
1011
|
+
// must only be made once a reservation ACTUALLY opened — never from
|
|
1012
|
+
// `projectionDegraded` alone, which is true before `admitAndExecute`
|
|
1013
|
+
// ever runs an attempt. An early return above it (system-prompt-too-
|
|
1014
|
+
// large, unreadable system prompt — `admitAndExecute`'s own guards)
|
|
1015
|
+
// carries `attempts: 0` and no `reservations` at all; asserting "this
|
|
1016
|
+
// attempt reserves..." there would describe a reservation that never
|
|
1017
|
+
// happened. `result.reservations` is the exact signal: `admitAndExecute`
|
|
1018
|
+
// populates it ONLY by pushing `attemptResult.reservation`, i.e. only
|
|
1019
|
+
// once `spendLedger.reserve()` actually returned a reservation object
|
|
1020
|
+
// — the fenced-on-arrival case (`SpendReservationFencedError`, thrown
|
|
1021
|
+
// INSIDE `reserve()` before it returns one) leaves this empty too,
|
|
1022
|
+
// which is correct: nothing NEW was reserved there either.
|
|
1023
|
+
// `projectionDegraded` is kept as an explicit guard even though a
|
|
1024
|
+
// non-empty `reservations` on a `reservesSpend: false` binding already
|
|
1025
|
+
// implies it (`runAttempt`'s reserve gate) — stated here rather than
|
|
1026
|
+
// relying on that invariant silently holding.
|
|
1027
|
+
if (
|
|
1028
|
+
projectionDegraded &&
|
|
1029
|
+
result.reservations !== undefined &&
|
|
1030
|
+
result.reservations.length > 0
|
|
1031
|
+
) {
|
|
1032
|
+
result.stderrTail = `${result.stderrTail}\n${AMBIENT_METERED_ANNOTATION}`;
|
|
1033
|
+
}
|
|
1034
|
+
return result;
|
|
1035
|
+
} catch (error) {
|
|
1036
|
+
await this.onBeforeCredentialProjectionDestroy?.().catch(() => {});
|
|
1037
|
+
this.appendDestroyFailure(await this.destroyProjection(projection));
|
|
1038
|
+
throw error;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
private buildChildEnv(
|
|
1043
|
+
projection: CredentialProjection | undefined,
|
|
1044
|
+
): Record<string, string> {
|
|
1045
|
+
// No projection means CI (credential projection needs darwin plus
|
|
1046
|
+
// /usr/bin/security — `claudeCredentialBroker`, runner-authority.ts), and
|
|
1047
|
+
// that path is byte-identical to what it always was: the enumerated
|
|
1048
|
+
// passthrough, credentials included. Stripping there is what cost
|
|
1049
|
+
// pr-hero's first CI self-review.
|
|
1050
|
+
if (projection === undefined) return this.projectedEnv;
|
|
1051
|
+
// The projection owns HOME/TMPDIR/config identity AND the credential
|
|
1052
|
+
// itself (§6.1, and PROJECTION_OWNED_KEYS above for why the credential
|
|
1053
|
+
// joined the list); strip the enumerated-passthrough copies first so real
|
|
1054
|
+
// values cannot survive.
|
|
1055
|
+
const stripped = { ...this.projectedEnv };
|
|
1056
|
+
for (const key of PROJECTION_OWNED_KEYS) delete stripped[key];
|
|
1057
|
+
return { ...stripped, ...projection.env };
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
// Returns true when destruction failed (caller annotates the result).
|
|
1061
|
+
private async destroyProjection(
|
|
1062
|
+
projection: CredentialProjection | undefined,
|
|
1063
|
+
): Promise<boolean> {
|
|
1064
|
+
if (projection === undefined) return false;
|
|
1065
|
+
try {
|
|
1066
|
+
await projection.destroy();
|
|
1067
|
+
return false;
|
|
1068
|
+
} catch {
|
|
1069
|
+
return true;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
private appendDestroyFailure(
|
|
1074
|
+
destroyFailed: boolean,
|
|
1075
|
+
result?: StepResult,
|
|
1076
|
+
): void {
|
|
1077
|
+
if (!destroyFailed) return;
|
|
1078
|
+
const line = DESTROY_FAILED_ANNOTATION;
|
|
1079
|
+
if (result !== undefined) {
|
|
1080
|
+
result.stderrTail += `${result.stderrTail}\n${line}`;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// §5.3 step 2: once the lease is invalidated every data-plane write is
|
|
1085
|
+
// refused AND counted on the fence's rejectedEvents.
|
|
1086
|
+
private async guardedDataPlaneWrite(
|
|
1087
|
+
settlement: SettlementSession,
|
|
1088
|
+
write: () => Promise<void>,
|
|
1089
|
+
): Promise<void> {
|
|
1090
|
+
if (!settlement.writeLease.valid) {
|
|
1091
|
+
settlement.rejectDataPlaneEvents(1);
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
await write();
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
// Control-plane persistence (§5.3 step 7): the receipt is written through
|
|
1098
|
+
// its own always-valid atomic path, deliberately NOT lease-guarded — the
|
|
1099
|
+
// receipt itself records the closed fence, so refusing this write would
|
|
1100
|
+
// erase the evidence that late data-plane events were refused.
|
|
1101
|
+
private async persistSettlement(
|
|
1102
|
+
step: StepSpec,
|
|
1103
|
+
session: ActiveSession,
|
|
1104
|
+
receipt: SettlementReceipt,
|
|
1105
|
+
): Promise<void> {
|
|
1106
|
+
// Per-STEP *and* per-attempt filename (§422). Two independent collisions
|
|
1107
|
+
// are being prevented here and an audit record needs both:
|
|
1108
|
+
// - attempt vs attempt: every retry receives its own receipt, so a later
|
|
1109
|
+
// attempt never clobbers an earlier one's rejected-event / fence record.
|
|
1110
|
+
// - step vs step: every step of a run shares ONE steps/ directory
|
|
1111
|
+
// (hunters, summarizer, refuters, scout all write beside each other),
|
|
1112
|
+
// so a name keyed only on the attempt made each step overwrite the
|
|
1113
|
+
// previous step's receipt. That is not hypothetical — a four-step run
|
|
1114
|
+
// left four attempt logs and exactly one receipt on disk, and three
|
|
1115
|
+
// audit records were destroyed silently.
|
|
1116
|
+
// Keying on the step name is what makes the write target unique, which in
|
|
1117
|
+
// turn gives writeJsonAtomically's `${outPath}.tmp` staging file a unique
|
|
1118
|
+
// name — parallel steps otherwise raced on one tmp path and could corrupt
|
|
1119
|
+
// each other mid-rename rather than merely losing the loser.
|
|
1120
|
+
await writeJsonAtomically(receiptPathFor(step, session.attempt), receipt);
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
// §5.2: deadlines are declared transport capabilities, not request fields;
|
|
1124
|
+
// an unusable capability report falls back to the CLI/POSIX row default.
|
|
1125
|
+
private async resolveCancellationDeadlineMs(
|
|
1126
|
+
transport: ProviderTransport = this.transport,
|
|
1127
|
+
): Promise<number> {
|
|
1128
|
+
try {
|
|
1129
|
+
const report = await transport.capabilities();
|
|
1130
|
+
const declared = report.cancellation?.deadlineMs;
|
|
1131
|
+
if (
|
|
1132
|
+
typeof declared === "number" &&
|
|
1133
|
+
Number.isFinite(declared) &&
|
|
1134
|
+
declared > 0
|
|
1135
|
+
) {
|
|
1136
|
+
return declared;
|
|
1137
|
+
}
|
|
1138
|
+
} catch {
|
|
1139
|
+
// Fall through to the §5.2 default.
|
|
1140
|
+
}
|
|
1141
|
+
return DEFAULT_CANCELLATION_DEADLINE_MS;
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
private async executeSession(args: {
|
|
1145
|
+
readonly step: StepSpec;
|
|
1146
|
+
readonly request: TransportRequest;
|
|
1147
|
+
readonly transport: ProviderTransport;
|
|
1148
|
+
readonly deadlineMs: number;
|
|
1149
|
+
readonly harnessWatchdogMs?: number;
|
|
1150
|
+
readonly onData: (
|
|
1151
|
+
outcome: TransportOutcome,
|
|
1152
|
+
settlement: SettlementSession,
|
|
1153
|
+
) => Promise<AttemptDelivery>;
|
|
1154
|
+
}): Promise<{
|
|
1155
|
+
readonly session: ActiveSession;
|
|
1156
|
+
readonly settlement: SettlementSession;
|
|
1157
|
+
readonly outcome?: TransportOutcome;
|
|
1158
|
+
readonly delivery?: AttemptDelivery;
|
|
1159
|
+
readonly cancelled: boolean;
|
|
1160
|
+
// D1-08 PR5a (§9.2): every return path below builds one via `finalize`
|
|
1161
|
+
// before returning — exposed so `runAttempt` can react to
|
|
1162
|
+
// `local_fenced_remote_unconfirmed` without re-deriving it.
|
|
1163
|
+
readonly receipt: SettlementReceipt;
|
|
1164
|
+
}> {
|
|
1165
|
+
const { step, request, transport, deadlineMs, harnessWatchdogMs, onData } =
|
|
1166
|
+
args;
|
|
1167
|
+
|
|
1168
|
+
const settlement = createSettlement(request.sessionId, request.attempt, {
|
|
1169
|
+
now: this.nowIso,
|
|
1170
|
+
});
|
|
1171
|
+
const lease = settlement.writeLease;
|
|
1172
|
+
const controller = new AbortController();
|
|
1173
|
+
|
|
1174
|
+
let resolveSettled!: (receipt: SettlementReceipt) => void;
|
|
1175
|
+
const settled = new Promise<SettlementReceipt>((resolve) => {
|
|
1176
|
+
resolveSettled = resolve;
|
|
1177
|
+
});
|
|
1178
|
+
|
|
1179
|
+
let fenceClosed = false;
|
|
1180
|
+
let outcome: TransportOutcome | undefined;
|
|
1181
|
+
let finalized = false;
|
|
1182
|
+
|
|
1183
|
+
// §5.1: settled always resolves — invariant failures convert below, and a
|
|
1184
|
+
// totally broken settlement context still yields a hand-built unconfirmed
|
|
1185
|
+
// receipt rather than a rejection.
|
|
1186
|
+
const finalize = (build: () => SettlementReceipt): SettlementReceipt => {
|
|
1187
|
+
let receipt: SettlementReceipt;
|
|
1188
|
+
try {
|
|
1189
|
+
receipt = build();
|
|
1190
|
+
} catch (error) {
|
|
1191
|
+
try {
|
|
1192
|
+
receipt = synthesizeInternalFailure(settlement, error);
|
|
1193
|
+
} catch {
|
|
1194
|
+
const fallbackNow = new Date().toISOString();
|
|
1195
|
+
receipt = {
|
|
1196
|
+
sessionId: request.sessionId,
|
|
1197
|
+
attempt: request.attempt,
|
|
1198
|
+
outcome: "local_termination_unconfirmed",
|
|
1199
|
+
termination: { requested: true, confirmation: "unconfirmed" },
|
|
1200
|
+
resources: {
|
|
1201
|
+
localReleased: true,
|
|
1202
|
+
processGroupAlive: "unknown",
|
|
1203
|
+
remoteStatus: "unknown_may_continue",
|
|
1204
|
+
},
|
|
1205
|
+
timestamps: { startedAt: fallbackNow, settledAt: fallbackNow },
|
|
1206
|
+
lateWriteFence: {
|
|
1207
|
+
leaseId: lease.id,
|
|
1208
|
+
closed: true,
|
|
1209
|
+
rejectedEvents: 0,
|
|
1210
|
+
},
|
|
1211
|
+
warnings: [
|
|
1212
|
+
"internal settlement invariant failure; receipt hand-built",
|
|
1213
|
+
],
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
if (!finalized) {
|
|
1218
|
+
finalized = true;
|
|
1219
|
+
resolveSettled(receipt);
|
|
1220
|
+
}
|
|
1221
|
+
return receipt;
|
|
1222
|
+
};
|
|
1223
|
+
|
|
1224
|
+
// §13 line 747: after sink closure a late event cannot reach the data
|
|
1225
|
+
// plane — it is refused and counted on the fence.
|
|
1226
|
+
const sink: AsyncEventSink = {
|
|
1227
|
+
push: async (event) => {
|
|
1228
|
+
if (!lease.valid || fenceClosed) {
|
|
1229
|
+
settlement.rejectDataPlaneEvents(1);
|
|
1230
|
+
return "closed";
|
|
1231
|
+
}
|
|
1232
|
+
if (event.type === "terminal") {
|
|
1233
|
+
settlement.acceptTerminal(event.origin, event.status, event.proof);
|
|
1234
|
+
}
|
|
1235
|
+
return "accepted";
|
|
1236
|
+
},
|
|
1237
|
+
close: () => {
|
|
1238
|
+
fenceClosed = true;
|
|
1239
|
+
if (lease.valid) {
|
|
1240
|
+
lease.invalidate("event sink closed");
|
|
1241
|
+
settlement.markLeaseInvalidated();
|
|
1242
|
+
}
|
|
1243
|
+
},
|
|
1244
|
+
};
|
|
1245
|
+
|
|
1246
|
+
const execPromise: Promise<void> = transport
|
|
1247
|
+
.execute(request, { signal: controller.signal, events: sink })
|
|
1248
|
+
.then((resolved) => {
|
|
1249
|
+
if (fenceClosed || !lease.valid) {
|
|
1250
|
+
// §5.3 step 5: a terminal arriving after grace expiry is counted by
|
|
1251
|
+
// the compare-and-set slot but never applied; its payload can no
|
|
1252
|
+
// longer reach parse or artifacts either.
|
|
1253
|
+
settlement.acceptTerminal(
|
|
1254
|
+
"transport",
|
|
1255
|
+
completionToStatus(resolved),
|
|
1256
|
+
resolved.terminalProof,
|
|
1257
|
+
);
|
|
1258
|
+
settlement.rejectDataPlaneEvents(1);
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
outcome = resolved;
|
|
1262
|
+
});
|
|
1263
|
+
|
|
1264
|
+
// A throwing transport must not hang settlement; it surfaces through the
|
|
1265
|
+
// missing-outcome invariant path instead.
|
|
1266
|
+
const execGuarded = execPromise.catch(() => {});
|
|
1267
|
+
|
|
1268
|
+
const session: ActiveSession = {
|
|
1269
|
+
id: request.sessionId,
|
|
1270
|
+
attempt: request.attempt,
|
|
1271
|
+
controller,
|
|
1272
|
+
transport: transport.backend,
|
|
1273
|
+
writeLease: lease,
|
|
1274
|
+
cancellationDeadlineMs: deadlineMs,
|
|
1275
|
+
settled,
|
|
1276
|
+
};
|
|
1277
|
+
|
|
1278
|
+
let onCancel: (() => void) | undefined;
|
|
1279
|
+
const cancelPromise = new Promise<"cancel">((resolve) => {
|
|
1280
|
+
const sig = this.cancelSignal;
|
|
1281
|
+
if (!sig) return;
|
|
1282
|
+
if (sig.aborted) {
|
|
1283
|
+
resolve("cancel");
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
onCancel = () => resolve("cancel");
|
|
1287
|
+
sig.addEventListener("abort", onCancel, { once: true });
|
|
1288
|
+
});
|
|
1289
|
+
|
|
1290
|
+
let watchdogTimer: ReturnType<typeof setTimeout> | undefined;
|
|
1291
|
+
const watchdogPromise =
|
|
1292
|
+
harnessWatchdogMs !== undefined && harnessWatchdogMs > 0
|
|
1293
|
+
? new Promise<"watchdog">((resolve) => {
|
|
1294
|
+
watchdogTimer = setTimeout(
|
|
1295
|
+
() => resolve("watchdog"),
|
|
1296
|
+
harnessWatchdogMs,
|
|
1297
|
+
);
|
|
1298
|
+
})
|
|
1299
|
+
: undefined;
|
|
1300
|
+
|
|
1301
|
+
try {
|
|
1302
|
+
const racers: Promise<"outcome" | "cancel" | "watchdog">[] = [
|
|
1303
|
+
execGuarded.then(() => "outcome" as const),
|
|
1304
|
+
cancelPromise,
|
|
1305
|
+
];
|
|
1306
|
+
if (watchdogPromise !== undefined) {
|
|
1307
|
+
racers.push(watchdogPromise);
|
|
1308
|
+
}
|
|
1309
|
+
const raced = await Promise.race(racers);
|
|
1310
|
+
|
|
1311
|
+
if (raced === "watchdog") {
|
|
1312
|
+
settlement.markAbortRequested();
|
|
1313
|
+
if (lease.valid) {
|
|
1314
|
+
lease.invalidate("harness watchdog timeout");
|
|
1315
|
+
settlement.markLeaseInvalidated();
|
|
1316
|
+
}
|
|
1317
|
+
fenceClosed = true;
|
|
1318
|
+
sink.close();
|
|
1319
|
+
controller.abort();
|
|
1320
|
+
|
|
1321
|
+
const graceMs = deadlineMs + this.graceMarginMs;
|
|
1322
|
+
await Promise.race([
|
|
1323
|
+
execGuarded,
|
|
1324
|
+
new Promise<void>((resolve) => setTimeout(resolve, graceMs)),
|
|
1325
|
+
]);
|
|
1326
|
+
|
|
1327
|
+
const transportTerminal = settlement.terminal;
|
|
1328
|
+
let receipt: SettlementReceipt;
|
|
1329
|
+
const cancellationSemantics =
|
|
1330
|
+
transport.cancellationSemantics ??
|
|
1331
|
+
(transport.backend === "claude-code"
|
|
1332
|
+
? "process-exit"
|
|
1333
|
+
: this.isTestFake
|
|
1334
|
+
? transport.backend === "opencode"
|
|
1335
|
+
? "provider-proof"
|
|
1336
|
+
: "process-exit"
|
|
1337
|
+
: undefined);
|
|
1338
|
+
const isProviderProof = cancellationSemantics === "provider-proof";
|
|
1339
|
+
|
|
1340
|
+
if (
|
|
1341
|
+
transportTerminal !== undefined &&
|
|
1342
|
+
transportTerminal.origin !== "harness" &&
|
|
1343
|
+
(!isProviderProof || transportTerminal.proof !== undefined)
|
|
1344
|
+
) {
|
|
1345
|
+
receipt = finalize(() =>
|
|
1346
|
+
settlement.receipt("failed", {
|
|
1347
|
+
confirmation:
|
|
1348
|
+
cancellationSemantics === "process-exit"
|
|
1349
|
+
? "process_group_exited"
|
|
1350
|
+
: "sdk_abort_confirmed",
|
|
1351
|
+
processGroupAlive: isProviderProof ? "not_applicable" : "unknown",
|
|
1352
|
+
remoteStatus: "failed",
|
|
1353
|
+
}),
|
|
1354
|
+
);
|
|
1355
|
+
} else {
|
|
1356
|
+
settlement.acceptTerminal("harness", "failed");
|
|
1357
|
+
receipt = finalize(() =>
|
|
1358
|
+
synthesizeUnconfirmed(settlement, {
|
|
1359
|
+
processGroupAlive: isProviderProof ? "not_applicable" : "unknown",
|
|
1360
|
+
outcome: isProviderProof
|
|
1361
|
+
? "local_fenced_remote_unconfirmed"
|
|
1362
|
+
: undefined,
|
|
1363
|
+
warning: `Step timed out after ${harnessWatchdogMs}ms (harness-owned watchdog)`,
|
|
1364
|
+
}),
|
|
1365
|
+
);
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
await this.persistSettlement(step, session, receipt);
|
|
1369
|
+
this.onSessionSettled?.({ session, settlement, receipt });
|
|
1370
|
+
return {
|
|
1371
|
+
session,
|
|
1372
|
+
settlement,
|
|
1373
|
+
outcome: {
|
|
1374
|
+
completion: "failed",
|
|
1375
|
+
protocolIntegrity: "unverified",
|
|
1376
|
+
finalText: "",
|
|
1377
|
+
usage: normalizeUnavailableUsage({
|
|
1378
|
+
wallMs: harnessWatchdogMs ?? 0,
|
|
1379
|
+
billingMode: transport.billingMode,
|
|
1380
|
+
}),
|
|
1381
|
+
stderrTail: `Step timed out after ${harnessWatchdogMs}ms`,
|
|
1382
|
+
timedOut: true,
|
|
1383
|
+
},
|
|
1384
|
+
cancelled: false,
|
|
1385
|
+
receipt,
|
|
1386
|
+
};
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
if (raced === "outcome") {
|
|
1390
|
+
const resolved = outcome;
|
|
1391
|
+
if (resolved === undefined) {
|
|
1392
|
+
sink.close();
|
|
1393
|
+
const receipt = finalize(() =>
|
|
1394
|
+
synthesizeUnconfirmed(settlement, {
|
|
1395
|
+
warning:
|
|
1396
|
+
"transport promise settled without producing an outcome (§5.1 invariant conversion)",
|
|
1397
|
+
}),
|
|
1398
|
+
);
|
|
1399
|
+
await this.persistSettlement(step, session, receipt);
|
|
1400
|
+
this.onSessionSettled?.({ session, settlement, receipt });
|
|
1401
|
+
return { session, settlement, cancelled: true, receipt };
|
|
1402
|
+
}
|
|
1403
|
+
settlement.acceptTerminal(
|
|
1404
|
+
"transport",
|
|
1405
|
+
completionToStatus(resolved),
|
|
1406
|
+
resolved.terminalProof,
|
|
1407
|
+
);
|
|
1408
|
+
if (resolved.terminalProof !== undefined) {
|
|
1409
|
+
settlement.markTerminationConfirmed();
|
|
1410
|
+
}
|
|
1411
|
+
// Data-plane delivery runs while the lease is still valid; settle
|
|
1412
|
+
// (sink closure + fence) happens strictly after it. A throw here
|
|
1413
|
+
// (e.g. an attempt-log I/O failure) must NOT skip settlement: §5.1
|
|
1414
|
+
// requires settled to always resolve, so the catch below settles a
|
|
1415
|
+
// failed session instead of leaving the lease valid forever.
|
|
1416
|
+
let delivery: AttemptDelivery;
|
|
1417
|
+
try {
|
|
1418
|
+
delivery = await onData(resolved, settlement);
|
|
1419
|
+
} catch (error) {
|
|
1420
|
+
if (lease.valid) {
|
|
1421
|
+
lease.invalidate("data-plane write failure");
|
|
1422
|
+
}
|
|
1423
|
+
fenceClosed = true;
|
|
1424
|
+
sink.close();
|
|
1425
|
+
settlement.acceptTerminal("harness", "failed");
|
|
1426
|
+
const receipt = finalize(() =>
|
|
1427
|
+
synthesizeInternalFailure(settlement, error),
|
|
1428
|
+
);
|
|
1429
|
+
await this.persistSettlement(step, session, receipt);
|
|
1430
|
+
this.onSessionSettled?.({ session, settlement, receipt });
|
|
1431
|
+
return {
|
|
1432
|
+
session,
|
|
1433
|
+
settlement,
|
|
1434
|
+
outcome: resolved,
|
|
1435
|
+
cancelled: false,
|
|
1436
|
+
receipt,
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
sink.close();
|
|
1440
|
+
const receipt = finalize(() =>
|
|
1441
|
+
settlement.receipt(
|
|
1442
|
+
resolved.completion === "success" ? "completed" : "failed",
|
|
1443
|
+
{
|
|
1444
|
+
confirmation: resolved.terminalProof
|
|
1445
|
+
? "process_group_exited"
|
|
1446
|
+
: "not_required",
|
|
1447
|
+
processGroupAlive: "unknown",
|
|
1448
|
+
remoteStatus:
|
|
1449
|
+
resolved.completion === "success" ? "completed" : "failed",
|
|
1450
|
+
},
|
|
1451
|
+
),
|
|
1452
|
+
);
|
|
1453
|
+
await this.persistSettlement(step, session, receipt);
|
|
1454
|
+
this.onSessionSettled?.({ session, settlement, receipt });
|
|
1455
|
+
return {
|
|
1456
|
+
session,
|
|
1457
|
+
settlement,
|
|
1458
|
+
outcome: resolved,
|
|
1459
|
+
delivery,
|
|
1460
|
+
cancelled: false,
|
|
1461
|
+
receipt,
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
// §5.3 steps 2–6 in order.
|
|
1465
|
+
settlement.markAbortRequested(); // step 3 precondition: cancel recorded
|
|
1466
|
+
if (lease.valid) {
|
|
1467
|
+
lease.invalidate("cancellation fence (§5.3 step 2)");
|
|
1468
|
+
settlement.markLeaseInvalidated();
|
|
1469
|
+
}
|
|
1470
|
+
fenceClosed = true;
|
|
1471
|
+
sink.close(); // step 2: sinks close with the lease; slot stays open
|
|
1472
|
+
controller.abort(); // step 3: abort each active controller exactly once
|
|
1473
|
+
|
|
1474
|
+
const graceMs = deadlineMs + this.graceMarginMs;
|
|
1475
|
+
await Promise.race([
|
|
1476
|
+
execGuarded,
|
|
1477
|
+
new Promise<void>((resolve) => setTimeout(resolve, graceMs)),
|
|
1478
|
+
]);
|
|
1479
|
+
|
|
1480
|
+
// Whether the transport's terminal landed before or after the fence
|
|
1481
|
+
// closed, the compare-and-set slot is the single source of truth: a
|
|
1482
|
+
// transport-origin terminal recorded there means the cancellation was
|
|
1483
|
+
// confirmed within grace; otherwise §5.3 steps 5–6 apply.
|
|
1484
|
+
const transportTerminal = settlement.terminal;
|
|
1485
|
+
let receipt: SettlementReceipt;
|
|
1486
|
+
const cancellationSemantics =
|
|
1487
|
+
transport.cancellationSemantics ??
|
|
1488
|
+
(transport.backend === "claude-code"
|
|
1489
|
+
? "process-exit"
|
|
1490
|
+
: this.isTestFake
|
|
1491
|
+
? transport.backend === "opencode"
|
|
1492
|
+
? "provider-proof"
|
|
1493
|
+
: "process-exit"
|
|
1494
|
+
: undefined);
|
|
1495
|
+
const isProviderProof = cancellationSemantics === "provider-proof";
|
|
1496
|
+
|
|
1497
|
+
if (
|
|
1498
|
+
transportTerminal !== undefined &&
|
|
1499
|
+
transportTerminal.origin !== "harness" &&
|
|
1500
|
+
(!isProviderProof || transportTerminal.proof !== undefined)
|
|
1501
|
+
) {
|
|
1502
|
+
receipt = finalize(() =>
|
|
1503
|
+
settlement.receipt("cancelled_confirmed", {
|
|
1504
|
+
confirmation:
|
|
1505
|
+
cancellationSemantics === "process-exit"
|
|
1506
|
+
? "process_group_exited"
|
|
1507
|
+
: "sdk_abort_confirmed",
|
|
1508
|
+
processGroupAlive: isProviderProof ? "not_applicable" : "unknown",
|
|
1509
|
+
remoteStatus: "cancelled",
|
|
1510
|
+
}),
|
|
1511
|
+
);
|
|
1512
|
+
} else {
|
|
1513
|
+
// §5.3 step 5: exactly one harness-origin non-success terminal, won
|
|
1514
|
+
// atomically by the compare-and-set slot.
|
|
1515
|
+
settlement.acceptTerminal("harness", "cancelled");
|
|
1516
|
+
// §5.3 step 6: no transport receipt facts → synthesize + quarantine.
|
|
1517
|
+
receipt = finalize(() =>
|
|
1518
|
+
synthesizeUnconfirmed(settlement, {
|
|
1519
|
+
processGroupAlive: isProviderProof ? "not_applicable" : "unknown",
|
|
1520
|
+
outcome: isProviderProof
|
|
1521
|
+
? "local_fenced_remote_unconfirmed"
|
|
1522
|
+
: undefined,
|
|
1523
|
+
}),
|
|
1524
|
+
);
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
await this.persistSettlement(step, session, receipt);
|
|
1528
|
+
this.onSessionSettled?.({ session, settlement, receipt });
|
|
1529
|
+
return { session, settlement, cancelled: true, receipt };
|
|
1530
|
+
} finally {
|
|
1531
|
+
if (watchdogTimer !== undefined) {
|
|
1532
|
+
clearTimeout(watchdogTimer);
|
|
1533
|
+
}
|
|
1534
|
+
if (onCancel && this.cancelSignal) {
|
|
1535
|
+
this.cancelSignal.removeEventListener("abort", onCancel);
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
private async admitAndExecute(args: {
|
|
1541
|
+
readonly step: StepSpec;
|
|
1542
|
+
readonly canonicalCwd: string;
|
|
1543
|
+
readonly verifiedBinaryPath: string;
|
|
1544
|
+
readonly childEnv: Readonly<Record<string, string>>;
|
|
1545
|
+
readonly projection?: CredentialProjection;
|
|
1546
|
+
// #279: whether THIS run's projection degraded onto the unstripped
|
|
1547
|
+
// operator env — see the WHY on `run()`'s degrade catch. Threaded down
|
|
1548
|
+
// to `runAttempt` so every attempt of this step can open the same
|
|
1549
|
+
// degraded-fence reservation.
|
|
1550
|
+
readonly projectionDegraded: boolean;
|
|
1551
|
+
readonly transport: ProviderTransport;
|
|
1552
|
+
readonly admissionIdentity: {
|
|
1553
|
+
readonly executable: string;
|
|
1554
|
+
readonly provider: string;
|
|
1555
|
+
};
|
|
1556
|
+
// #150: `isolation.credentialProjectionId` to stamp when `projection` is
|
|
1557
|
+
// undefined — see the call site in `run()` for why this is not always
|
|
1558
|
+
// "operator-env-fallback" any more.
|
|
1559
|
+
readonly fallbackCredentialProjectionId: string;
|
|
1560
|
+
}): Promise<StepResult> {
|
|
1561
|
+
const {
|
|
1562
|
+
step,
|
|
1563
|
+
canonicalCwd,
|
|
1564
|
+
verifiedBinaryPath,
|
|
1565
|
+
childEnv,
|
|
1566
|
+
projection,
|
|
1567
|
+
projectionDegraded,
|
|
1568
|
+
transport,
|
|
1569
|
+
admissionIdentity,
|
|
1570
|
+
fallbackCredentialProjectionId,
|
|
1571
|
+
} = args;
|
|
1572
|
+
|
|
1573
|
+
// 3. Admission gate: called once after successful authorization
|
|
1574
|
+
if (this.admissionGate) {
|
|
1575
|
+
await this.admissionGate.admit(step);
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
// 3.5 §6.3 prompt integrity inputs: read once, bound it, and pin its hash
|
|
1579
|
+
// into every TransportRequest so the transport can re-verify the bytes
|
|
1580
|
+
// immediately before spawn (the gap in between is exactly the TOCTOU the
|
|
1581
|
+
// transport check exists to close).
|
|
1582
|
+
let systemPromptSha256: string;
|
|
1583
|
+
try {
|
|
1584
|
+
const promptBytes = await Bun.file(step.systemPromptPath).bytes();
|
|
1585
|
+
if (promptBytes.byteLength > MAX_SYSTEM_PROMPT_BYTES) {
|
|
1586
|
+
return {
|
|
1587
|
+
name: step.name,
|
|
1588
|
+
status: "failed",
|
|
1589
|
+
usage: zeroUsage(),
|
|
1590
|
+
attempts: 0,
|
|
1591
|
+
stderrTail: `System prompt exceeds the predeclared ${MAX_SYSTEM_PROMPT_BYTES}-byte bound (${promptBytes.byteLength} bytes); step failed before any spawn`,
|
|
1592
|
+
resultText: "",
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
1596
|
+
hasher.update(promptBytes);
|
|
1597
|
+
systemPromptSha256 = hasher.digest("hex");
|
|
1598
|
+
} catch {
|
|
1599
|
+
return {
|
|
1600
|
+
name: step.name,
|
|
1601
|
+
status: "failed",
|
|
1602
|
+
usage: zeroUsage(),
|
|
1603
|
+
attempts: 0,
|
|
1604
|
+
stderrTail: `Could not read system prompt at ${step.systemPromptPath}; step failed before any spawn`,
|
|
1605
|
+
resultText: "",
|
|
1606
|
+
};
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
// 4. Execution loop with retries — D1-08 PR0 (D1-07 wiring): ONE
|
|
1610
|
+
// `runAttempt` per attempt (transient retry AND format retry are now the
|
|
1611
|
+
// same shape), driven by the live §7 retry decision
|
|
1612
|
+
// (decideRetryDisposition/resolveFailureCause) instead of the legacy
|
|
1613
|
+
// step-runner failure classifier.
|
|
1614
|
+
// D1-08 PR2 (§8): NormalizedUsage is the single source of truth across
|
|
1615
|
+
// attempts now; the legacy SessionUsage is recomputed from it via
|
|
1616
|
+
// `projectLegacyUsage` at every return point below, never accumulated
|
|
1617
|
+
// independently — that is what keeps the two shapes unable to drift.
|
|
1618
|
+
// `undefined` until the first attempt settles: seeding it with a
|
|
1619
|
+
// billingMode/costSource-less zero would force `sumNormalizedUsage` to
|
|
1620
|
+
// collapse the run's real billing mode to "unknown" on the very first sum.
|
|
1621
|
+
let totalUsageV2: NormalizedUsage | undefined;
|
|
1622
|
+
let attempts = 0;
|
|
1623
|
+
let lastOutcome: TransportOutcome | undefined;
|
|
1624
|
+
// The transport cause of the most recent failed attempt. Reset by every attempt
|
|
1625
|
+
// that fails without one, so the step reports only the cause it ended on.
|
|
1626
|
+
let lastFailureCause: TransportFailureCause | undefined;
|
|
1627
|
+
let cancelHit = false;
|
|
1628
|
+
// D1-08 PR5b (§9.1): every attempt that reached the reserve step, in
|
|
1629
|
+
// order, each already finalized to a terminal reservation state by the
|
|
1630
|
+
// time `runAttempt` returns. Empty when no `spendLedger` is configured.
|
|
1631
|
+
const reservations: SpendReservation[] = [];
|
|
1632
|
+
// Coupling 2 (spend-limiter.ts): attempt 1 always has a token —
|
|
1633
|
+
// `beginStep()` — because there is no disposition yet to consult. Every
|
|
1634
|
+
// later cycle's token comes from `nextCycle(disposition)`, which is
|
|
1635
|
+
// `undefined` for a `terminal` disposition — structurally preventing a
|
|
1636
|
+
// terminal attempt from opening a new reserve+admit cycle.
|
|
1637
|
+
let reserveToken: ReserveToken = beginStep();
|
|
1638
|
+
|
|
1639
|
+
const deadlineMs = await this.resolveCancellationDeadlineMs(transport);
|
|
1640
|
+
|
|
1641
|
+
// D3: bind the new transient budget to the EXISTING per-step knob so a
|
|
1642
|
+
// step's worst-case spawn count stays byte-identical to the legacy loop
|
|
1643
|
+
// (1 initial + up to `maxAttempts - 1` transient retries).
|
|
1644
|
+
// failure-policy's own default (3) would silently raise worst-case
|
|
1645
|
+
// spawns from 3 to 5 per step — a 67% cost increase nobody asked for.
|
|
1646
|
+
const maxTransientAttempts = Math.max(0, step.maxAttempts - 1);
|
|
1647
|
+
|
|
1648
|
+
let retryState: RetryState = {
|
|
1649
|
+
transientAttemptsUsed: 0,
|
|
1650
|
+
formatRetriesUsed: 0,
|
|
1651
|
+
};
|
|
1652
|
+
// Set only by a `retry_format_reminder` disposition; consumed by the
|
|
1653
|
+
// very next iteration, which is what makes that iteration a
|
|
1654
|
+
// "format-retry" rather than a plain "attempt" (row 7).
|
|
1655
|
+
let pendingFormatPrompt: string | undefined;
|
|
1656
|
+
|
|
1657
|
+
for (;;) {
|
|
1658
|
+
// §5.3 step 1: no new attempt starts once cancellation is admitted —
|
|
1659
|
+
// covers both a fresh transient attempt and a format-reminder retry,
|
|
1660
|
+
// which are now the same loop iteration shape (row 6).
|
|
1661
|
+
if (this.cancelSignal?.aborted) {
|
|
1662
|
+
cancelHit = true;
|
|
1663
|
+
break;
|
|
1664
|
+
}
|
|
1665
|
+
attempts++;
|
|
1666
|
+
const kind: "attempt" | "format-retry" =
|
|
1667
|
+
pendingFormatPrompt === undefined ? "attempt" : "format-retry";
|
|
1668
|
+
|
|
1669
|
+
const request: TransportRequest = {
|
|
1670
|
+
sessionId: `${step.name}-${Date.now()}-${attempts}`,
|
|
1671
|
+
attempt: attempts,
|
|
1672
|
+
executionModel: step.model,
|
|
1673
|
+
effort: step.effort,
|
|
1674
|
+
route:
|
|
1675
|
+
step.route ??
|
|
1676
|
+
(transport.defaultRoute
|
|
1677
|
+
? {
|
|
1678
|
+
...transport.defaultRoute,
|
|
1679
|
+
modelSnapshot:
|
|
1680
|
+
step.model || transport.defaultRoute.modelSnapshot,
|
|
1681
|
+
}
|
|
1682
|
+
: {
|
|
1683
|
+
backend: transport.backend,
|
|
1684
|
+
provider: admissionIdentity.provider,
|
|
1685
|
+
modelFamily:
|
|
1686
|
+
transport.backend === "claude-code"
|
|
1687
|
+
? "claude"
|
|
1688
|
+
: admissionIdentity.provider,
|
|
1689
|
+
modelSnapshot: step.model,
|
|
1690
|
+
}),
|
|
1691
|
+
systemPromptPath: step.systemPromptPath,
|
|
1692
|
+
systemPromptSha256,
|
|
1693
|
+
userPrompt: pendingFormatPrompt ?? step.prompt,
|
|
1694
|
+
cwd: canonicalCwd,
|
|
1695
|
+
tools: step.tools,
|
|
1696
|
+
mcpConfigPath: step.mcpConfigPath,
|
|
1697
|
+
isolation: projection
|
|
1698
|
+
? {
|
|
1699
|
+
credentialProjectionId: projection.projectionId,
|
|
1700
|
+
env: childEnv,
|
|
1701
|
+
syntheticHome: projection.syntheticHome,
|
|
1702
|
+
syntheticConfigHome: projection.syntheticConfigHome,
|
|
1703
|
+
syntheticTmp: projection.syntheticTmp,
|
|
1704
|
+
verifiedBinaryPath,
|
|
1705
|
+
}
|
|
1706
|
+
: {
|
|
1707
|
+
// Must describe what ACTUALLY ran, not claim a synthetic
|
|
1708
|
+
// identity the env contradicts (§6.1 invariant: env.HOME ===
|
|
1709
|
+
// syntheticHome). The marker id makes the absence auditable in
|
|
1710
|
+
// artifacts — "operator-env-fallback" (no broker, or a
|
|
1711
|
+
// missing_subscription_record degrade) or, since #150,
|
|
1712
|
+
// "server-lifetime" (the transport declared it never wanted a
|
|
1713
|
+
// per-step projection in the first place; see
|
|
1714
|
+
// `fallbackCredentialProjectionId` above).
|
|
1715
|
+
credentialProjectionId: fallbackCredentialProjectionId,
|
|
1716
|
+
env: childEnv,
|
|
1717
|
+
syntheticHome: childEnv.HOME ?? "",
|
|
1718
|
+
syntheticConfigHome:
|
|
1719
|
+
childEnv.CLAUDE_CONFIG_DIR ??
|
|
1720
|
+
path.join(childEnv.HOME ?? "", ".claude"),
|
|
1721
|
+
syntheticTmp: childEnv.TMPDIR ?? "",
|
|
1722
|
+
verifiedBinaryPath,
|
|
1723
|
+
},
|
|
1724
|
+
};
|
|
1725
|
+
pendingFormatPrompt = undefined;
|
|
1726
|
+
|
|
1727
|
+
const attemptResult = await this.runAttempt({
|
|
1728
|
+
step,
|
|
1729
|
+
attempt: attempts,
|
|
1730
|
+
kind,
|
|
1731
|
+
request,
|
|
1732
|
+
deadlineMs,
|
|
1733
|
+
transport,
|
|
1734
|
+
reserveToken,
|
|
1735
|
+
projectionDegraded,
|
|
1736
|
+
harnessWatchdogMs: step.timeoutMs,
|
|
1737
|
+
});
|
|
1738
|
+
|
|
1739
|
+
if (attemptResult.reservation !== undefined) {
|
|
1740
|
+
reservations.push(attemptResult.reservation);
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
if (attemptResult.kind === "cancelled") {
|
|
1744
|
+
cancelHit = true;
|
|
1745
|
+
break;
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
lastOutcome = attemptResult.outcome;
|
|
1749
|
+
totalUsageV2 =
|
|
1750
|
+
totalUsageV2 === undefined
|
|
1751
|
+
? attemptResult.outcome.usage
|
|
1752
|
+
: sumNormalizedUsage(totalUsageV2, attemptResult.outcome.usage);
|
|
1753
|
+
|
|
1754
|
+
if (attemptResult.kind === "delivered") {
|
|
1755
|
+
return {
|
|
1756
|
+
name: step.name,
|
|
1757
|
+
status: "ok",
|
|
1758
|
+
output: attemptResult.parsed,
|
|
1759
|
+
usage: projectLegacyUsage(totalUsageV2),
|
|
1760
|
+
usageV2: totalUsageV2,
|
|
1761
|
+
attempts,
|
|
1762
|
+
...(reservations.length === 0 ? {} : { reservations }),
|
|
1763
|
+
stderrTail: lastOutcome?.stderrTail ?? "",
|
|
1764
|
+
resultText: lastOutcome?.finalText.slice(-8192) ?? "",
|
|
1765
|
+
};
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
// attemptResult.kind === "failed"
|
|
1769
|
+
lastFailureCause =
|
|
1770
|
+
attemptResult.resolution.kind === "cause" &&
|
|
1771
|
+
isTransportFailureCause(attemptResult.resolution.cause)
|
|
1772
|
+
? attemptResult.resolution.cause
|
|
1773
|
+
: undefined;
|
|
1774
|
+
if (attemptResult.resolution.kind === "legacy_terminal") {
|
|
1775
|
+
break;
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
const disposition = decideRetryDisposition(
|
|
1779
|
+
attemptResult.resolution.cause,
|
|
1780
|
+
retryState,
|
|
1781
|
+
{ maxTransientAttempts },
|
|
1782
|
+
);
|
|
1783
|
+
|
|
1784
|
+
if (
|
|
1785
|
+
disposition.action === "retry_now" ||
|
|
1786
|
+
disposition.action === "retry_after"
|
|
1787
|
+
) {
|
|
1788
|
+
retryState = {
|
|
1789
|
+
...retryState,
|
|
1790
|
+
transientAttemptsUsed: retryState.transientAttemptsUsed + 1,
|
|
1791
|
+
};
|
|
1792
|
+
notifyRetry(step, {
|
|
1793
|
+
step: step.name,
|
|
1794
|
+
attempt: attempts + 1,
|
|
1795
|
+
maxAttempts: step.maxAttempts,
|
|
1796
|
+
reason: disposition.budget,
|
|
1797
|
+
});
|
|
1798
|
+
if (disposition.action === "retry_after") {
|
|
1799
|
+
if (this.cancelSignal) {
|
|
1800
|
+
await cancellableSleep(disposition.delayMs, this.cancelSignal);
|
|
1801
|
+
} else {
|
|
1802
|
+
await this.sleep(disposition.delayMs);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
// Non-null: `nextCycle` only returns `undefined` for a `terminal`
|
|
1806
|
+
// disposition, and this branch is reached only on `retry_now`/
|
|
1807
|
+
// `retry_after`.
|
|
1808
|
+
// biome-ignore lint/style/noNonNullAssertion: guaranteed by nextCycle's own contract for a non-terminal disposition
|
|
1809
|
+
reserveToken = nextCycle(disposition)!;
|
|
1810
|
+
continue;
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
if (disposition.action === "retry_format_reminder") {
|
|
1814
|
+
retryState = {
|
|
1815
|
+
...retryState,
|
|
1816
|
+
formatRetriesUsed: retryState.formatRetriesUsed + 1,
|
|
1817
|
+
};
|
|
1818
|
+
notifyRetry(step, {
|
|
1819
|
+
step: step.name,
|
|
1820
|
+
attempt: attempts + 1,
|
|
1821
|
+
maxAttempts: step.maxAttempts,
|
|
1822
|
+
reason: disposition.budget,
|
|
1823
|
+
});
|
|
1824
|
+
pendingFormatPrompt =
|
|
1825
|
+
step.prompt + formatRetryReminder(attemptResult.parseError);
|
|
1826
|
+
// biome-ignore lint/style/noNonNullAssertion: guaranteed by nextCycle's own contract for a non-terminal disposition
|
|
1827
|
+
reserveToken = nextCycle(disposition)!;
|
|
1828
|
+
continue;
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
// disposition.action === "terminal"
|
|
1832
|
+
break;
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
if (cancelHit) {
|
|
1836
|
+
// Name the receipt that was ACTUALLY written for THIS step and attempt.
|
|
1837
|
+
// The previous text pointed at a bare "settlement.json" that has never
|
|
1838
|
+
// existed on disk. Cancellation before the first attempt admits no
|
|
1839
|
+
// session at all, so there is no receipt to name — saying so beats
|
|
1840
|
+
// naming attempt0.json, which would be the same lie in a new font.
|
|
1841
|
+
const receiptPointer =
|
|
1842
|
+
attempts >= 1
|
|
1843
|
+
? `see ${receiptPathFor(step, attempts)}`
|
|
1844
|
+
: "no attempt was admitted, so no receipt was written";
|
|
1845
|
+
return {
|
|
1846
|
+
name: step.name,
|
|
1847
|
+
status: "failed",
|
|
1848
|
+
usage:
|
|
1849
|
+
totalUsageV2 === undefined
|
|
1850
|
+
? zeroUsage()
|
|
1851
|
+
: projectLegacyUsage(totalUsageV2),
|
|
1852
|
+
...(totalUsageV2 === undefined ? {} : { usageV2: totalUsageV2 }),
|
|
1853
|
+
attempts,
|
|
1854
|
+
...(reservations.length === 0 ? {} : { reservations }),
|
|
1855
|
+
stderrTail: [
|
|
1856
|
+
lastOutcome?.stderrTail ?? "",
|
|
1857
|
+
`[pr-hero] step cancelled; settled per §5.3 (${receiptPointer})`,
|
|
1858
|
+
]
|
|
1859
|
+
.filter(Boolean)
|
|
1860
|
+
.join("\n"),
|
|
1861
|
+
resultText: lastOutcome?.finalText.slice(-8192) ?? "",
|
|
1862
|
+
};
|
|
1863
|
+
}
|
|
1864
|
+
|
|
1865
|
+
return {
|
|
1866
|
+
name: step.name,
|
|
1867
|
+
status: "failed",
|
|
1868
|
+
usage:
|
|
1869
|
+
totalUsageV2 === undefined
|
|
1870
|
+
? zeroUsage()
|
|
1871
|
+
: projectLegacyUsage(totalUsageV2),
|
|
1872
|
+
...(totalUsageV2 === undefined ? {} : { usageV2: totalUsageV2 }),
|
|
1873
|
+
attempts,
|
|
1874
|
+
...(reservations.length === 0 ? {} : { reservations }),
|
|
1875
|
+
...(lastFailureCause === undefined
|
|
1876
|
+
? {}
|
|
1877
|
+
: { failureCause: lastFailureCause }),
|
|
1878
|
+
...(lastOutcome?.failureDetail === undefined
|
|
1879
|
+
? {}
|
|
1880
|
+
: { failureDetail: lastOutcome.failureDetail }),
|
|
1881
|
+
...(lastOutcome?.failureCredential === undefined
|
|
1882
|
+
? {}
|
|
1883
|
+
: { failureCredential: lastOutcome.failureCredential }),
|
|
1884
|
+
stderrTail: lastOutcome?.stderrTail ?? "",
|
|
1885
|
+
resultText: lastOutcome?.finalText.slice(-8192) ?? "",
|
|
1886
|
+
};
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
// D1-08 PR0: the ONE place a single attempt (transient OR format-retry —
|
|
1890
|
+
// `kind` only affects logging/request framing) executes, gets logged, and
|
|
1891
|
+
// has its failure resolved to a §7 cause. Replaces the two
|
|
1892
|
+
// near-duplicate closures `admitAndExecute` used to run inline, one for
|
|
1893
|
+
// the initial/transient loop and a second, slightly-out-of-sync one for
|
|
1894
|
+
// the format retry (row 9's `.catch` asymmetry was exactly that drift).
|
|
1895
|
+
private async runAttempt(args: {
|
|
1896
|
+
readonly step: StepSpec;
|
|
1897
|
+
readonly attempt: number;
|
|
1898
|
+
readonly kind: "attempt" | "format-retry";
|
|
1899
|
+
readonly request: TransportRequest;
|
|
1900
|
+
readonly deadlineMs: number;
|
|
1901
|
+
readonly transport: ProviderTransport;
|
|
1902
|
+
readonly reserveToken: ReserveToken;
|
|
1903
|
+
readonly projectionDegraded: boolean;
|
|
1904
|
+
readonly harnessWatchdogMs?: number;
|
|
1905
|
+
}): Promise<AttemptRunResult> {
|
|
1906
|
+
const {
|
|
1907
|
+
step,
|
|
1908
|
+
attempt,
|
|
1909
|
+
kind,
|
|
1910
|
+
request,
|
|
1911
|
+
deadlineMs,
|
|
1912
|
+
transport,
|
|
1913
|
+
reserveToken,
|
|
1914
|
+
projectionDegraded,
|
|
1915
|
+
harnessWatchdogMs,
|
|
1916
|
+
} = args;
|
|
1917
|
+
|
|
1918
|
+
// D1-08 PR5b (§9.1 five-step order): acquire (1) → reserve (2) →
|
|
1919
|
+
// execute (3) → settle-or-unresolved (4) → release-if-unstarted (5),
|
|
1920
|
+
// all inside ONE acquire/finally-release wrapper. Lease release stays a
|
|
1921
|
+
// SEPARATE `finally`, independent of the ledger transition (§9.2: an
|
|
1922
|
+
// abort AFTER the provider attempt started releases the local
|
|
1923
|
+
// concurrency slot but NOT the spend reservation) — this is why the
|
|
1924
|
+
// ledger's own finalization lives in the `try` body below, never in the
|
|
1925
|
+
// outer `finally` beside `lease?.release()`.
|
|
1926
|
+
let lease: AttemptLease | undefined;
|
|
1927
|
+
try {
|
|
1928
|
+
if (this.attemptAdmissionGate) {
|
|
1929
|
+
const acquireSignal =
|
|
1930
|
+
this.cancelSignal ??
|
|
1931
|
+
AbortSignal.timeout(this.attemptAdmissionTimeoutMs);
|
|
1932
|
+
lease = await this.attemptAdmissionGate.acquire({
|
|
1933
|
+
sessionId: request.sessionId,
|
|
1934
|
+
attempt,
|
|
1935
|
+
rateLimitBucketId: this.rateLimitBucketId,
|
|
1936
|
+
signal: acquireSignal,
|
|
1937
|
+
});
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
// Step 2: reserve spend transactionally. Left unconfigured, no
|
|
1941
|
+
// `SpendLedger` call is made at all — attempts reserve nothing,
|
|
1942
|
+
// exactly the ledger-free shape PR5a shipped. A refusing ledger
|
|
1943
|
+
// (fenced bucket, spec: "Unresolved bucket blocks the next paid
|
|
1944
|
+
// attempt") throws HERE, before the transport is ever invoked.
|
|
1945
|
+
//
|
|
1946
|
+
// #279: `this.reservesSpend` is the BINDING's own decision (a metered
|
|
1947
|
+
// or free-credential route) and defaults `true` so every existing
|
|
1948
|
+
// caller that configures a `spendLedger` directly keeps reserving
|
|
1949
|
+
// exactly as it did before this option existed (see its WHY on
|
|
1950
|
+
// `StepExecutionHarnessOptions`). A `false` binding (a subscription
|
|
1951
|
+
// route) still opens ONE reservation when THIS attempt's own
|
|
1952
|
+
// projection degraded onto an ambient key/token that bills metered —
|
|
1953
|
+
// `projectionDegraded` is per-`run()`, the fact only the harness
|
|
1954
|
+
// knows. The two conditions are structurally exclusive, never both at
|
|
1955
|
+
// once: a degraded projection requires a `credentialBroker`, and a
|
|
1956
|
+
// metered/free binding never gets one attached
|
|
1957
|
+
// (`credentialKindForRoute`, runner-authority.ts — a broker's
|
|
1958
|
+
// presence is what forces the subscription kind in the first place).
|
|
1959
|
+
const degradedAmbientSpend =
|
|
1960
|
+
projectionDegraded && envBillsMetered(request.isolation.env);
|
|
1961
|
+
// #279: a degraded-only reservation (the binding itself does not
|
|
1962
|
+
// reserve) fences the credential that ACTUALLY incurred the spend —
|
|
1963
|
+
// the ambient key/token, via `degradedProjectionBucketId` (the bucket
|
|
1964
|
+
// a METERED binding of this SAME provider+credential would compute,
|
|
1965
|
+
// production-runtime.ts) — never this binding's own subscription
|
|
1966
|
+
// bucket. Fencing the subscription bucket would block the NEXT
|
|
1967
|
+
// non-degraded attempt on the SAME OAuth credential over dollars that
|
|
1968
|
+
// credential never spent. NO fallback to `rateLimitBucketId` on this
|
|
1969
|
+
// branch: the constructor refuses to build a `reservesSpend: false` +
|
|
1970
|
+
// `spendLedger` harness without `degradedProjectionBucketId` (see the
|
|
1971
|
+
// throw in the constructor), so a silent wrong-bucket fence can never
|
|
1972
|
+
// happen here — only `rateLimitBucketId` itself, on the OTHER branch,
|
|
1973
|
+
// is a real fallback (the `reservesSpend: true` shape, which always
|
|
1974
|
+
// wants its own bucket).
|
|
1975
|
+
let bucketId = this.rateLimitBucketId;
|
|
1976
|
+
if (!this.reservesSpend && degradedAmbientSpend) {
|
|
1977
|
+
// biome-ignore lint/style/noNonNullAssertion: guaranteed by the constructor guard above — reservesSpend:false with a spendLedger cannot exist without degradedProjectionBucketId.
|
|
1978
|
+
bucketId = this.degradedProjectionBucketId!;
|
|
1979
|
+
}
|
|
1980
|
+
let reservation: SpendReservation | undefined;
|
|
1981
|
+
if (this.spendLedger && (this.reservesSpend || degradedAmbientSpend)) {
|
|
1982
|
+
reservation = await this.spendLedger.reserve(
|
|
1983
|
+
{
|
|
1984
|
+
bucketId,
|
|
1985
|
+
reservedUsd: this.reservedUsdPerAttempt,
|
|
1986
|
+
sessionId: request.sessionId,
|
|
1987
|
+
attempt,
|
|
1988
|
+
},
|
|
1989
|
+
reserveToken,
|
|
1990
|
+
);
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
// §9.2 / spec "Abort timing determines release": the ONLY point at
|
|
1994
|
+
// which the provider attempt has PROVABLY never started — a
|
|
1995
|
+
// cancellation that raced ahead of the reservation itself, before
|
|
1996
|
+
// `runAdmittedAttempt` (and therefore the transport) is ever called.
|
|
1997
|
+
// Once execution begins below, an abort must NOT release the
|
|
1998
|
+
// reservation; it settles as unresolved instead (finalizeReservation).
|
|
1999
|
+
if (reservation !== undefined && this.cancelSignal?.aborted) {
|
|
2000
|
+
await this.spendLedger?.releaseUnstarted(
|
|
2001
|
+
reservation.reservationId,
|
|
2002
|
+
`${reservation.reservationId}:release-unstarted`,
|
|
2003
|
+
);
|
|
2004
|
+
return {
|
|
2005
|
+
kind: "cancelled",
|
|
2006
|
+
reservation: { ...reservation, state: "released_unstarted" },
|
|
2007
|
+
};
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
// Step 3: mark the attempt started and execute.
|
|
2011
|
+
const result = await this.runAdmittedAttempt({
|
|
2012
|
+
step,
|
|
2013
|
+
attempt,
|
|
2014
|
+
kind,
|
|
2015
|
+
request,
|
|
2016
|
+
deadlineMs,
|
|
2017
|
+
transport,
|
|
2018
|
+
harnessWatchdogMs,
|
|
2019
|
+
});
|
|
2020
|
+
|
|
2021
|
+
// Step 4: settle actual known cost exactly once (or route to
|
|
2022
|
+
// markUnresolvedRemote when usage completeness never became
|
|
2023
|
+
// "complete" — coupling 1, spend-limiter.ts).
|
|
2024
|
+
if (reservation === undefined) return result;
|
|
2025
|
+
const finalized = await this.finalizeReservation(
|
|
2026
|
+
reservation,
|
|
2027
|
+
result,
|
|
2028
|
+
transport,
|
|
2029
|
+
);
|
|
2030
|
+
// #182 follow-up fail-fast: a free-declared route that settled with
|
|
2031
|
+
// reason `free_nonzero_cost` means the model flipped from free to
|
|
2032
|
+
// metered mid-flight and the provider priced the work. Unresolved alone
|
|
2033
|
+
// only fences the bucket while the retry loop keeps spending attempts on
|
|
2034
|
+
// the flipped model, so the attempt FAILS CLOSED here with no retry —
|
|
2035
|
+
// `legacy_terminal` breaks the run loop before `decideRetryDisposition`
|
|
2036
|
+
// can resurrect it, and the fence from `markUnresolvedRemote` (already
|
|
2037
|
+
// applied inside `finalizeReservation` — the spend may be real) blocks
|
|
2038
|
+
// the next reserve. Free usage with NO cash figure carries no reason and
|
|
2039
|
+
// releases without fencing (finalizeReservation's free-route fence
|
|
2040
|
+
// scope) — no cash figure means no evidence of billing. Incomplete
|
|
2041
|
+
// usage with a non-free mode still fences: no usage mode to exonerate
|
|
2042
|
+
// it, so fail closed.
|
|
2043
|
+
if (
|
|
2044
|
+
finalized.decision.kind === "unresolved" &&
|
|
2045
|
+
finalized.decision.reason === "free_nonzero_cost"
|
|
2046
|
+
) {
|
|
2047
|
+
const known =
|
|
2048
|
+
finalized.decision.knownUsd ?? finalized.reservation.knownUsd;
|
|
2049
|
+
const flipNote =
|
|
2050
|
+
`[pr-hero] free-route cost flip: provider reported $${known} cash ` +
|
|
2051
|
+
`on a free-declared route; the model may have flipped from free to ` +
|
|
2052
|
+
`metered — re-probe before retrying (no retry attempted)`;
|
|
2053
|
+
if (result.kind === "cancelled") {
|
|
2054
|
+
return {
|
|
2055
|
+
kind: "failed",
|
|
2056
|
+
outcome: {
|
|
2057
|
+
completion: "failed",
|
|
2058
|
+
protocolIntegrity: "unverified",
|
|
2059
|
+
finalText: "",
|
|
2060
|
+
usage: normalizeUnavailableUsage({ wallMs: 0 }),
|
|
2061
|
+
stderrTail: flipNote,
|
|
2062
|
+
},
|
|
2063
|
+
resolution: { kind: "legacy_terminal" },
|
|
2064
|
+
reservation: finalized.reservation,
|
|
2065
|
+
};
|
|
2066
|
+
}
|
|
2067
|
+
// A DELIVERED attempt already persisted its parsed output
|
|
2068
|
+
// (writeJsonAtomically in runAdmittedAttempt) and the loop returns ok
|
|
2069
|
+
// on delivered without consulting the resolution — so rewriting it to
|
|
2070
|
+
// failed would claim an artifact was never written. Preserve the
|
|
2071
|
+
// delivered kind and append the flip note to stderrTail instead. The
|
|
2072
|
+
// fence is already applied above, and no retry follows delivered
|
|
2073
|
+
// anyway, so nothing about fail-closed is weakened: the next reserve
|
|
2074
|
+
// on this bucket still refuses.
|
|
2075
|
+
if (result.kind === "delivered") {
|
|
2076
|
+
return {
|
|
2077
|
+
kind: "delivered",
|
|
2078
|
+
outcome: {
|
|
2079
|
+
...result.outcome,
|
|
2080
|
+
stderrTail: [result.outcome.stderrTail, flipNote]
|
|
2081
|
+
.filter(Boolean)
|
|
2082
|
+
.join("\n"),
|
|
2083
|
+
},
|
|
2084
|
+
parsed: result.parsed,
|
|
2085
|
+
reservation: finalized.reservation,
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2088
|
+
return {
|
|
2089
|
+
kind: "failed",
|
|
2090
|
+
outcome: {
|
|
2091
|
+
...result.outcome,
|
|
2092
|
+
stderrTail: [result.outcome.stderrTail, flipNote]
|
|
2093
|
+
.filter(Boolean)
|
|
2094
|
+
.join("\n"),
|
|
2095
|
+
},
|
|
2096
|
+
resolution: { kind: "legacy_terminal" },
|
|
2097
|
+
reservation: finalized.reservation,
|
|
2098
|
+
};
|
|
2099
|
+
}
|
|
2100
|
+
return { ...result, reservation: finalized.reservation };
|
|
2101
|
+
} catch (error) {
|
|
2102
|
+
if (
|
|
2103
|
+
error instanceof ConcurrencyAdmissionAbortedError ||
|
|
2104
|
+
error instanceof BucketBreakerTrippedError
|
|
2105
|
+
) {
|
|
2106
|
+
return { kind: "cancelled" };
|
|
2107
|
+
}
|
|
2108
|
+
if (error instanceof SpendReservationFencedError) {
|
|
2109
|
+
return {
|
|
2110
|
+
kind: "failed",
|
|
2111
|
+
outcome: {
|
|
2112
|
+
completion: "failed",
|
|
2113
|
+
protocolIntegrity: "unverified",
|
|
2114
|
+
finalText: "",
|
|
2115
|
+
usage: normalizeUnavailableUsage({ wallMs: 0 }),
|
|
2116
|
+
stderrTail: error.message,
|
|
2117
|
+
},
|
|
2118
|
+
resolution: { kind: "legacy_terminal" },
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
2121
|
+
throw error;
|
|
2122
|
+
} finally {
|
|
2123
|
+
lease?.release();
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
// Coupling 1 consumer: routes a finished attempt's usage through
|
|
2128
|
+
// `settlementFromUsage` and applies whichever CAS transition it names.
|
|
2129
|
+
// `result.kind === "cancelled"` here means the provider attempt WAS
|
|
2130
|
+
// invoked (executeSession calls the transport unconditionally) but no
|
|
2131
|
+
// usage ever arrived — that case can never settle as a number either, so
|
|
2132
|
+
// it takes the same unresolved path with `knownUsd: undefined`.
|
|
2133
|
+
//
|
|
2134
|
+
// Returns the decision alongside the terminal reservation so `runAttempt`
|
|
2135
|
+
// can fail closed on `free_nonzero_cost` (#182 follow-up) — the only caller.
|
|
2136
|
+
private async finalizeReservation(
|
|
2137
|
+
reservation: SpendReservation,
|
|
2138
|
+
result: AttemptRunResult,
|
|
2139
|
+
transport?: ProviderTransport,
|
|
2140
|
+
): Promise<{
|
|
2141
|
+
readonly reservation: SpendReservation;
|
|
2142
|
+
readonly decision: SettlementDecision;
|
|
2143
|
+
}> {
|
|
2144
|
+
const ledger = this.spendLedger;
|
|
2145
|
+
// Unreachable via `runAttempt` (it returns before calling here when no
|
|
2146
|
+
// reservation was opened, which is exactly when no ledger is configured),
|
|
2147
|
+
// but kept total: report the pure decision without applying any CAS
|
|
2148
|
+
// transition, so the shape holds and no fence is claimed.
|
|
2149
|
+
if (ledger === undefined) {
|
|
2150
|
+
const usage =
|
|
2151
|
+
result.kind === "cancelled" ? undefined : result.outcome.usage;
|
|
2152
|
+
return {
|
|
2153
|
+
reservation,
|
|
2154
|
+
decision:
|
|
2155
|
+
usage === undefined
|
|
2156
|
+
? { kind: "unresolved", knownUsd: undefined }
|
|
2157
|
+
: settlementFromUsage(usage),
|
|
2158
|
+
};
|
|
2159
|
+
}
|
|
2160
|
+
const idempotencyKey = `${reservation.reservationId}:finalize`;
|
|
2161
|
+
const usage =
|
|
2162
|
+
result.kind === "cancelled" ? undefined : result.outcome.usage;
|
|
2163
|
+
if (usage === undefined) {
|
|
2164
|
+
await ledger.markUnresolvedRemote(
|
|
2165
|
+
reservation.reservationId,
|
|
2166
|
+
undefined,
|
|
2167
|
+
idempotencyKey,
|
|
2168
|
+
);
|
|
2169
|
+
return {
|
|
2170
|
+
reservation: {
|
|
2171
|
+
...reservation,
|
|
2172
|
+
state: "unresolved_remote",
|
|
2173
|
+
knownUsd: undefined,
|
|
2174
|
+
},
|
|
2175
|
+
decision: { kind: "unresolved", knownUsd: undefined },
|
|
2176
|
+
};
|
|
2177
|
+
}
|
|
2178
|
+
const decision = settlementFromUsage(usage);
|
|
2179
|
+
if (decision.kind === "settle") {
|
|
2180
|
+
await ledger.settle(reservation.reservationId, decision, idempotencyKey);
|
|
2181
|
+
return {
|
|
2182
|
+
reservation: {
|
|
2183
|
+
...reservation,
|
|
2184
|
+
state: "settled",
|
|
2185
|
+
settledUsd: decision.actualUsd,
|
|
2186
|
+
},
|
|
2187
|
+
decision,
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
// #182 follow-up, free-route fence scope: a free binding reserves so the
|
|
2191
|
+
// free-nonzero rule can fence on a priced flip — but that made EVERY
|
|
2192
|
+
// terminal free outcome fence the shared credential bucket, so one
|
|
2193
|
+
// transient failure (parse failure, watchdog timeout shape) killed the
|
|
2194
|
+
// whole all-free run. That contradicts the neighboring subscription
|
|
2195
|
+
// reasoning in production-runtime.ts MultiProviderRunner.run: on a route
|
|
2196
|
+
// that bills nothing there are no dollars to stop spending over.
|
|
2197
|
+
//
|
|
2198
|
+
// So an unresolved free outcome WITHOUT reason releases instead of
|
|
2199
|
+
// fencing: no cash figure means no spend evidence on a free route, and a
|
|
2200
|
+
// terminal released_unstarted fences nothing — the same-step retry
|
|
2201
|
+
// reserves fine and sibling steps are unaffected. Coupling-1 is
|
|
2202
|
+
// preserved: a non-complete usage still never settles as a number, it
|
|
2203
|
+
// just releases instead of fencing. Cancelled/undefined-usage keeps the
|
|
2204
|
+
// fence (no usage means no evidence either way — fail closed), the
|
|
2205
|
+
// free_nonzero_cost reason keeps the fence (priced work may be real), and
|
|
2206
|
+
// the metered path below is untouched. Downstream-safe:
|
|
2207
|
+
// collectUnresolvedSpend (pipeline.ts) and renderResult (ui/result.ts)
|
|
2208
|
+
// only read unresolved_remote reservations, so a released_unstarted free
|
|
2209
|
+
// attempt is invisible to both — no floor marker, no unresolved row.
|
|
2210
|
+
//
|
|
2211
|
+
// #187: watchdog timeout on a free route synthesizes an unavailable usage;
|
|
2212
|
+
// preserving the transport's free billing mode ensures it releases
|
|
2213
|
+
// unstarted instead of fencing the bucket, allowing attempt 2 to retry.
|
|
2214
|
+
const isFreeRoute =
|
|
2215
|
+
usage.billingMode === "free" || transport?.billingMode === "free";
|
|
2216
|
+
if (isFreeRoute && decision.reason === undefined) {
|
|
2217
|
+
await ledger.releaseUnstarted(reservation.reservationId, idempotencyKey);
|
|
2218
|
+
return {
|
|
2219
|
+
reservation: { ...reservation, state: "released_unstarted" },
|
|
2220
|
+
decision,
|
|
2221
|
+
};
|
|
2222
|
+
}
|
|
2223
|
+
await ledger.markUnresolvedRemote(
|
|
2224
|
+
reservation.reservationId,
|
|
2225
|
+
decision.knownUsd,
|
|
2226
|
+
idempotencyKey,
|
|
2227
|
+
);
|
|
2228
|
+
return {
|
|
2229
|
+
reservation: {
|
|
2230
|
+
...reservation,
|
|
2231
|
+
state: "unresolved_remote",
|
|
2232
|
+
knownUsd: decision.knownUsd,
|
|
2233
|
+
},
|
|
2234
|
+
decision,
|
|
2235
|
+
};
|
|
2236
|
+
}
|
|
2237
|
+
|
|
2238
|
+
// The body of a single attempt, unchanged from PR0 except for the §9.2
|
|
2239
|
+
// breaker-trip check — split out so `runAttempt` itself stays a thin
|
|
2240
|
+
// acquire/finally-release wrapper (D1-08 PR5a) around it.
|
|
2241
|
+
private async runAdmittedAttempt(args: {
|
|
2242
|
+
readonly step: StepSpec;
|
|
2243
|
+
readonly attempt: number;
|
|
2244
|
+
readonly kind: "attempt" | "format-retry";
|
|
2245
|
+
readonly request: TransportRequest;
|
|
2246
|
+
readonly deadlineMs: number;
|
|
2247
|
+
readonly transport: ProviderTransport;
|
|
2248
|
+
readonly harnessWatchdogMs?: number;
|
|
2249
|
+
}): Promise<AttemptRunResult> {
|
|
2250
|
+
const {
|
|
2251
|
+
step,
|
|
2252
|
+
attempt,
|
|
2253
|
+
kind,
|
|
2254
|
+
request,
|
|
2255
|
+
deadlineMs,
|
|
2256
|
+
transport,
|
|
2257
|
+
harnessWatchdogMs,
|
|
2258
|
+
} = args;
|
|
2259
|
+
|
|
2260
|
+
const frozenIdentity = await freezeAttemptEvidenceIdentity(request, step);
|
|
2261
|
+
const execution = await this.executeSession({
|
|
2262
|
+
step,
|
|
2263
|
+
request,
|
|
2264
|
+
transport,
|
|
2265
|
+
deadlineMs,
|
|
2266
|
+
harnessWatchdogMs,
|
|
2267
|
+
onData: async (outcome, settlement): Promise<AttemptDelivery> => {
|
|
2268
|
+
// A failed attempt the transport already explained with a typed cause (the
|
|
2269
|
+
// Claude CLI's error envelope) has no draft: its text is the provider's message,
|
|
2270
|
+
// not the agent's answer. Parsing it would only add a parse_error beside the
|
|
2271
|
+
// real cause in the attempt log. The cause decides the retry, exactly as it does
|
|
2272
|
+
// for a draft that failed to parse.
|
|
2273
|
+
if (
|
|
2274
|
+
outcome.completion === "failed" &&
|
|
2275
|
+
outcome.failureCause !== undefined
|
|
2276
|
+
) {
|
|
2277
|
+
const resolution = resolveFailureCause({
|
|
2278
|
+
outcome,
|
|
2279
|
+
classifyFailure: transport.classifyFailure,
|
|
2280
|
+
parseThrew: false,
|
|
2281
|
+
});
|
|
2282
|
+
const classification = legacyClassificationFromCause(resolution);
|
|
2283
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2284
|
+
writeAttemptLog(
|
|
2285
|
+
step,
|
|
2286
|
+
attempt,
|
|
2287
|
+
kind,
|
|
2288
|
+
outcome,
|
|
2289
|
+
classification,
|
|
2290
|
+
resolution.kind === "cause"
|
|
2291
|
+
? resolution.cause
|
|
2292
|
+
: "legacy_terminal",
|
|
2293
|
+
),
|
|
2294
|
+
).catch(() => {});
|
|
2295
|
+
if (classification === "transient") {
|
|
2296
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2297
|
+
Bun.file(step.outPath)
|
|
2298
|
+
.unlink()
|
|
2299
|
+
.then(() => {})
|
|
2300
|
+
.catch(() => {}),
|
|
2301
|
+
);
|
|
2302
|
+
}
|
|
2303
|
+
return { delivered: false, resolution };
|
|
2304
|
+
}
|
|
2305
|
+
// Set only when `step.parse` itself threw: the catch below also
|
|
2306
|
+
// receives artifact-write failures, which are not the parser's verdict.
|
|
2307
|
+
let parseError: string | undefined;
|
|
2308
|
+
try {
|
|
2309
|
+
let parsed: unknown;
|
|
2310
|
+
try {
|
|
2311
|
+
parsed = step.parse(outcome.finalText);
|
|
2312
|
+
} catch (parseFailure) {
|
|
2313
|
+
parseError =
|
|
2314
|
+
parseFailure instanceof Error
|
|
2315
|
+
? parseFailure.message
|
|
2316
|
+
: String(parseFailure);
|
|
2317
|
+
throw parseFailure;
|
|
2318
|
+
}
|
|
2319
|
+
if (
|
|
2320
|
+
isVacuousHunt({
|
|
2321
|
+
tools: step.tools,
|
|
2322
|
+
toolInvocations: outcome.toolInvocations,
|
|
2323
|
+
parsed,
|
|
2324
|
+
})
|
|
2325
|
+
) {
|
|
2326
|
+
// The JSON parsed, with any number of findings. format_violation
|
|
2327
|
+
// would spend a paid reminder retry on the same dump. §7 has no
|
|
2328
|
+
// cause for "did not look";
|
|
2329
|
+
// inventing one drifts the frozen vocabulary, so this is the
|
|
2330
|
+
// legacy_terminal ruling — stop, no retry. The attempt log
|
|
2331
|
+
// carries `tool_invocations: 0` as the fact.
|
|
2332
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2333
|
+
writeAttemptLog(
|
|
2334
|
+
step,
|
|
2335
|
+
attempt,
|
|
2336
|
+
kind,
|
|
2337
|
+
outcome,
|
|
2338
|
+
"terminal",
|
|
2339
|
+
"legacy_terminal",
|
|
2340
|
+
),
|
|
2341
|
+
).catch(() => {});
|
|
2342
|
+
return {
|
|
2343
|
+
delivered: false,
|
|
2344
|
+
resolution: { kind: "legacy_terminal" },
|
|
2345
|
+
};
|
|
2346
|
+
}
|
|
2347
|
+
// Artifact first, then the "ok" log: written the other way round, a
|
|
2348
|
+
// failed artifact write on a full or read-only disk left a log
|
|
2349
|
+
// reading `classification: ok` for a step that failed, whenever the
|
|
2350
|
+
// failure log that should replace it hit the same disk.
|
|
2351
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2352
|
+
writeJsonAtomically(step.outPath, parsed),
|
|
2353
|
+
);
|
|
2354
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2355
|
+
writeAttemptLog(step, attempt, kind, outcome, "ok"),
|
|
2356
|
+
);
|
|
2357
|
+
return { delivered: true, parsed };
|
|
2358
|
+
} catch (failure) {
|
|
2359
|
+
// D1-08 PR0: cause resolution now goes through the transport's OWN
|
|
2360
|
+
// failure classifier first (the second, previously unwired
|
|
2361
|
+
// mechanism this slice closes) before ever falling back to the
|
|
2362
|
+
// legacy step-runner one.
|
|
2363
|
+
//
|
|
2364
|
+
// Only a refused draft is a format violation. When the draft parsed
|
|
2365
|
+
// and writing its artifacts threw (EISDIR, ENOSPC), a reminder retry
|
|
2366
|
+
// pays for the same answer and hits the same disk, so the step stops.
|
|
2367
|
+
// Routing it through `parseThrew: false` would not help: with a
|
|
2368
|
+
// clean stderr the legacy classifier also answers "format".
|
|
2369
|
+
const resolution: CauseResolution =
|
|
2370
|
+
parseError !== undefined
|
|
2371
|
+
? resolveFailureCause({
|
|
2372
|
+
outcome,
|
|
2373
|
+
classifyFailure: transport.classifyFailure,
|
|
2374
|
+
parseThrew: true,
|
|
2375
|
+
})
|
|
2376
|
+
: { kind: "legacy_terminal" };
|
|
2377
|
+
const rejection =
|
|
2378
|
+
parseError !== undefined
|
|
2379
|
+
? { label: "parse_error" as const, message: parseError }
|
|
2380
|
+
: {
|
|
2381
|
+
label: "persist_error" as const,
|
|
2382
|
+
message:
|
|
2383
|
+
failure instanceof Error
|
|
2384
|
+
? failure.message
|
|
2385
|
+
: String(failure),
|
|
2386
|
+
};
|
|
2387
|
+
const classification = legacyClassificationFromCause(resolution);
|
|
2388
|
+
// A failing diagnostic log must never escape the failure handler
|
|
2389
|
+
// itself — that second throw is what left settlements unwritten
|
|
2390
|
+
// (pr-hero F001 on this very PR). Unified across BOTH `kind`s now
|
|
2391
|
+
// (row 9: the format-retry twin used to be missing this `.catch`).
|
|
2392
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2393
|
+
writeAttemptLog(
|
|
2394
|
+
step,
|
|
2395
|
+
attempt,
|
|
2396
|
+
kind,
|
|
2397
|
+
outcome,
|
|
2398
|
+
classification,
|
|
2399
|
+
resolution.kind === "cause"
|
|
2400
|
+
? resolution.cause
|
|
2401
|
+
: "legacy_terminal",
|
|
2402
|
+
rejection,
|
|
2403
|
+
),
|
|
2404
|
+
).catch(() => {});
|
|
2405
|
+
// Transient cleanup of the stale artifact is data-plane too, so it
|
|
2406
|
+
// runs under the same lease guard while it is still valid. Scope
|
|
2407
|
+
// widened (row 8) to both transient retry arms — retry_now AND
|
|
2408
|
+
// retry_after mean "another attempt is coming" alike.
|
|
2409
|
+
if (classification === "transient") {
|
|
2410
|
+
await this.guardedDataPlaneWrite(settlement, () =>
|
|
2411
|
+
Bun.file(step.outPath)
|
|
2412
|
+
.unlink()
|
|
2413
|
+
.then(() => {})
|
|
2414
|
+
.catch(() => {}),
|
|
2415
|
+
);
|
|
2416
|
+
}
|
|
2417
|
+
return {
|
|
2418
|
+
delivered: false,
|
|
2419
|
+
resolution,
|
|
2420
|
+
...(parseError === undefined ? {} : { parseError }),
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
},
|
|
2424
|
+
});
|
|
2425
|
+
|
|
2426
|
+
let evidenceTimer: ReturnType<typeof setTimeout> | undefined;
|
|
2427
|
+
try {
|
|
2428
|
+
await Promise.race([
|
|
2429
|
+
persistAttemptEvidence(step, request, {
|
|
2430
|
+
...execution,
|
|
2431
|
+
frozenIdentity,
|
|
2432
|
+
}).catch(() => {}),
|
|
2433
|
+
new Promise<void>((resolve) => {
|
|
2434
|
+
evidenceTimer = setTimeout(
|
|
2435
|
+
resolve,
|
|
2436
|
+
Math.min(100, Math.max(1, step.timeoutMs)),
|
|
2437
|
+
);
|
|
2438
|
+
}),
|
|
2439
|
+
]);
|
|
2440
|
+
} finally {
|
|
2441
|
+
clearTimeout(evidenceTimer);
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
// D1-08 PR5a (§9.2 "Circuit Breaker Fences An Unconfirmed-Abort
|
|
2445
|
+
// Bucket"): an SDK abort the harness could not confirm remotely stopped
|
|
2446
|
+
// fences this bucket for the rest of the run — the NEXT admission
|
|
2447
|
+
// attempt (this step's own retry, or a different step sharing the
|
|
2448
|
+
// credential) must refuse before ever reaching the transport again.
|
|
2449
|
+
if (execution.receipt.outcome === "local_fenced_remote_unconfirmed") {
|
|
2450
|
+
this.attemptAdmissionGate?.reportUnconfirmedRemote?.(
|
|
2451
|
+
this.rateLimitBucketId,
|
|
2452
|
+
);
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
if (execution.cancelled) {
|
|
2456
|
+
return { kind: "cancelled" };
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
// Both remaining branches (delivered/failed) always carry `outcome`:
|
|
2460
|
+
// `execution.cancelled` is the only path executeSession takes without
|
|
2461
|
+
// ever invoking `onData`, and `onData` is what both delivery variants
|
|
2462
|
+
// above come from.
|
|
2463
|
+
const outcome = execution.outcome as TransportOutcome;
|
|
2464
|
+
|
|
2465
|
+
if (execution.delivery?.delivered) {
|
|
2466
|
+
return { kind: "delivered", outcome, parsed: execution.delivery.parsed };
|
|
2467
|
+
}
|
|
2468
|
+
|
|
2469
|
+
const resolution =
|
|
2470
|
+
execution.delivery?.resolution ??
|
|
2471
|
+
resolveFailureCause({
|
|
2472
|
+
outcome,
|
|
2473
|
+
classifyFailure: transport.classifyFailure,
|
|
2474
|
+
parseThrew: false,
|
|
2475
|
+
});
|
|
2476
|
+
|
|
2477
|
+
// Attempt log persistence: the attempt log (logs/<step>.<attempt>.log) is
|
|
2478
|
+
// diagnostic post-mortem persistence, parallel to the settlement receipt.
|
|
2479
|
+
// When execution.delivery is undefined (e.g. watchdog timeout or an onData
|
|
2480
|
+
// throw where onData did not complete), the attempt log must still be
|
|
2481
|
+
// flushed so pipeline.json's attemptLogPath points to a valid file (#185).
|
|
2482
|
+
// This write is deliberately not lease-guarded — the write lease was
|
|
2483
|
+
// already invalidated at fence closure, and refusing diagnostic
|
|
2484
|
+
// persistence would erase the post-mortem record.
|
|
2485
|
+
if (execution.delivery === undefined) {
|
|
2486
|
+
const classification = legacyClassificationFromCause(resolution);
|
|
2487
|
+
await writeAttemptLog(
|
|
2488
|
+
step,
|
|
2489
|
+
attempt,
|
|
2490
|
+
kind,
|
|
2491
|
+
outcome,
|
|
2492
|
+
classification,
|
|
2493
|
+
resolution.kind === "cause" ? resolution.cause : "legacy_terminal",
|
|
2494
|
+
).catch(() => {});
|
|
2495
|
+
}
|
|
2496
|
+
|
|
2497
|
+
const parseError =
|
|
2498
|
+
execution.delivery?.delivered === false
|
|
2499
|
+
? execution.delivery.parseError
|
|
2500
|
+
: undefined;
|
|
2501
|
+
return {
|
|
2502
|
+
kind: "failed",
|
|
2503
|
+
outcome,
|
|
2504
|
+
resolution,
|
|
2505
|
+
...(parseError === undefined ? {} : { parseError }),
|
|
2506
|
+
};
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
|
|
2510
|
+
// TransportOutcome.completion → terminal slot status vocabulary.
|
|
2511
|
+
function completionToStatus(
|
|
2512
|
+
outcome: TransportOutcome,
|
|
2513
|
+
): "completed" | "failed" | "cancelled" {
|
|
2514
|
+
return outcome.completion === "success" ? "completed" : outcome.completion;
|
|
2515
|
+
}
|
|
2516
|
+
|
|
2517
|
+
type AttemptDelivery =
|
|
2518
|
+
| { readonly delivered: true; readonly parsed: unknown }
|
|
2519
|
+
| {
|
|
2520
|
+
readonly delivered: false;
|
|
2521
|
+
readonly resolution: CauseResolution;
|
|
2522
|
+
// The validator's message when `step.parse` refused the draft: the
|
|
2523
|
+
// format-retry prompt hands it back to the model.
|
|
2524
|
+
readonly parseError?: string;
|
|
2525
|
+
};
|
|
2526
|
+
|
|
2527
|
+
// D1-08 PR0: `runAttempt`'s result — one attempt's outcome reduced to what
|
|
2528
|
+
// the retry loop needs to decide what happens next.
|
|
2529
|
+
// D1-08 PR5b: every variant gains an optional `reservation` — the
|
|
2530
|
+
// finalized `SpendReservation` for this attempt, present exactly when a
|
|
2531
|
+
// `SpendLedger` is configured. `runAdmittedAttempt` never sets it (it knows
|
|
2532
|
+
// nothing about the ledger); only `runAttempt` attaches it once the
|
|
2533
|
+
// reservation has reached its terminal state.
|
|
2534
|
+
type AttemptRunResult =
|
|
2535
|
+
| { readonly kind: "cancelled"; readonly reservation?: SpendReservation }
|
|
2536
|
+
| {
|
|
2537
|
+
readonly kind: "delivered";
|
|
2538
|
+
readonly outcome: TransportOutcome;
|
|
2539
|
+
readonly parsed: unknown;
|
|
2540
|
+
readonly reservation?: SpendReservation;
|
|
2541
|
+
}
|
|
2542
|
+
| {
|
|
2543
|
+
readonly kind: "failed";
|
|
2544
|
+
readonly outcome: TransportOutcome;
|
|
2545
|
+
readonly resolution: CauseResolution;
|
|
2546
|
+
readonly parseError?: string;
|
|
2547
|
+
readonly reservation?: SpendReservation;
|
|
2548
|
+
};
|