@tangle-network/traces 0.15.1 → 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/{chunk-7DOBU7BJ.js → chunk-U4EB4HK7.js} +529 -2497
- package/dist/cli.js +49 -9
- package/dist/index.d.ts +76 -879
- package/dist/index.js +6 -91
- package/docs/replay-verify.md +6 -5
- package/package.json +5 -5
package/dist/index.d.ts
CHANGED
|
@@ -2,13 +2,15 @@ import { TraceValidation, SpanKind, Outcome } from '@tangle-network/agent-trace-
|
|
|
2
2
|
export { Capability, ConformanceFinding, ContractSpan, SpanLink, TraceValidation } from '@tangle-network/agent-trace-contract';
|
|
3
3
|
import { SpanSourceReader, TraceAnalysisStore, RedactionRule, RedactionReport } from '@tangle-network/agent-eval/traces';
|
|
4
4
|
export { DEFAULT_REDACTION_RULES, ErrorCluster, RedactionReport, RedactionRule, TraceAnalysisStore, redactString, redactValue } from '@tangle-network/agent-eval/traces';
|
|
5
|
-
import { ToolUseMetrics, InMemoryTraceStore, CostProvenance
|
|
5
|
+
import { ToolUseMetrics, InMemoryTraceStore, CostProvenance } from '@tangle-network/agent-eval';
|
|
6
6
|
import { StuckLoopReport, FailureClusterReport } from '@tangle-network/agent-eval/pipelines';
|
|
7
7
|
import { TraceAnalystDefinition, TraceAnalysisEngine, AnalystRegistry, AnalystFinding, buildDefaultAnalystRegistry, AnalystRunSummary, AnalystRunResult, Analyst, RawAnalystFinding, AnalystUsageReceipt, TraceAnalystLimits } from '@tangle-network/agent-eval/analyst';
|
|
8
8
|
export { Analyst, AnalystContext, AnalystFinding, AnalystRegistry, buildDefaultAnalystRegistry, makeFinding } from '@tangle-network/agent-eval/analyst';
|
|
9
9
|
import { ExecutionReport } from '@tangle-network/agent-eval/contract';
|
|
10
10
|
import { SpawnEvent, NodeStatus, Runtime, Budget, Spend } from '@tangle-network/agent-runtime/kernel';
|
|
11
11
|
import { SupervisorRunNodeRole, SupervisorRunReader, SupervisorRunSources } from '@tangle-network/agent-eval/supervisor-run';
|
|
12
|
+
import { ReplayExecBackend, VerifiableFinding, FindingReplaySource, VerifyFindingsRun, ChatCompletionCaller } from '@tangle-network/agent-eval/trajectory-replay';
|
|
13
|
+
export * from '@tangle-network/agent-eval/trajectory-replay';
|
|
12
14
|
import { TraceSpanEvent } from '@tangle-network/agent-eval/hosted';
|
|
13
15
|
export { HostedClient, createHostedClient, hostedClientFromEnv } from '@tangle-network/agent-eval/hosted';
|
|
14
16
|
|
|
@@ -1235,6 +1237,12 @@ declare function stampSpanWorkdirRepoAttrs(spans: readonly OtlpSpan[]): Promise<
|
|
|
1235
1237
|
*
|
|
1236
1238
|
* Shared by the claudish / openclaw / nanoclaw forks via aliases — they
|
|
1237
1239
|
* write the same transcript shape.
|
|
1240
|
+
*
|
|
1241
|
+
* The same reader takes the `claude -p --output-format stream-json --verbose`
|
|
1242
|
+
* output a CI job captures. Its `system`/`init` record lists the tools the
|
|
1243
|
+
* harness offered the model, which the root span keeps as the OTel
|
|
1244
|
+
* `gen_ai.tool.definitions` attribute, and its `result` record states whether
|
|
1245
|
+
* the run failed, which the root span keeps as `run.status`.
|
|
1238
1246
|
*/
|
|
1239
1247
|
|
|
1240
1248
|
declare class ClaudeAdapter implements HarnessTraceAdapter {
|
|
@@ -2420,6 +2428,52 @@ declare function writeSessionBundleManifest(outDir: string, manifest: SessionBun
|
|
|
2420
2428
|
* package wrote.
|
|
2421
2429
|
*/
|
|
2422
2430
|
declare function readSessionBundleManifest(bundleDir: string): Promise<SessionBundleManifest>;
|
|
2431
|
+
type SessionBundleIssueKind =
|
|
2432
|
+
/** Listed in the manifest, absent on disk. */
|
|
2433
|
+
'missing'
|
|
2434
|
+
/** On disk, not listed in the manifest. */
|
|
2435
|
+
| 'unexpected'
|
|
2436
|
+
/** Listed, present, and its bytes differ from the recorded size or SHA-256. */
|
|
2437
|
+
| 'mismatch'
|
|
2438
|
+
/** A manifest path that is absolute, climbs out of the bundle, or is not `/`-separated. */
|
|
2439
|
+
| 'unsafe-path'
|
|
2440
|
+
/** A symbolic link or other non-regular entry: the manifest can only vouch for regular files. */
|
|
2441
|
+
| 'not-a-file';
|
|
2442
|
+
interface SessionBundleIssue {
|
|
2443
|
+
readonly kind: SessionBundleIssueKind;
|
|
2444
|
+
readonly path: string;
|
|
2445
|
+
readonly detail: string;
|
|
2446
|
+
}
|
|
2447
|
+
interface SessionBundleVerification {
|
|
2448
|
+
/** True only when every listed file matches and nothing else is present. */
|
|
2449
|
+
readonly ok: boolean;
|
|
2450
|
+
readonly directory: string;
|
|
2451
|
+
readonly view: SessionBundleView;
|
|
2452
|
+
/** Files listed in the manifest. */
|
|
2453
|
+
readonly listed: number;
|
|
2454
|
+
/** Listed files whose size and SHA-256 matched. */
|
|
2455
|
+
readonly matched: number;
|
|
2456
|
+
readonly issues: readonly SessionBundleIssue[];
|
|
2457
|
+
/**
|
|
2458
|
+
* For a projected view: whether the full bundle it names still has the
|
|
2459
|
+
* manifest bytes the view recorded. `unavailable` when that directory is
|
|
2460
|
+
* gone, which is not a failure of this view.
|
|
2461
|
+
*/
|
|
2462
|
+
readonly source?: {
|
|
2463
|
+
readonly directory: string;
|
|
2464
|
+
readonly status: 'match' | 'mismatch' | 'unavailable';
|
|
2465
|
+
};
|
|
2466
|
+
}
|
|
2467
|
+
/**
|
|
2468
|
+
* Check a bundle directory against its own manifest: every listed file is
|
|
2469
|
+
* present with the recorded size and SHA-256, and nothing unlisted sits beside
|
|
2470
|
+
* them. The manifest is written last and never lists itself, so
|
|
2471
|
+
* `manifest.json` is the one file exempt from both checks.
|
|
2472
|
+
*
|
|
2473
|
+
* This proves the bytes are the bytes the bundler wrote. It does not prove who
|
|
2474
|
+
* wrote them or when: anyone who can rewrite a file can rewrite the manifest.
|
|
2475
|
+
*/
|
|
2476
|
+
declare function verifySessionBundle(bundleDir: string): Promise<SessionBundleVerification>;
|
|
2423
2477
|
|
|
2424
2478
|
/**
|
|
2425
2479
|
* `traces bundle-view` — project a full session bundle down to one view.
|
|
@@ -4516,66 +4570,17 @@ interface SessionBatch {
|
|
|
4516
4570
|
declare function collectSessions(opts?: CollectOptions): Promise<SessionBatch[]>;
|
|
4517
4571
|
|
|
4518
4572
|
/**
|
|
4519
|
-
*
|
|
4520
|
-
*
|
|
4521
|
-
*
|
|
4522
|
-
*
|
|
4523
|
-
*
|
|
4524
|
-
*
|
|
4525
|
-
*
|
|
4526
|
-
*
|
|
4527
|
-
*
|
|
4528
|
-
* Built on the agent-eval counterfactual scaffold: the trajectory is ingested
|
|
4529
|
-
* into a TraceStore, each arm is a `runCounterfactual` meta-run, and the
|
|
4530
|
-
* sandbox-backed `CounterfactualRunner` here supplies the `executeFrom`
|
|
4531
|
-
* callback that scaffold deliberately leaves to consumers.
|
|
4532
|
-
*
|
|
4533
|
-
* Honest limits:
|
|
4534
|
-
* - Only SWE-style trajectories carry `docker_config.base_image`; tasks whose
|
|
4535
|
-
* environment needs external compose peers cannot be replayed this way.
|
|
4536
|
-
* - Copy-on-write forks (`box.branch()`) are firecracker-only; on the docker
|
|
4537
|
-
* driver each arm replays the prefix in its own fresh sandbox instead.
|
|
4538
|
-
* - Prefix divergence (recorded vs replayed returncode) is recorded per step
|
|
4539
|
-
* and surfaced in the verdict, never hidden: a high divergence count is a
|
|
4540
|
-
* finding about replayability, not an error of the harness.
|
|
4573
|
+
* The `replay-verify`, `replay-verify-batch` and `verify-findings` commands.
|
|
4574
|
+
*
|
|
4575
|
+
* Replay verification itself lives in `@tangle-network/agent-eval/trajectory-replay`:
|
|
4576
|
+
* prefix replay, both arms, corpus enumeration, the fix loop, finding proofs and
|
|
4577
|
+
* their receipts. That package depends on no sandbox client, so this module adds
|
|
4578
|
+
* only what the CLI owns: a Tangle Sandbox execution backend, an
|
|
4579
|
+
* OpenAI-compatible chat caller for arm-B fixes, the reachability preflight, and
|
|
4580
|
+
* flag parsing.
|
|
4541
4581
|
*/
|
|
4542
4582
|
|
|
4543
|
-
|
|
4544
|
-
* (terminal submit steps carry no returncode tag). */
|
|
4545
|
-
declare function parseRecordedReturncode(observation: string | null): number | null;
|
|
4546
|
-
/** Text between the observation's <output> tags, or the raw observation when
|
|
4547
|
-
* the tags are absent. */
|
|
4548
|
-
declare function parseObservationOutput(observation: string | null): string;
|
|
4549
|
-
/**
|
|
4550
|
-
* Stable failure-signature candidate: the first line of the recorded output
|
|
4551
|
-
* that contains the word "error". Null when no such line exists — the verdict
|
|
4552
|
-
* then falls back to returncode-only matching and says so.
|
|
4553
|
-
* Pass an explicit signature to override (compiler quote glyphs vary with
|
|
4554
|
-
* locale, so a hand-picked ASCII substring is often more robust).
|
|
4555
|
-
*/
|
|
4556
|
-
declare function deriveFailureSignature(observation: string | null): string | null;
|
|
4557
|
-
interface IngestedTrajectory {
|
|
4558
|
-
runId: string;
|
|
4559
|
-
stepCount: number;
|
|
4560
|
-
}
|
|
4561
|
-
/**
|
|
4562
|
-
* Emit one tool span per trajectory step, in order, with a monotonic
|
|
4563
|
-
* injected clock so `buildTrajectory` ordering is deterministic even when
|
|
4564
|
-
* two spans would share a Date.now() millisecond.
|
|
4565
|
-
*/
|
|
4566
|
-
declare function ingestCodeTraceBenchSteps(store: TraceStore, steps: readonly CodeTraceBenchStep[], caseId: string): Promise<IngestedTrajectory>;
|
|
4567
|
-
interface ReplayExecResult {
|
|
4568
|
-
exitCode: number;
|
|
4569
|
-
stdout: string;
|
|
4570
|
-
stderr: string;
|
|
4571
|
-
}
|
|
4572
|
-
interface ReplayExecSession {
|
|
4573
|
-
exec(command: string, timeoutMs: number): Promise<ReplayExecResult>;
|
|
4574
|
-
close(): Promise<void>;
|
|
4575
|
-
}
|
|
4576
|
-
interface ReplayExecBackend {
|
|
4577
|
-
open(): Promise<ReplayExecSession>;
|
|
4578
|
-
}
|
|
4583
|
+
declare const DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
4579
4584
|
interface SandboxReplayBackendOptions {
|
|
4580
4585
|
image: string;
|
|
4581
4586
|
apiKey: string;
|
|
@@ -4592,840 +4597,32 @@ interface SandboxReplayBackendOptions {
|
|
|
4592
4597
|
/** One fresh sandbox per open(); close() deletes it. */
|
|
4593
4598
|
declare function sandboxReplayBackend(options: SandboxReplayBackendOptions): ReplayExecBackend;
|
|
4594
4599
|
/**
|
|
4595
|
-
*
|
|
4596
|
-
*
|
|
4597
|
-
* recorded actions — by piping the base64 of the action into `sh` after
|
|
4598
|
-
* cd-ing to the workdir. Exit code is sh's, i.e. the action's.
|
|
4599
|
-
*/
|
|
4600
|
-
declare function wrapActionForExec(action: string, cwd: string): string;
|
|
4601
|
-
interface PrefixDivergence {
|
|
4602
|
-
step: number;
|
|
4603
|
-
expectedReturncode: number | null;
|
|
4604
|
-
actualExit: number;
|
|
4605
|
-
}
|
|
4606
|
-
interface PrefixReplayResult {
|
|
4607
|
-
prefixExecuted: number;
|
|
4608
|
-
prefixDivergences: PrefixDivergence[];
|
|
4609
|
-
wallMs: number;
|
|
4610
|
-
}
|
|
4611
|
-
interface ArmExecutionResult {
|
|
4612
|
-
command: string;
|
|
4613
|
-
exitCode: number;
|
|
4614
|
-
stdout: string;
|
|
4615
|
-
stderr: string;
|
|
4616
|
-
wallMs: number;
|
|
4617
|
-
}
|
|
4618
|
-
interface SandboxCounterfactualRunnerOptions {
|
|
4619
|
-
cwd: string;
|
|
4620
|
-
stepTimeoutMs: number;
|
|
4621
|
-
/** Execute at most this many prefix steps (from the start). Fast-iteration
|
|
4622
|
-
* knob; a truncated prefix weakens state fidelity and the verdict says so. */
|
|
4623
|
-
prefixLimit?: number;
|
|
4624
|
-
onProgress?: (message: string) => void;
|
|
4625
|
-
}
|
|
4626
|
-
/**
|
|
4627
|
-
* Replays `ctx.prefix` in a fresh sandbox session, then executes the mutated
|
|
4628
|
-
* step. Divergences are recorded and never abort the replay. Results land on
|
|
4629
|
-
* `lastPrefix` / `lastArm` for the caller; spans for every exec land in the
|
|
4630
|
-
* counterfactual meta-run.
|
|
4631
|
-
*/
|
|
4632
|
-
declare class SandboxCounterfactualRunner implements CounterfactualRunner {
|
|
4633
|
-
private readonly backend;
|
|
4634
|
-
private readonly options;
|
|
4635
|
-
lastPrefix: PrefixReplayResult | null;
|
|
4636
|
-
lastArm: ArmExecutionResult | null;
|
|
4637
|
-
constructor(backend: ReplayExecBackend, options: SandboxCounterfactualRunnerOptions);
|
|
4638
|
-
executeFrom(ctx: CounterfactualContext, emitter: TraceEmitter): Promise<void>;
|
|
4639
|
-
}
|
|
4640
|
-
interface ReplayVerifyOptions {
|
|
4641
|
-
stepsPath: string;
|
|
4642
|
-
image: string;
|
|
4643
|
-
/** 1-based step_id of the error-critical step (matches CodeTraceBench). */
|
|
4644
|
-
at: number;
|
|
4645
|
-
/** Corrected command for arm B. Omit to run arm A only. */
|
|
4646
|
-
fixCommand?: string;
|
|
4647
|
-
cwd: string;
|
|
4648
|
-
out: string;
|
|
4649
|
-
caseId?: string;
|
|
4650
|
-
/** Override the auto-derived failure signature substring. */
|
|
4651
|
-
signature?: string;
|
|
4652
|
-
stepTimeoutMs?: number;
|
|
4653
|
-
prefixLimit?: number;
|
|
4654
|
-
/** Label of the orchestrator driver backing the sandbox (reported, not probed). */
|
|
4655
|
-
driverLabel?: string;
|
|
4656
|
-
/** Injectable for tests; defaults to sandboxReplayBackend(). */
|
|
4657
|
-
backend?: ReplayExecBackend;
|
|
4658
|
-
apiKey?: string;
|
|
4659
|
-
baseUrl?: string;
|
|
4660
|
-
maxLifetimeSeconds?: number;
|
|
4661
|
-
onProgress?: (message: string) => void;
|
|
4662
|
-
}
|
|
4663
|
-
interface ReplayArmVerdict {
|
|
4664
|
-
command: string;
|
|
4665
|
-
exitCode: number;
|
|
4666
|
-
wallMs: number;
|
|
4667
|
-
prefix: PrefixReplayResult;
|
|
4668
|
-
}
|
|
4669
|
-
interface ReplayVerdict {
|
|
4670
|
-
case: string;
|
|
4671
|
-
image: string;
|
|
4672
|
-
driver: string;
|
|
4673
|
-
k: number;
|
|
4674
|
-
cwd: string;
|
|
4675
|
-
recordedReturncode: number | null;
|
|
4676
|
-
signature: string | null;
|
|
4677
|
-
signatureBasis: 'returncode+output-substring' | 'returncode-only';
|
|
4678
|
-
prefixExecuted: number;
|
|
4679
|
-
prefixDivergences: PrefixDivergence[];
|
|
4680
|
-
armA: ReplayArmVerdict & {
|
|
4681
|
-
failureSignatureMatch: boolean;
|
|
4682
|
-
};
|
|
4683
|
-
armB: (ReplayArmVerdict & {
|
|
4684
|
-
failureVanished: boolean;
|
|
4685
|
-
}) | null;
|
|
4686
|
-
timings: {
|
|
4687
|
-
armAMs: number;
|
|
4688
|
-
armBMs: number | null;
|
|
4689
|
-
totalMs: number;
|
|
4690
|
-
};
|
|
4691
|
-
runIds: {
|
|
4692
|
-
original: string;
|
|
4693
|
-
armA: string;
|
|
4694
|
-
armB: string | null;
|
|
4695
|
-
};
|
|
4696
|
-
}
|
|
4697
|
-
declare function replayVerify(options: ReplayVerifyOptions): Promise<ReplayVerdict>;
|
|
4698
|
-
interface ReplayVerifyCliArgs {
|
|
4699
|
-
stepsPath: string;
|
|
4700
|
-
image: string;
|
|
4701
|
-
at: number;
|
|
4702
|
-
fixCommand?: string;
|
|
4703
|
-
cwd: string;
|
|
4704
|
-
out: string;
|
|
4705
|
-
caseId?: string;
|
|
4706
|
-
signature?: string;
|
|
4707
|
-
stepTimeoutMs?: number;
|
|
4708
|
-
prefixLimit?: number;
|
|
4709
|
-
driverLabel?: string;
|
|
4710
|
-
baseUrl: string;
|
|
4711
|
-
apiKeyEnv: string;
|
|
4712
|
-
maxLifetimeSeconds?: number;
|
|
4713
|
-
}
|
|
4714
|
-
declare function parseReplayVerifyArgs(argv: readonly string[]): ReplayVerifyCliArgs | 'help';
|
|
4715
|
-
declare function replayVerifyUsage(): string;
|
|
4716
|
-
declare function cmdReplayVerify(argv: readonly string[]): Promise<void>;
|
|
4717
|
-
|
|
4718
|
-
/**
|
|
4719
|
-
* Corpus enumeration for batch replay-verification.
|
|
4720
|
-
*
|
|
4721
|
-
* A CodeTraceBench corpus is a gold label file (array of trajectory entries
|
|
4722
|
-
* with `incorrect_stages[].incorrect_step_ids`) plus a prepared directory:
|
|
4723
|
-
* <prepared>/normalized/<traj_id>/steps.json CodeTracer-normalized steps
|
|
4724
|
-
* <prepared>/normalized/<traj_id>/task.md optional task statement
|
|
4725
|
-
* <prepared>/extracted/<traj_id>/swe_raw/** raw mini-SWE trajectory
|
|
4726
|
-
*
|
|
4727
|
-
* Only SWE-style trajectories are replayable: the raw trajectory must carry
|
|
4728
|
-
* `info.docker_config.base_image`. Every excluded case is returned with a
|
|
4729
|
-
* machine-readable reason — the batch report surfaces the full exclusion
|
|
4730
|
-
* table, never a silently shrunk denominator.
|
|
4731
|
-
*/
|
|
4732
|
-
|
|
4733
|
-
interface CorpusSpec {
|
|
4734
|
-
readonly name: string;
|
|
4735
|
-
readonly labelsPath: string;
|
|
4736
|
-
readonly preparedDir: string;
|
|
4737
|
-
}
|
|
4738
|
-
/** Parses `name=<labelsPath>::<preparedDir>` (paths may contain `=`, not `::`). */
|
|
4739
|
-
declare function parseCorpusFlag(value: string): CorpusSpec;
|
|
4740
|
-
type ReplayExclusionReason = 'no-swe-raw-trajectory' | 'ambiguous-swe-raw-trajectory' | 'unreadable-raw-trajectory' | 'no-docker-image' | 'no-gold-incorrect-step' | 'gold-only-submit-step' | 'missing-steps-json' | 'gold-step-outside-steps' | 'cwd-underivable';
|
|
4741
|
-
/** mini-SWE's end-of-run submit convention: the agent echoes this sentinel
|
|
4742
|
-
* and dumps the diff. A gold label on this step marks a bad SUBMIT DECISION,
|
|
4743
|
-
* not a failed command — there is no executable failure to reproduce, so it
|
|
4744
|
-
* is never a counterfactual replay target (split3's gold labels concentrate
|
|
4745
|
-
* here, which is why a constant accuse-the-submit rule scores well there). */
|
|
4746
|
-
declare const SUBMIT_ACTION_SIGNATURE = "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT";
|
|
4747
|
-
declare function isSubmitAction(action: string): boolean;
|
|
4748
|
-
interface ExcludedCase {
|
|
4749
|
-
readonly corpus: string;
|
|
4750
|
-
readonly trajId: string;
|
|
4751
|
-
readonly reason: ReplayExclusionReason;
|
|
4752
|
-
readonly detail?: string;
|
|
4753
|
-
}
|
|
4754
|
-
type CwdSource = 'run-config' | 'docker-config' | 'pwd-observation';
|
|
4755
|
-
/** Everything needed to invoke replay-verify for one trajectory. */
|
|
4756
|
-
interface CaseResources {
|
|
4757
|
-
readonly corpus: string;
|
|
4758
|
-
readonly trajId: string;
|
|
4759
|
-
readonly stepsPath: string;
|
|
4760
|
-
readonly steps: readonly CodeTraceBenchStep[];
|
|
4761
|
-
readonly taskStatement: string | null;
|
|
4762
|
-
readonly image: string;
|
|
4763
|
-
readonly cwd: string;
|
|
4764
|
-
readonly cwdSource: CwdSource;
|
|
4765
|
-
/** Per-step timeout the recorded run used, when the raw config carries one. */
|
|
4766
|
-
readonly recordedStepTimeoutMs: number | null;
|
|
4767
|
-
}
|
|
4768
|
-
interface ReplayableCase extends CaseResources {
|
|
4769
|
-
/** 1-based gold incorrect step ids, ascending. */
|
|
4770
|
-
readonly goldIncorrectSteps: readonly number[];
|
|
4771
|
-
/** k — the first gold incorrect step that is a real mid-trajectory action;
|
|
4772
|
-
* submit-command golds are skipped (see SUBMIT_ACTION_SIGNATURE). */
|
|
4773
|
-
readonly k: number;
|
|
4774
|
-
/** Gold steps before k skipped because their action is the submit command. */
|
|
4775
|
-
readonly submitGoldsSkipped: number;
|
|
4776
|
-
/** Recorded returncode at k; null when the observation carries none. */
|
|
4777
|
-
readonly recordedReturncodeAtK: number | null;
|
|
4778
|
-
}
|
|
4779
|
-
interface LabelEntry {
|
|
4780
|
-
readonly traj_id: string;
|
|
4781
|
-
readonly incorrect_stages?: readonly {
|
|
4782
|
-
readonly incorrect_step_ids?: readonly number[];
|
|
4783
|
-
}[];
|
|
4784
|
-
}
|
|
4785
|
-
declare function readLabelEntries(labelsPath: string): LabelEntry[];
|
|
4786
|
-
declare function goldIncorrectSteps(entry: LabelEntry): number[];
|
|
4787
|
-
type ResourceResolution = {
|
|
4788
|
-
readonly resolved: true;
|
|
4789
|
-
readonly resources: CaseResources;
|
|
4790
|
-
} | {
|
|
4791
|
-
readonly resolved: false;
|
|
4792
|
-
readonly reason: ReplayExclusionReason;
|
|
4793
|
-
readonly detail?: string;
|
|
4794
|
-
};
|
|
4795
|
-
/**
|
|
4796
|
-
* Resolves the replay resources for one trajectory, independent of gold
|
|
4797
|
-
* labels — the analyst wire uses this with a finding-supplied step instead.
|
|
4798
|
-
*/
|
|
4799
|
-
declare function resolveCaseResources(corpus: CorpusSpec, trajId: string): ResourceResolution;
|
|
4800
|
-
interface EnumerationResult {
|
|
4801
|
-
readonly replayable: ReplayableCase[];
|
|
4802
|
-
readonly excluded: ExcludedCase[];
|
|
4803
|
-
readonly labelEntryCount: number;
|
|
4804
|
-
}
|
|
4805
|
-
/**
|
|
4806
|
-
* Replayable = SWE-style raw trajectory with a docker image AND at least one
|
|
4807
|
-
* gold incorrect step that is a real mid-trajectory action. Gold steps whose
|
|
4808
|
-
* action is the submit command are skipped when choosing k (counted per case);
|
|
4809
|
-
* a case whose golds are ALL submit steps is excluded as gold-only-submit-step.
|
|
4810
|
-
* Exclusion reasons are reported in resolution order:
|
|
4811
|
-
* raw trajectory → docker image → gold labels → steps.json → k range → cwd.
|
|
4812
|
-
*/
|
|
4813
|
-
declare function enumerateReplayableCases(corpora: readonly CorpusSpec[]): EnumerationResult;
|
|
4814
|
-
|
|
4815
|
-
/**
|
|
4816
|
-
* Counterfactual patch synthesis: one LLM call per case that turns a gold
|
|
4817
|
-
* incorrect step into a corrected shell command for replay-verify arm B.
|
|
4818
|
-
*
|
|
4819
|
-
* The caller is a typed outcome boundary ({ succeeded, value, error }) so a
|
|
4820
|
-
* provider failure is a per-case report row, never a thrown batch abort and
|
|
4821
|
-
* never a silent empty fix.
|
|
4600
|
+
* Fails loud when the sandbox SDK adapter is not answering. Executed findings
|
|
4601
|
+
* require real infrastructure; verification is never silently skipped.
|
|
4822
4602
|
*/
|
|
4823
|
-
|
|
4824
|
-
interface ChatUsage {
|
|
4825
|
-
readonly promptTokens: number;
|
|
4826
|
-
readonly completionTokens: number;
|
|
4827
|
-
}
|
|
4828
|
-
type ChatOutcome = {
|
|
4829
|
-
readonly succeeded: true;
|
|
4830
|
-
readonly value: {
|
|
4831
|
-
content: string;
|
|
4832
|
-
usage: ChatUsage | null;
|
|
4833
|
-
};
|
|
4834
|
-
} | {
|
|
4835
|
-
readonly succeeded: false;
|
|
4836
|
-
readonly error: string;
|
|
4837
|
-
};
|
|
4838
|
-
interface ChatCompletionCaller {
|
|
4839
|
-
complete(system: string, user: string): Promise<ChatOutcome>;
|
|
4840
|
-
}
|
|
4603
|
+
declare function assertSandboxReachable(baseUrl: string): Promise<void>;
|
|
4841
4604
|
interface ZaiChatCallerOptions {
|
|
4842
4605
|
readonly baseUrl: string;
|
|
4843
4606
|
readonly apiKey: string;
|
|
4844
4607
|
readonly model: string;
|
|
4845
4608
|
readonly maxTokens?: number;
|
|
4846
4609
|
readonly timeoutMs?: number;
|
|
4847
|
-
readonly fetchImpl?: typeof fetch;
|
|
4848
4610
|
}
|
|
4849
4611
|
/** OpenAI-compatible /chat/completions caller (z.ai coding endpoint shape). */
|
|
4850
4612
|
declare function zaiChatCaller(options: ZaiChatCallerOptions): ChatCompletionCaller;
|
|
4851
|
-
|
|
4852
|
-
|
|
4853
|
-
readonly steps: readonly CodeTraceBenchStep[];
|
|
4854
|
-
/** 1-based step_id of the gold incorrect step. */
|
|
4855
|
-
readonly k: number;
|
|
4856
|
-
/** Steps of context on each side of k (default 3). */
|
|
4857
|
-
readonly contextRadius?: number;
|
|
4858
|
-
}
|
|
4859
|
-
/** Head+tail excerpt with an elision marker; identity below the limit. */
|
|
4860
|
-
declare function clipText(text: string, limit: number): string;
|
|
4861
|
-
declare function buildFixPrompt(input: FixPromptInput): {
|
|
4862
|
-
system: string;
|
|
4863
|
-
user: string;
|
|
4864
|
-
};
|
|
4865
|
-
/** One prior attempt of the fix loop, rendered into the retry prompt. */
|
|
4866
|
-
interface FailedFixAttempt {
|
|
4867
|
-
readonly attempt: number;
|
|
4868
|
-
/** Null when the model call itself failed before producing a command. */
|
|
4869
|
-
readonly command: string | null;
|
|
4870
|
-
readonly exitCode: number | null;
|
|
4871
|
-
readonly stdoutTail: string | null;
|
|
4872
|
-
readonly stderrTail: string | null;
|
|
4873
|
-
readonly llmError: string | null;
|
|
4874
|
-
}
|
|
4875
|
-
/**
|
|
4876
|
-
* Retry prompt for fix-loop attempts ≥2: the original context plus every prior
|
|
4877
|
-
* attempt with its REAL executed output, and permission to answer with a short
|
|
4878
|
-
* script (the block still executes as one /bin/sh unit).
|
|
4879
|
-
*/
|
|
4880
|
-
declare function buildRetryFixPrompt(input: FixPromptInput, priorAttempts: readonly FailedFixAttempt[], maxScriptCommands?: number): {
|
|
4881
|
-
system: string;
|
|
4882
|
-
user: string;
|
|
4883
|
-
};
|
|
4884
|
-
/** Non-empty, non-comment lines of a fix script — the loop's script-size cap. */
|
|
4885
|
-
declare function countScriptCommands(script: string): number;
|
|
4886
|
-
/** Last fenced code block, else the whole trimmed content; null when empty. */
|
|
4887
|
-
declare function extractFixCommand(content: string): string | null;
|
|
4888
|
-
type FixGenerationOutcome = {
|
|
4889
|
-
readonly succeeded: true;
|
|
4890
|
-
readonly value: {
|
|
4891
|
-
command: string;
|
|
4892
|
-
usage: ChatUsage | null;
|
|
4893
|
-
};
|
|
4894
|
-
} | {
|
|
4895
|
-
readonly succeeded: false;
|
|
4896
|
-
readonly error: string;
|
|
4897
|
-
};
|
|
4898
|
-
declare function generateFixCommand(caller: ChatCompletionCaller, input: FixPromptInput): Promise<FixGenerationOutcome>;
|
|
4899
|
-
|
|
4900
|
-
/**
|
|
4901
|
-
* Iterative counterfactual fix loop.
|
|
4902
|
-
*
|
|
4903
|
-
* Attempt 1 is exactly the one-shot generator (same prompt, same caller), so
|
|
4904
|
-
* flip@1 stays comparable to `--fix generate`. When an attempt does not flip
|
|
4905
|
-
* the failure — nonzero exit, signature still present, or the model call
|
|
4906
|
-
* itself failed — the next attempt's prompt carries every prior command and
|
|
4907
|
-
* its REAL executed stdout/stderr, up to a fixed attempt budget.
|
|
4908
|
-
*
|
|
4909
|
-
* Isolation invariant: every attempt executes through the injected executor,
|
|
4910
|
-
* which must provide a FRESH sandbox with the same replayed prefix. A used
|
|
4911
|
-
* sandbox is never mutated mid-arm; a flip therefore always proves the
|
|
4912
|
-
* corrected step against the recorded prefix state, not against debris from
|
|
4913
|
-
* an earlier attempt.
|
|
4914
|
-
*/
|
|
4915
|
-
|
|
4916
|
-
/** Result of executing one corrected command as a full arm. */
|
|
4917
|
-
interface FixArmExecution {
|
|
4918
|
-
readonly exitCode: number;
|
|
4919
|
-
readonly prefixExecuted: number;
|
|
4920
|
-
readonly prefixDivergences: number;
|
|
4921
|
-
readonly failureVanished: boolean;
|
|
4922
|
-
readonly stdout: string;
|
|
4923
|
-
readonly stderr: string;
|
|
4924
|
-
}
|
|
4925
|
-
/** Runs one corrected command as a full arm: fresh sandbox + prefix replay. */
|
|
4926
|
-
type FixArmExecutor = (command: string, attempt: number) => Promise<FixArmExecution>;
|
|
4927
|
-
interface FixLoopOptions {
|
|
4928
|
-
/** Total LLM attempts per case (≥1); 1 degenerates to the one-shot path. */
|
|
4929
|
-
readonly maxAttempts: number;
|
|
4930
|
-
/** Command-line cap on retry scripts (default 5). */
|
|
4931
|
-
readonly maxScriptCommands?: number;
|
|
4932
|
-
/** Chars kept per stdout/stderr tail in records and retry prompts. */
|
|
4933
|
-
readonly outputTailChars?: number;
|
|
4934
|
-
readonly onProgress?: (message: string) => void;
|
|
4935
|
-
}
|
|
4936
|
-
interface FixLoopAttemptRecord {
|
|
4937
|
-
readonly attempt: number;
|
|
4938
|
-
readonly command: string | null;
|
|
4939
|
-
readonly llmError: string | null;
|
|
4940
|
-
readonly usage: {
|
|
4941
|
-
readonly promptTokens: number;
|
|
4942
|
-
readonly completionTokens: number;
|
|
4943
|
-
} | null;
|
|
4944
|
-
readonly executed: boolean;
|
|
4945
|
-
readonly exitCode: number | null;
|
|
4946
|
-
readonly prefixExecuted: number | null;
|
|
4947
|
-
readonly prefixDivergences: number | null;
|
|
4948
|
-
readonly failureVanished: boolean | null;
|
|
4949
|
-
readonly stdoutTail: string | null;
|
|
4950
|
-
readonly stderrTail: string | null;
|
|
4951
|
-
readonly armBError: string | null;
|
|
4952
|
-
readonly wallMs: number;
|
|
4953
|
-
}
|
|
4954
|
-
interface FixLoopResult {
|
|
4955
|
-
readonly flipped: boolean;
|
|
4956
|
-
readonly flippedAtAttempt: number | null;
|
|
4957
|
-
readonly attempts: readonly FixLoopAttemptRecord[];
|
|
4958
|
-
/** True when a sandbox error ended the loop before the attempt budget. */
|
|
4959
|
-
readonly aborted: boolean;
|
|
4960
|
-
readonly llmCalls: number;
|
|
4961
|
-
/** Calls that produced no runnable fix: transport errors, empty replies,
|
|
4962
|
-
* and retry scripts over the command cap. */
|
|
4963
|
-
readonly llmFailures: number;
|
|
4964
|
-
readonly promptTokens: number;
|
|
4965
|
-
readonly completionTokens: number;
|
|
4966
|
-
}
|
|
4967
|
-
declare function runFixLoop(caller: ChatCompletionCaller, input: FixPromptInput, executor: FixArmExecutor, options: FixLoopOptions): Promise<FixLoopResult>;
|
|
4968
|
-
|
|
4969
|
-
/**
|
|
4970
|
-
* Batch replay-verification across gold-labeled CodeTraceBench corpora.
|
|
4971
|
-
*
|
|
4972
|
-
* For every replayable case (SWE-style raw trajectory with a docker image AND
|
|
4973
|
-
* at least one gold incorrect step) the batch:
|
|
4974
|
-
* 1. derives the uid-1000 replay image (the sandbox platform pins customer
|
|
4975
|
-
* commands to a non-root identity, so root-owned trees must be chowned),
|
|
4976
|
-
* 2. replays the prefix and runs arm A — the recorded gold step k, and
|
|
4977
|
-
* 3. optionally generates a corrected command with one LLM call and runs
|
|
4978
|
-
* arm B in its own fresh sandbox.
|
|
4979
|
-
*
|
|
4980
|
-
* Headline metrics:
|
|
4981
|
-
* replayability rate — fraction of replayable cases where the prefix
|
|
4982
|
-
* replays with ≤10% returncode divergence AND arm A reproduces the
|
|
4983
|
-
* recorded returncode at k;
|
|
4984
|
-
* fix-flip rate — fraction of arm-B-executed cases where the failure
|
|
4985
|
-
* vanished (exit 0, signature absent).
|
|
4986
|
-
*
|
|
4987
|
-
* Image pulls and sandbox execs run strictly serially: pulls contend on disk
|
|
4988
|
-
* and registry bandwidth, and serial cases keep wall-time attribution per
|
|
4989
|
-
* case honest. Pull failures are report rows, never silent skips.
|
|
4990
|
-
*/
|
|
4991
|
-
|
|
4992
|
-
type ImagePreparation = {
|
|
4993
|
-
readonly succeeded: true;
|
|
4994
|
-
readonly value: {
|
|
4995
|
-
derivedImage: string;
|
|
4996
|
-
pulled: boolean;
|
|
4997
|
-
built: boolean;
|
|
4998
|
-
};
|
|
4999
|
-
} | {
|
|
5000
|
-
readonly succeeded: false;
|
|
5001
|
-
readonly error: string;
|
|
5002
|
-
};
|
|
5003
|
-
interface ImagePreparer {
|
|
5004
|
-
ensure(image: string, cwd: string): Promise<ImagePreparation>;
|
|
5005
|
-
}
|
|
5006
|
-
interface DockerImagePreparerOptions {
|
|
5007
|
-
readonly pullTimeoutMs?: number;
|
|
5008
|
-
readonly buildTimeoutMs?: number;
|
|
5009
|
-
}
|
|
5010
|
-
declare function derivedImageTag(image: string, cwd: string): string;
|
|
5011
|
-
/**
|
|
5012
|
-
* Pulls the base image when absent and builds `FROM <base>; RUN chown -R
|
|
5013
|
-
* 1000:1000 <cwd>` tagged by content hash, so repeated batches reuse both
|
|
5014
|
-
* the pull and the build. cwd `/` skips the chown (never chown -R /) and
|
|
5015
|
-
* replays on the base image directly.
|
|
5016
|
-
*/
|
|
5017
|
-
declare function dockerImagePreparer(options?: DockerImagePreparerOptions): ImagePreparer;
|
|
5018
|
-
interface ReplayBatchOptions {
|
|
5019
|
-
readonly corpora: readonly CorpusSpec[];
|
|
5020
|
-
readonly out: string;
|
|
5021
|
-
readonly baseUrl?: string;
|
|
5022
|
-
readonly apiKey?: string;
|
|
5023
|
-
/** 'generate' = one LLM call per arm-A-reproduced case, then arm B.
|
|
5024
|
-
* 'loop' = iterative: failed arms feed their real output into up to
|
|
5025
|
-
* --fix-attempts prompts, each executed in its own fresh sandbox. */
|
|
5026
|
-
readonly fix: 'none' | 'generate' | 'loop';
|
|
5027
|
-
/** Attempt budget per case in loop mode (default 3). */
|
|
5028
|
-
readonly fixAttempts?: number;
|
|
5029
|
-
readonly fixCaller?: ChatCompletionCaller;
|
|
5030
|
-
readonly fixModelLabel?: string;
|
|
5031
|
-
/** Cap on LLM fix calls; eligible cases beyond it are seeded-sampled out. */
|
|
5032
|
-
readonly maxFixCases?: number;
|
|
5033
|
-
readonly seed?: number;
|
|
5034
|
-
/** Overrides the per-case recorded step timeout. */
|
|
5035
|
-
readonly stepTimeoutMs?: number;
|
|
5036
|
-
readonly prefixLimit?: number;
|
|
5037
|
-
/** Run only cases whose trajId contains this substring (smoke knob). */
|
|
5038
|
-
readonly caseFilter?: string;
|
|
5039
|
-
/** Run only the first N replayable cases (smoke knob). */
|
|
5040
|
-
readonly caseLimit?: number;
|
|
5041
|
-
readonly maxLifetimeSeconds?: number;
|
|
5042
|
-
readonly preparer?: ImagePreparer;
|
|
5043
|
-
/** Test seam: receives the derived image, returns the exec backend. */
|
|
5044
|
-
readonly backendFactory?: (derivedImage: string) => ReplayExecBackend;
|
|
5045
|
-
readonly onProgress?: (message: string) => void;
|
|
5046
|
-
}
|
|
5047
|
-
interface ReplayBatchFixResult {
|
|
5048
|
-
/** false when the case was eligible but seeded-sampled out of the cap. */
|
|
5049
|
-
readonly attempted: boolean;
|
|
5050
|
-
readonly sampledOut: boolean;
|
|
5051
|
-
readonly command: string | null;
|
|
5052
|
-
readonly llmError: string | null;
|
|
5053
|
-
/** Loop mode: token totals summed across every attempt (null when the
|
|
5054
|
-
* provider reported no usage). */
|
|
5055
|
-
readonly usage: ChatUsage | null;
|
|
5056
|
-
readonly armBExit: number | null;
|
|
5057
|
-
readonly armBPrefixExecuted: number | null;
|
|
5058
|
-
readonly armBPrefixDivergences: number | null;
|
|
5059
|
-
readonly failureVanished: boolean | null;
|
|
5060
|
-
readonly armBError: string | null;
|
|
5061
|
-
/** Loop mode only: the full per-attempt trail. Null in generate mode. */
|
|
5062
|
-
readonly attempts: readonly FixLoopAttemptRecord[] | null;
|
|
5063
|
-
/** 1-based attempt that flipped the failure; null when none did.
|
|
5064
|
-
* Generate mode: 1 when the single attempt flipped. */
|
|
5065
|
-
readonly flippedAtAttempt: number | null;
|
|
5066
|
-
}
|
|
5067
|
-
interface ReplayBatchCaseRow {
|
|
5068
|
-
readonly corpus: string;
|
|
5069
|
-
readonly trajId: string;
|
|
5070
|
-
readonly image: string;
|
|
5071
|
-
readonly derivedImage: string | null;
|
|
5072
|
-
readonly cwd: string;
|
|
5073
|
-
readonly cwdSource: string;
|
|
5074
|
-
readonly k: number;
|
|
5075
|
-
readonly stepCount: number;
|
|
5076
|
-
readonly goldIncorrectSteps: readonly number[];
|
|
5077
|
-
/** Gold steps before k skipped because their action is the submit command. */
|
|
5078
|
-
readonly submitGoldsSkipped: number;
|
|
5079
|
-
readonly recordedReturncodeAtK: number | null;
|
|
5080
|
-
readonly signature: string | null;
|
|
5081
|
-
readonly status: 'ok' | 'image-unavailable' | 'replay-error';
|
|
5082
|
-
readonly error: string | null;
|
|
5083
|
-
readonly imagePulled: boolean;
|
|
5084
|
-
readonly imageBuilt: boolean;
|
|
5085
|
-
readonly prefixExecuted: number | null;
|
|
5086
|
-
readonly prefixDivergences: number | null;
|
|
5087
|
-
readonly prefixDivergencePct: number | null;
|
|
5088
|
-
readonly armAExit: number | null;
|
|
5089
|
-
readonly armAReturncodeMatch: boolean;
|
|
5090
|
-
readonly armASignatureMatch: boolean;
|
|
5091
|
-
/** Headline predicate: divergence ≤10% AND arm A reproduced the returncode. */
|
|
5092
|
-
readonly replayed: boolean;
|
|
5093
|
-
readonly fix: ReplayBatchFixResult | null;
|
|
5094
|
-
readonly wallMs: number;
|
|
5095
|
-
}
|
|
5096
|
-
interface ReplayBatchReport {
|
|
5097
|
-
readonly generatedAt: string;
|
|
5098
|
-
readonly corpora: readonly {
|
|
5099
|
-
name: string;
|
|
5100
|
-
labelsPath: string;
|
|
5101
|
-
preparedDir: string;
|
|
5102
|
-
}[];
|
|
5103
|
-
readonly totals: {
|
|
5104
|
-
readonly labelEntries: number;
|
|
5105
|
-
readonly replayable: number;
|
|
5106
|
-
readonly executed: number;
|
|
5107
|
-
readonly excludedByReason: Record<string, number>;
|
|
5108
|
-
/** Per-corpus submit-gold accounting: cases dropped because every gold is
|
|
5109
|
-
* the submit command, and golds skipped inside still-replayable cases. */
|
|
5110
|
-
readonly submitGoldsByCorpus: Record<string, {
|
|
5111
|
-
submitOnlyCases: number;
|
|
5112
|
-
goldsSkippedWithinReplayable: number;
|
|
5113
|
-
}>;
|
|
5114
|
-
};
|
|
5115
|
-
readonly headline: {
|
|
5116
|
-
readonly replayabilityRate: {
|
|
5117
|
-
numerator: number;
|
|
5118
|
-
denominator: number;
|
|
5119
|
-
value: number | null;
|
|
5120
|
-
};
|
|
5121
|
-
readonly signatureStrictRate: {
|
|
5122
|
-
numerator: number;
|
|
5123
|
-
denominator: number;
|
|
5124
|
-
value: number | null;
|
|
5125
|
-
};
|
|
5126
|
-
readonly fixFlipRate: {
|
|
5127
|
-
numerator: number;
|
|
5128
|
-
denominator: number;
|
|
5129
|
-
value: number | null;
|
|
5130
|
-
} | null;
|
|
5131
|
-
/** Fix-flip restricted to cases whose recorded returncode at k is nonzero —
|
|
5132
|
-
* real recorded failures, where "the failure vanished" is not vacuous. */
|
|
5133
|
-
readonly fixFlipRateNonzeroRc: {
|
|
5134
|
-
numerator: number;
|
|
5135
|
-
denominator: number;
|
|
5136
|
-
value: number | null;
|
|
5137
|
-
} | null;
|
|
5138
|
-
/** Loop mode only: flips at attempt 1 over cases whose attempt 1 executed —
|
|
5139
|
-
* the number directly comparable to the one-shot fixFlipRate. */
|
|
5140
|
-
readonly fixFlipAttempt1: {
|
|
5141
|
-
numerator: number;
|
|
5142
|
-
denominator: number;
|
|
5143
|
-
value: number | null;
|
|
5144
|
-
} | null;
|
|
5145
|
-
/** Loop mode only: flip count keyed by the attempt number that flipped. */
|
|
5146
|
-
readonly flipsByAttempt: Record<string, number> | null;
|
|
5147
|
-
};
|
|
5148
|
-
readonly llm: {
|
|
5149
|
-
readonly model: string;
|
|
5150
|
-
readonly calls: number;
|
|
5151
|
-
readonly failures: number;
|
|
5152
|
-
readonly promptTokens: number;
|
|
5153
|
-
readonly completionTokens: number;
|
|
5154
|
-
} | null;
|
|
5155
|
-
readonly excluded: readonly {
|
|
5156
|
-
corpus: string;
|
|
5157
|
-
trajId: string;
|
|
5158
|
-
reason: string;
|
|
5159
|
-
detail?: string;
|
|
5160
|
-
}[];
|
|
5161
|
-
readonly pullFailures: readonly {
|
|
5162
|
-
corpus: string;
|
|
5163
|
-
trajId: string;
|
|
5164
|
-
image: string;
|
|
5165
|
-
error: string;
|
|
5166
|
-
}[];
|
|
5167
|
-
readonly cases: readonly ReplayBatchCaseRow[];
|
|
5168
|
-
}
|
|
5169
|
-
declare function seededSample<T>(items: readonly T[], size: number, seed: number): Set<T>;
|
|
5170
|
-
declare function runReplayBatch(options: ReplayBatchOptions): Promise<ReplayBatchReport>;
|
|
5171
|
-
declare function renderBatchReport(report: ReplayBatchReport): string;
|
|
5172
|
-
interface ReplayBatchCliArgs {
|
|
5173
|
-
readonly corpora: CorpusSpec[];
|
|
5174
|
-
readonly out: string;
|
|
5175
|
-
readonly fix: 'none' | 'generate' | 'loop';
|
|
5176
|
-
readonly fixAttempts: number;
|
|
5177
|
-
readonly fixModel: string;
|
|
5178
|
-
readonly fixBaseUrl: string;
|
|
5179
|
-
readonly fixApiKeyEnv: string;
|
|
5180
|
-
readonly maxFixCases: number;
|
|
5181
|
-
readonly seed: number;
|
|
5182
|
-
readonly stepTimeoutMs?: number;
|
|
5183
|
-
readonly prefixLimit?: number;
|
|
5184
|
-
readonly caseFilter?: string;
|
|
5185
|
-
readonly caseLimit?: number;
|
|
5186
|
-
readonly baseUrl: string;
|
|
5187
|
-
readonly apiKeyEnv: string;
|
|
5188
|
-
readonly maxLifetimeSeconds?: number;
|
|
5189
|
-
readonly enumerateOnly: boolean;
|
|
5190
|
-
}
|
|
5191
|
-
declare function parseReplayBatchArgs(argv: readonly string[]): ReplayBatchCliArgs | 'help';
|
|
5192
|
-
declare function replayBatchUsage(): string;
|
|
5193
|
-
declare function cmdReplayVerifyBatch(argv: readonly string[]): Promise<void>;
|
|
5194
|
-
|
|
5195
|
-
/**
|
|
5196
|
-
* Finding-to-replay wire: the entry point the analyst product calls to turn
|
|
5197
|
-
* a cited incorrect-steps finding into an executed replay-verify proof.
|
|
5198
|
-
*
|
|
5199
|
-
* A CodeTraceBench analyst finding names its trajectory and carries a subject
|
|
5200
|
-
* of the form `incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`.
|
|
5201
|
-
* The wire maps that to a replay-verify invocation: `--at` = the finding's
|
|
5202
|
-
* first incorrect step (which may differ from the gold label), the docker
|
|
5203
|
-
* image and cwd from the trajectory's raw config, and — optionally — an arm-B
|
|
5204
|
-
* corrected command from the counterfactual fix generator.
|
|
5205
|
-
*/
|
|
5206
|
-
|
|
5207
|
-
interface IncorrectStepsSubject {
|
|
5208
|
-
readonly firstStep: number;
|
|
5209
|
-
readonly lastStep: number;
|
|
5210
|
-
readonly escapeStatus: 'escaped' | 'unescaped';
|
|
5211
|
-
readonly consequenceStep: number;
|
|
5212
|
-
}
|
|
5213
|
-
/** Null when the subject is not an incorrect-steps finding subject. */
|
|
5214
|
-
declare function parseIncorrectStepsSubject(subject: string): IncorrectStepsSubject | null;
|
|
5215
|
-
interface AnalystReplayFinding {
|
|
5216
|
-
readonly trajId: string;
|
|
5217
|
-
readonly subject: string;
|
|
5218
|
-
}
|
|
5219
|
-
interface ResolvedReplayInvocation {
|
|
5220
|
-
readonly resources: CaseResources;
|
|
5221
|
-
readonly subject: IncorrectStepsSubject;
|
|
5222
|
-
/** 1-based step_id replay-verify receives as --at: the finding's first incorrect step. */
|
|
5223
|
-
readonly at: number;
|
|
5224
|
-
}
|
|
5225
|
-
/**
|
|
5226
|
-
* Maps a finding onto replay resources, searching the given corpora for the
|
|
5227
|
-
* trajectory. Throws with the precise reason when the finding cannot be
|
|
5228
|
-
* replayed (malformed subject, unknown trajectory, non-SWE case, step out of
|
|
5229
|
-
* range) — the analyst product surfaces that reason instead of a proof.
|
|
5230
|
-
*/
|
|
5231
|
-
declare function resolveFindingInvocation(finding: AnalystReplayFinding, corpora: readonly CorpusSpec[]): ResolvedReplayInvocation;
|
|
5232
|
-
interface ReplayFindingOptions {
|
|
5233
|
-
readonly corpora: readonly CorpusSpec[];
|
|
5234
|
-
readonly out: string;
|
|
5235
|
-
/** Generates the arm-B corrected command; omit to run arm A only. */
|
|
5236
|
-
readonly fixCaller?: ChatCompletionCaller;
|
|
5237
|
-
/** Pre-supplied arm-B command; mutually exclusive with fixCaller. */
|
|
5238
|
-
readonly fixCommand?: string;
|
|
5239
|
-
readonly stepTimeoutMs?: number;
|
|
5240
|
-
readonly prefixLimit?: number;
|
|
5241
|
-
readonly backend?: ReplayExecBackend;
|
|
5242
|
-
readonly apiKey?: string;
|
|
5243
|
-
readonly baseUrl?: string;
|
|
5244
|
-
readonly onProgress?: (message: string) => void;
|
|
5245
|
-
}
|
|
5246
|
-
interface ReplayFindingResult {
|
|
5247
|
-
readonly invocation: ResolvedReplayInvocation;
|
|
5248
|
-
readonly fixCommand: string | null;
|
|
5249
|
-
readonly verdict: ReplayVerdict;
|
|
5250
|
-
}
|
|
5251
|
-
/**
|
|
5252
|
-
* The shape the analyst product calls: finding in, executed proof out.
|
|
5253
|
-
* Note the caller supplies the image via resources; when no injected backend
|
|
5254
|
-
* is given the image must already be replay-ready (uid-1000 derived) — batch
|
|
5255
|
-
* preparation lives in replay-batch's ImagePreparer.
|
|
5256
|
-
*/
|
|
5257
|
-
declare function replayVerifyFinding(finding: AnalystReplayFinding, options: ReplayFindingOptions): Promise<ReplayFindingResult>;
|
|
5258
|
-
|
|
5259
|
-
/**
|
|
5260
|
-
* Proof-carrying findings: execute analyst findings as sandbox replays.
|
|
5261
|
-
*
|
|
5262
|
-
* An analyst finding is a cited claim ("step 12 is where the run went
|
|
5263
|
-
* wrong"). This module turns each finding into an executed verdict by
|
|
5264
|
-
* replaying the trajectory prefix in a real sandbox and running the accused
|
|
5265
|
-
* step (arm A), optionally followed by a corrected step (arm B):
|
|
5266
|
-
*
|
|
5267
|
-
* `reproduced` arm A re-produced the recorded failure signature
|
|
5268
|
-
* (returncode + stable output substring).
|
|
5269
|
-
* `fix-flipped` arm A reproduced AND arm B's corrected command made the
|
|
5270
|
-
* failure vanish — the strongest per-finding proof.
|
|
5271
|
-
* `divergent` arm A executed but the recorded failure did NOT
|
|
5272
|
-
* reproduce; evidence against the finding (or against
|
|
5273
|
-
* replay fidelity — the receipt carries prefix
|
|
5274
|
-
* divergences so the reader can tell which).
|
|
5275
|
-
* `not-replayable` the finding could not be executed at all; the receipt
|
|
5276
|
-
* carries the precise reason (no step subject, unknown
|
|
5277
|
-
* trajectory, no docker image, submit step, …).
|
|
5278
|
-
*
|
|
5279
|
-
* Verification is execution, not generation: no LLM is involved unless the
|
|
5280
|
-
* caller supplies a corrected command for arm B. Sandbox infrastructure being
|
|
5281
|
-
* absent while a finding needs execution is an error, never a silent skip.
|
|
5282
|
-
*
|
|
5283
|
-
* Findings are matched by the shape the analyst product emits
|
|
5284
|
-
* (agent-eval `AnalystFinding`): `subject` (`incorrect-step-<n>` or the wire's
|
|
5285
|
-
* `incorrect-steps-<f>-<l>-<escaped|unescaped>-consequence-<c>`),
|
|
5286
|
-
* `metadata.block_first_step`, and `trace://<traj>/…` evidence refs.
|
|
5287
|
-
*/
|
|
5288
|
-
|
|
5289
|
-
interface VerifiableFinding {
|
|
5290
|
-
readonly finding_id?: string;
|
|
5291
|
-
readonly analyst_id?: string;
|
|
5292
|
-
readonly subject?: string;
|
|
5293
|
-
readonly area?: string;
|
|
5294
|
-
readonly claim?: string;
|
|
5295
|
-
readonly evidence_refs?: readonly {
|
|
5296
|
-
readonly kind?: string;
|
|
5297
|
-
readonly uri?: string;
|
|
5298
|
-
readonly excerpt?: string;
|
|
5299
|
-
}[];
|
|
5300
|
-
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
5301
|
-
}
|
|
5302
|
-
/**
|
|
5303
|
-
* 1-based step the finding accuses, or null when the finding names none.
|
|
5304
|
-
* `metadata.block_first_step` wins over the subject: the analyst records the
|
|
5305
|
-
* block's first incorrect step there even when the subject names a later
|
|
5306
|
-
* step of the same block.
|
|
5307
|
-
*/
|
|
5308
|
-
declare function findingReplayStep(finding: VerifiableFinding): number | null;
|
|
5309
|
-
/** Trajectory id from the finding's `trace://<traj>/…` evidence refs, or null. */
|
|
5310
|
-
declare function findingTrajectoryId(finding: VerifiableFinding): string | null;
|
|
5311
|
-
/**
|
|
5312
|
-
* Where the executable trajectory lives.
|
|
5313
|
-
* `direct` — one trajectory's steps.json plus a replay-ready image (the
|
|
5314
|
-
* caller owns image preparation, exactly like `traces replay-verify`).
|
|
5315
|
-
* `corpus` — CodeTraceBench corpora; each finding's trajectory is resolved
|
|
5316
|
-
* by its `trace://` evidence and the image is derived through the batch
|
|
5317
|
-
* preparer (uid-1000 chown) unless a test backend is injected.
|
|
5318
|
-
*/
|
|
5319
|
-
type FindingReplaySource = {
|
|
5320
|
-
readonly kind: 'direct';
|
|
5321
|
-
readonly stepsPath: string;
|
|
5322
|
-
readonly image: string;
|
|
5323
|
-
readonly cwd: string;
|
|
5324
|
-
readonly caseId?: string;
|
|
5325
|
-
} | {
|
|
5326
|
-
readonly kind: 'corpus';
|
|
5327
|
-
readonly corpora: readonly CorpusSpec[];
|
|
5328
|
-
readonly preparer?: ImagePreparer;
|
|
5329
|
-
};
|
|
5330
|
-
interface ResolvedFindingReplay {
|
|
5331
|
-
readonly caseId: string;
|
|
5332
|
-
readonly stepsPath: string;
|
|
5333
|
-
/** Raw image; corpus-mode execution derives the uid-1000 replay image from it. */
|
|
5334
|
-
readonly image: string;
|
|
5335
|
-
readonly cwd: string;
|
|
5336
|
-
/** 1-based step_id arm A executes. */
|
|
5337
|
-
readonly at: number;
|
|
5338
|
-
readonly recordedReturncode: number;
|
|
5339
|
-
readonly recordedStepTimeoutMs: number | null;
|
|
5340
|
-
}
|
|
5341
|
-
type FindingReplayability = {
|
|
5342
|
-
readonly replayable: true;
|
|
5343
|
-
readonly resolved: ResolvedFindingReplay;
|
|
5344
|
-
} | {
|
|
5345
|
-
readonly replayable: false;
|
|
5346
|
-
readonly reason: string;
|
|
5347
|
-
};
|
|
5348
|
-
/**
|
|
5349
|
-
* Decides whether one finding can be executed against the source, and with
|
|
5350
|
-
* what invocation. Never throws for a finding-shaped problem — every dead end
|
|
5351
|
-
* becomes a `not-replayable` reason the receipt can carry verbatim.
|
|
5352
|
-
*/
|
|
5353
|
-
declare function resolveFindingReplayability(finding: VerifiableFinding, source: FindingReplaySource): FindingReplayability;
|
|
5354
|
-
type FindingVerificationStatus = 'reproduced' | 'fix-flipped' | 'not-replayable' | 'divergent';
|
|
5355
|
-
interface FindingVerification {
|
|
5356
|
-
readonly finding_id: string | null;
|
|
5357
|
-
readonly subject: string | null;
|
|
5358
|
-
readonly trajectory_id: string | null;
|
|
5359
|
-
/** 1-based step the proof executed at; null when not replayable. */
|
|
5360
|
-
readonly step: number | null;
|
|
5361
|
-
readonly verified: FindingVerificationStatus;
|
|
5362
|
-
/** Present exactly when `verified` is `not-replayable`. */
|
|
5363
|
-
readonly reason: string | null;
|
|
5364
|
-
/** Receipt directory: receipt.json plus, when executed, replay-verdict.json + report.md. */
|
|
5365
|
-
readonly receipt: string;
|
|
5366
|
-
readonly verdict_path: string | null;
|
|
5367
|
-
/** Receipt dir of the executed proof this finding shares (same case, step, and fix). */
|
|
5368
|
-
readonly deduplicated_with: string | null;
|
|
5369
|
-
}
|
|
5370
|
-
interface VerifyFindingsRun {
|
|
5371
|
-
readonly out: string;
|
|
5372
|
-
readonly verifications: readonly FindingVerification[];
|
|
5373
|
-
readonly counts: Readonly<Record<FindingVerificationStatus, number>>;
|
|
5374
|
-
/** Sandbox executions actually performed (deduplicated proofs count once). */
|
|
5375
|
-
readonly executions: number;
|
|
5376
|
-
}
|
|
5377
|
-
interface VerifyFindingsOptions {
|
|
5378
|
-
readonly source: FindingReplaySource;
|
|
5379
|
-
/** Receipt root; one subdirectory per finding plus verifications.json. */
|
|
5380
|
-
readonly out: string;
|
|
5381
|
-
/** Corrected command for arm B on every executed finding; omit for arm A only. */
|
|
5382
|
-
readonly fixCommand?: string;
|
|
5383
|
-
readonly stepTimeoutMs?: number;
|
|
5384
|
-
readonly prefixLimit?: number;
|
|
5385
|
-
/** Injectable for tests; when set, no sandbox reachability check and no image preparation run. */
|
|
5386
|
-
readonly backend?: ReplayExecBackend;
|
|
5387
|
-
readonly apiKey?: string;
|
|
5388
|
-
readonly baseUrl?: string;
|
|
5389
|
-
readonly maxLifetimeSeconds?: number;
|
|
5390
|
-
readonly onProgress?: (message: string) => void;
|
|
5391
|
-
}
|
|
5392
|
-
/** Arm A reproduced → the fix flipping it beats plain reproduction; anything else diverged. */
|
|
5393
|
-
declare function classifyVerdict(verdict: ReplayVerdict): FindingVerificationStatus;
|
|
5394
|
-
declare const DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
5395
|
-
/**
|
|
5396
|
-
* Fails loud when the sandbox SDK adapter is not answering. Executed findings
|
|
5397
|
-
* require real infrastructure; verification is never silently skipped.
|
|
5398
|
-
*/
|
|
5399
|
-
declare function assertSandboxReachable(baseUrl: string): Promise<void>;
|
|
5400
|
-
/**
|
|
5401
|
-
* Verifies every finding against the source: resolves replayability, executes
|
|
5402
|
-
* one sandbox proof per distinct (case, step, fix) — findings accusing the
|
|
5403
|
-
* same step share the executed proof — and writes a receipt directory per
|
|
5404
|
-
* finding plus a run-level verifications.json.
|
|
5405
|
-
*/
|
|
5406
|
-
declare function verifyFindings(findings: readonly VerifiableFinding[], options: VerifyFindingsOptions): Promise<VerifyFindingsRun>;
|
|
5407
|
-
/** Markdown section the analyze report appends when --verify-findings ran. */
|
|
5408
|
-
declare function renderVerifiedFindingsSection(run: VerifyFindingsRun): string;
|
|
5409
|
-
/**
|
|
5410
|
-
* Accepts the two shapes findings travel in: a bare JSON array of analyst
|
|
5411
|
-
* findings, or an object with a `findings` array (e.g. an extracted
|
|
5412
|
-
* `observations[n]` from an agent-eval result.json).
|
|
5413
|
-
*/
|
|
5414
|
-
declare function readFindingsFile(path: string): VerifiableFinding[];
|
|
5415
|
-
interface VerifyFindingsCliArgs {
|
|
5416
|
-
findingsPath: string;
|
|
4613
|
+
/** Runs finding proofs against the Tangle Sandbox, refusing absent infrastructure. */
|
|
4614
|
+
declare function verifyFindingsInSandbox(findings: readonly VerifiableFinding[], options: {
|
|
5417
4615
|
source: FindingReplaySource;
|
|
5418
4616
|
out: string;
|
|
4617
|
+
sandbox: {
|
|
4618
|
+
apiKey: string;
|
|
4619
|
+
baseUrl: string;
|
|
4620
|
+
maxLifetimeSeconds?: number;
|
|
4621
|
+
};
|
|
5419
4622
|
fixCommand?: string;
|
|
5420
4623
|
stepTimeoutMs?: number;
|
|
5421
4624
|
prefixLimit?: number;
|
|
5422
|
-
|
|
5423
|
-
apiKeyEnv: string;
|
|
5424
|
-
maxLifetimeSeconds?: number;
|
|
5425
|
-
}
|
|
5426
|
-
declare function verifyFindingsUsage(): string;
|
|
5427
|
-
declare function parseVerifyFindingsArgs(argv: readonly string[]): VerifyFindingsCliArgs | 'help';
|
|
5428
|
-
declare function cmdVerifyFindings(argv: readonly string[]): Promise<void>;
|
|
4625
|
+
}): Promise<VerifyFindingsRun>;
|
|
5429
4626
|
|
|
5430
4627
|
/**
|
|
5431
4628
|
* Local upload-dedup state.
|
|
@@ -5529,4 +4726,4 @@ interface ExecuteOptions {
|
|
|
5529
4726
|
/** Send the final, new outbound items (or write their dry-run OTLP). */
|
|
5530
4727
|
declare function executeUpload(plan: UploadPlan, opts?: ExecuteOptions): Promise<UploadResult>;
|
|
5531
4728
|
|
|
5532
|
-
export { ACTOR_ATTR, ATTR, type Actor, type AdoptionOptions, type AdoptionReport, AmpAdapter, type AnalysisEngineFromEnvOptions, type AnalystReplayFinding, type AnalyzeOptions, type AnalyzeResult, type AnswerSchema, type ArmExecutionResult, type AssembleSessionBundleOptions, BLOBS_DIR, type BuildPolicyEvidenceOptions, type BuildSessionIndexOptions, type BuildTraceFindingPacketOptions, CHANGED_FILE_LIST_CAP, CITATION_NORMALIZATION_VERSION, CODING_REDACTION_RULES, CONTENT_CAP, COORDINATION_LOG_FILE, CORRECTIVE_REACTIONS, type CaseResources, type ChangedFileFact, type ChatCompletionCaller, type ChatOutcome, type ChatTrajectory, type ChatTrajectoryMessage, type ChatTrajectoryOptions, type ChatUsage, ClaudeAdapter, type CodeTraceBenchFileRef, type CodeTraceBenchImportOptions, type CodeTraceBenchImportReceipt, type CodeTraceBenchImportResult, type CodeTraceBenchImportedTrace, type CodeTraceBenchStep, CodexAdapter, CodexExecAdapter, CodexExecStreamError, CodexTaskScopeError, type CollectOptions, type CollectPolicyEvidenceOptions, type CollectSessionIndexOptions, type CollectSessionSelectionOptions, type CollectSessionWorkflowOptions, type ConformanceRenderOptions, type CoordinationLog, type CoordinationQuestion, type CoordinationSteer, CopilotAdapter, type CorpusSpec, type CorruptionMode, type CwdSource, DEFAULT_ANALYST_MODEL, DEFAULT_ASK_CONCURRENCY, DEFAULT_HARNESS, DEFAULT_SANDBOX_BASE_URL, type DockerImagePreparerOptions, EMPTY_SPAN_RUN_SNAPSHOT, EXPORT_RULE, EmptySessionError, type EnumerationResult, type EvidenceOnlyVerdict, type ExcludedCase, type ExcludedTurnFact, type ExecuteOptions, type ExportRule, type ExternalAnalysisKind, type ExternalAnalysisPayload, type ExternalAnalysisResult, type ExternalAnalyzer, type ExternalAnalyzerOptions, type ExternalDiscoveryCandidate, FACT_LIST_CAP, FACT_TEXT_CAP, FactoryAdapter, type FailedFixAttempt, type FailureFollowUp, type FailureFollowUpReport, type FileRunContextReaderOptions, type FinalMessageFact, type FindSessionTextLeaksOptions, type FindingRejectionCounts, type FindingRejectionReasons, type FindingRejectionTally, type FindingReplaySource, type FindingReplayability, type FindingVerification, type FindingVerificationStatus, type FixArmExecution, type FixArmExecutor, type FixGenerationOutcome, type FixLoopAttemptRecord, type FixLoopOptions, type FixLoopResult, type FixPromptInput, type FollowUpKind, ForgeAdapter, GeminiAdapter, type HaloAnalyzerOptions, type HaloReasoningEffort, type HarnessTraceAdapter, type HodoscopeAnalyzerOptions, type HodoscopeInput, type HodoscopeMessage, type HodoscopeSourceMapEntry, type HodoscopeTrajectory, INGEST_SOURCE_CLI, INHERITED_SOURCE_ATTR, INHERITED_SPANS_OMITTED_ATTR, INHERITED_SPAN_ATTR, INHERITED_SPAN_COUNT_ATTR, type ImagePreparation, type ImagePreparer, type IncorrectStepsSubject, type IngestedTrajectory, type InheritedSpanSource, type InspectSessionIndexOptions, JsonSourceError, type JsonlCorruptionReceipt, JsonlParseError, type JsonlReadOptions, type LiveBatchOptions, type LocateOptions, type LoopConvergence, type LoopConvergenceReport, type LoopIteration, type LoopTrend, type LoopTrendBasis, MAX_SHELL_DEPTH, MAX_TRACE_QUESTION_CHARS, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpIngestIssue, type OtlpIngestIssueKind, type OtlpInput, type OtlpInputFile, type OtlpRowConversion, type OtlpSpan, type OtlpSpanKind, type OtlpSpanLink, type OtlpStatusCode, POLICY_EVIDENCE_NOTE, PREPARED_CONTEXT_BYTE_CEILING, type ParentTaskResolution, type ParseOptions, PartialUploadError, PiAdapter, type PiAdapterOptions, type PipelineOptions, type PipelineReport, type PlanOptions, type PolicyEvidenceLoopSummary, type PolicyEvidenceRecord, type PolicyEvidenceToolSummary, type PrefixDivergence, type PrefixReplayResult, type PrimeAnalyzerOptions, type PrimeTransport, type PrimeTransportRequest, type PrimeTransportResponse, type ProjectSessionBundleOptions, type ProjectSessionBundleResult, type PullRequestFact, type PullRequestFacts, type PullRequestReading, QwenAdapter, REACTIONS, RESULT_FILE, RULE, type Reaction, type ReactionReport, type ReactionSessionReport, type Redactor, type ReplayArmVerdict, type ReplayBatchCaseRow, type ReplayBatchCliArgs, type ReplayBatchFixResult, type ReplayBatchOptions, type ReplayBatchReport, type ReplayExclusionReason, type ReplayExecBackend, type ReplayExecResult, type ReplayExecSession, type ReplayFindingOptions, type ReplayFindingResult, type ReplayVerdict, type ReplayVerifyCliArgs, type ReplayVerifyOptions, type ReplayableCase, type RepoAttrs, type RepoResolution, type RepoResolutionSource, type ResolvedFindingReplay, type ResolvedOtlpInput, type ResolvedReplayInvocation, type ResourceResolution, type RunContextJournal, type RunContextNode, type RunContextResult, type RunContextSnapshot, type RunResult, type RunWatchSnapshot, type RunWatchTarget, type RuntimeTrace, SESSION_FACTS_VERSION_SUFFIX, SESSION_TOKEN_TOTAL_ATTR, SOURCE_ATTRIBUTE_PREFIX, SPAN_SYNTHESIZED_ATTR, SPAWN_JOURNAL_FILE, STATUS_MARK, SUBMIT_ACTION_SIGNATURE, SYNTHESIZED_SOURCE_ATTR, SYNTHESIZED_SPAN_ATTR, SandboxCounterfactualRunner, type SandboxCounterfactualRunnerOptions, type SandboxReplayBackendOptions, type ScanOptions, type ScannedSession, type SessionBatch, type SessionBundleAbsence, type SessionBundleCarriedSignatures, type SessionBundleExclusion, type SessionBundleExclusionRule, type SessionBundleFile, type SessionBundleLeakCheck, type SessionBundleLedgerSlice, type SessionBundleManifest, type SessionBundleProjection, type SessionBundleResult, type SessionBundleSliceRule, type SessionBundleView, type SessionCorruptionReceipt, type SessionEnvironment, type SessionFact, type SessionFacts, type SessionFactsReport, type SessionIdentityConflict, type SessionIdentityIndex, type SessionIntegrity, type SessionRef, type SessionRelationship, type SessionRole, type SessionSeedGroup, type SessionSelection, type SessionSelectionRow, type SessionSourceFile, type SessionTextLeak, type SessionTextLeakReport, type SessionTurnFact, type SessionWorkflow, SessionWorkflowError, type SessionWorkflowErrorCode, type SessionWorkflowIssue, type SessionWorkflowSession, type SessionWorkflowSummary, type SkippedAnalysis, type SkippedInputFile, type SourceRecordReference, type SpanInput, type SpanNodeStatus, type SpanRedaction, type SpanRunNode, type SpanRunSnapshot, type SpanStructure, type SpawnedChildResolution, type SpendRoll, type SteeringChainReport, type SteeringEdge, type SubagentSpawnFact, TRACES_REDACTION_RULES, TRACE_QUESTION_PREVIEW_HEAD_CHARS, type TraceAgenticRoute, type TraceAgenticRouteReason, type TraceAgenticRouteReasonCode, type TraceContextIndex, type TraceContextRoot, type TraceEvidenceExportOptions, type TraceEvidenceExportResult, type TraceEvidenceFormatOption, type TraceEvidenceInputFormat, type TraceEvidenceRow, type TraceExecutionOptions, type TraceFindingPacket, type TraceImprovementArtifacts, type TraceImprovementOptions, type TraceImprovementResult, type TraceIndexedFile, type TraceInspectionArea, type TraceInspectionFinding, type TraceInspectionReport, type TraceInspectionSeverity, type TraceInvestigationOptions, type TraceInvestigationResult, type TraceLiveAction, type TraceLiveActionKind, type TraceLiveAnalysisContext, type TraceLiveAnalyst, type TraceLiveBatch, type TraceLiveEvidence, type TraceLiveFinding, type TraceLiveLoop, type TraceLiveSeverity, type TraceQuestion, type TraceQuestionAnswer, type TraceQuestionCitation, type TraceQuestionFailureKind, type TraceQuestionTrace, type TraceQuestionsArtifacts, type TraceQuestionsOptions, type TraceQuestionsResult, type TraceQuestionsTotals, type TraceSessionIndex, type TraceSessionIndexRow, type TraceStoreInvestigationOptions, type TraceStoreInvestigationResult, type TraceStreamEvent, type TraceStreamOptions, type TraceStreamReplayOptions, type TraceStreamSession, type TracesConfig, type TriggerPair, type TurnActorCount, type UnavailableCapabilities, type UploadBackend, type UploadIdentity, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, type UserPromptInput, type VerifiableFinding, type VerifyFindingsCliArgs, type VerifyFindingsOptions, type VerifyFindingsRun, type WatchRunTargetOptions, ZERO_ROLL, type ZaiChatCallerOptions, addSpend, alreadyUploaded, analysisEngineFromEnv, analyzeAdoption, analyzeLiveBatch, analyzeLoopConvergence, analyzeReactions, analyzeSpans, analyzeSteeringChain, answerSchemaErrors, applyRedactor, assembleSessionBundle, assertAnswerSchema, assertSandboxReachable, buildFixPrompt, buildPolicyEvidenceRecord, buildRetryFixPrompt, buildSessionFactsReport, buildSessionIndex, buildSessionIndexFromRows, buildSpanRunTree, buildTraceFindingPacket, capText, chatTrajectoryToSpans, classifyEvidenceOnlyPath, classifyFailureFollowUps, classifyLiveActions, classifyReaction, classifyVerdict, claudeActor, clipText, cmdReplayVerify, cmdReplayVerifyBatch, cmdVerifyFindings, codexActor, collectPolicyEvidence, collectSessionIndex, collectSessionSelection, collectSessionWorkflow, collectSessions, commandAnalyzer, commandRedactor, commandTextFromInput, computeSessionFacts, conformanceOfSpans, connectors, countScriptCommands, countSkillRunsJsonl, createBundleSourceReader, createFindingRejectionTally, cwdMatchesSelection, cwdMatchesSelectionResolved, defaultTraceLiveAnalysts, deriveFailureSignature, derivedImageTag, describeSessionRelationship, dockerImagePreparer, engineCallReservationFloorUsd, enumerateReplayableCases, environmentAttributes, environmentFromSpanAttributes, equivalentGitCwds, executeUpload, exportTraceEvidenceFile, exportTraceEvidenceRows, exportTraceEvidenceText, externalFailureMessage, extractFixCommand, fileRunContextSupervisorRunReader, fileSha256, findContextRoot, findSessionTextLeaks, findSpanSourceFiles, findingRejection, findingRejectionDetail, findingReplayStep, findingTrajectoryId, forEachTreeNode, formatFindingRejections, generateFixCommand, goldIncorrectSteps, haloAnalyzer, hashSessionBundleFiles, hodoscopeAnalyzer, httpJsonTransport, importCodeTraceBench, incompleteInputsNote, indexSessionIdsByTrace, inferRole, inferSpanKind, ingestCodeTraceBenchSteps, inspectSessionIndex, int, isBridgeMismatchError, isChatTrajectory, isChatTrajectoryMessage, isFileRunContextDir, isInheritedSpan, isSubmitAction, isSynthesizedSpan, knownHarnesses, listAdapters, listSessionBundleFiles, loadState, loadTraceQuestionsFile, loadTracesConfig, locateSessions, looksLikeAgentPrompt, mergeRolls, mergeTracesConfig, ms, normalizeAnalystCitations, normalizeRemote, normalizeTraceQuestions, otlpRowToSpan, outboundHash, parseCorpusFlag, parseIncorrectStepsSubject, parseIsoToEpochMs, parseJsonAnswer, parseObservationOutput, parseRecordedReturncode, parseReplayBatchArgs, parseReplayVerifyArgs, parseSession, parseSince, parseVerifyFindingsArgs, planTraceAgenticRoute, planUpload, primeAnalyzer, projectSessionBundle, readCoordinationLog, readFileRunContext, readFindingsFile, readJsonFile, readJsonl, readLabelEntries, readOtlpInput, readPullRequests, readRunContextJournal, readRunContextSnapshot, readRunWatchSnapshot, readSessionBundleManifest, readSessionIndexFile, readSpanRunSnapshot, recordSessionCorruption, redactSpans, renderBatchReport, renderConformance, renderInspectionReport, renderRunContextSnapshot, renderRunWatchSnapshot, renderSessionFacts, renderSessionFactsContext, renderSpanRunSnapshot, renderTraceQuestionInstructions, renderTraceQuestionPrompt, renderTraceQuestionsReport, renderValidation, renderVerifiedFindingsSection, replayBatchUsage, replayVerify, replayVerifyFinding, replayVerifyUsage, resolveAdapter, resolveCaseResources, resolveFindingInvocation, resolveFindingReplayability, resolveOtlpInputFiles, resolveRepoAttrs, resolveRunContextDir, resolveRunWatchTarget, resolveSessionRepoAttrs, runCommand, runExternalAnalyzers, runFixLoop, runPipelines, runReplayBatch, runTotals, runTraceImprovement, runTraceInvestigation, runTraceQuestions, runTraceStoreInvestigation, sandboxReplayBackend, saveReport, saveState, scanSessions, seededSample, selectAdapters, serializeInspectionReport, serializePolicyEvidence, serializeSessionIndex, serializeSpans, serializeTraceStreamEvent, sessionFactsContext, sessionHash, sessionIdFromAttributes, sessionIntegrityAttributes, sessionTextSignatures, shellCommands, skippedAnalyses, span, spanRunTotals, stampEnvironmentAttrs, stampRepoAttrs, stampSessionIdentity, stampSessionIntegrity, stampSpanWorkdirRepoAttrs, streamSessions, summarizeOtlpExecution, summarizeSpanExecution, summarizeSpanStructure, takeJsonl, textIsSynthetic, toOpenInferenceSpan, toRuntimeStore, toTraceSpanEvents, tokens, totalAgenticFailureMessage, totalFindingRejections, traceAgenticKinds, traceCitationsInText, traceStreamEventsFromSpans, unavailableCapabilities, uploadKey, usd, userPromptSpan, validateOtlpSpans, validationExitCode, verifyFindings, verifyFindingsUsage, watchRunTarget, watchSessions, withSessionFactsContext, wrap, wrapActionForExec, writeHodoscopeInput, writeInspectionReportFile, writeOtlpFile, writePolicyEvidenceFile, writeSessionBundleManifest, writeSessionIndexFile, writeTraceEvidenceExportFile, writeTraceImprovementArtifacts, writeTraceQuestionsArtifacts, zaiChatCaller, zeroPricedCost };
|
|
4729
|
+
export { ACTOR_ATTR, ATTR, type Actor, type AdoptionOptions, type AdoptionReport, AmpAdapter, type AnalysisEngineFromEnvOptions, type AnalyzeOptions, type AnalyzeResult, type AnswerSchema, type AssembleSessionBundleOptions, BLOBS_DIR, type BuildPolicyEvidenceOptions, type BuildSessionIndexOptions, type BuildTraceFindingPacketOptions, CHANGED_FILE_LIST_CAP, CITATION_NORMALIZATION_VERSION, CODING_REDACTION_RULES, CONTENT_CAP, COORDINATION_LOG_FILE, CORRECTIVE_REACTIONS, type ChangedFileFact, type ChatTrajectory, type ChatTrajectoryMessage, type ChatTrajectoryOptions, ClaudeAdapter, type CodeTraceBenchFileRef, type CodeTraceBenchImportOptions, type CodeTraceBenchImportReceipt, type CodeTraceBenchImportResult, type CodeTraceBenchImportedTrace, type CodeTraceBenchStep, CodexAdapter, CodexExecAdapter, CodexExecStreamError, CodexTaskScopeError, type CollectOptions, type CollectPolicyEvidenceOptions, type CollectSessionIndexOptions, type CollectSessionSelectionOptions, type CollectSessionWorkflowOptions, type ConformanceRenderOptions, type CoordinationLog, type CoordinationQuestion, type CoordinationSteer, CopilotAdapter, type CorruptionMode, DEFAULT_ANALYST_MODEL, DEFAULT_ASK_CONCURRENCY, DEFAULT_HARNESS, DEFAULT_SANDBOX_BASE_URL, EMPTY_SPAN_RUN_SNAPSHOT, EXPORT_RULE, EmptySessionError, type EvidenceOnlyVerdict, type ExcludedTurnFact, type ExecuteOptions, type ExportRule, type ExternalAnalysisKind, type ExternalAnalysisPayload, type ExternalAnalysisResult, type ExternalAnalyzer, type ExternalAnalyzerOptions, type ExternalDiscoveryCandidate, FACT_LIST_CAP, FACT_TEXT_CAP, FactoryAdapter, type FailureFollowUp, type FailureFollowUpReport, type FileRunContextReaderOptions, type FinalMessageFact, type FindSessionTextLeaksOptions, type FindingRejectionCounts, type FindingRejectionReasons, type FindingRejectionTally, type FollowUpKind, ForgeAdapter, GeminiAdapter, type HaloAnalyzerOptions, type HaloReasoningEffort, type HarnessTraceAdapter, type HodoscopeAnalyzerOptions, type HodoscopeInput, type HodoscopeMessage, type HodoscopeSourceMapEntry, type HodoscopeTrajectory, INGEST_SOURCE_CLI, INHERITED_SOURCE_ATTR, INHERITED_SPANS_OMITTED_ATTR, INHERITED_SPAN_ATTR, INHERITED_SPAN_COUNT_ATTR, type InheritedSpanSource, type InspectSessionIndexOptions, JsonSourceError, type JsonlCorruptionReceipt, JsonlParseError, type JsonlReadOptions, type LiveBatchOptions, type LocateOptions, type LoopConvergence, type LoopConvergenceReport, type LoopIteration, type LoopTrend, type LoopTrendBasis, MAX_SHELL_DEPTH, MAX_TRACE_QUESTION_CHARS, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpIngestIssue, type OtlpIngestIssueKind, type OtlpInput, type OtlpInputFile, type OtlpRowConversion, type OtlpSpan, type OtlpSpanKind, type OtlpSpanLink, type OtlpStatusCode, POLICY_EVIDENCE_NOTE, PREPARED_CONTEXT_BYTE_CEILING, type ParentTaskResolution, type ParseOptions, PartialUploadError, PiAdapter, type PiAdapterOptions, type PipelineOptions, type PipelineReport, type PlanOptions, type PolicyEvidenceLoopSummary, type PolicyEvidenceRecord, type PolicyEvidenceToolSummary, type PrimeAnalyzerOptions, type PrimeTransport, type PrimeTransportRequest, type PrimeTransportResponse, type ProjectSessionBundleOptions, type ProjectSessionBundleResult, type PullRequestFact, type PullRequestFacts, type PullRequestReading, QwenAdapter, REACTIONS, RESULT_FILE, RULE, type Reaction, type ReactionReport, type ReactionSessionReport, type Redactor, type RepoAttrs, type RepoResolution, type RepoResolutionSource, type ResolvedOtlpInput, type RunContextJournal, type RunContextNode, type RunContextResult, type RunContextSnapshot, type RunResult, type RunWatchSnapshot, type RunWatchTarget, type RuntimeTrace, SESSION_FACTS_VERSION_SUFFIX, SESSION_TOKEN_TOTAL_ATTR, SOURCE_ATTRIBUTE_PREFIX, SPAN_SYNTHESIZED_ATTR, SPAWN_JOURNAL_FILE, STATUS_MARK, SYNTHESIZED_SOURCE_ATTR, SYNTHESIZED_SPAN_ATTR, type SandboxReplayBackendOptions, type ScanOptions, type ScannedSession, type SessionBatch, type SessionBundleAbsence, type SessionBundleCarriedSignatures, type SessionBundleExclusion, type SessionBundleExclusionRule, type SessionBundleFile, type SessionBundleIssue, type SessionBundleIssueKind, type SessionBundleLeakCheck, type SessionBundleLedgerSlice, type SessionBundleManifest, type SessionBundleProjection, type SessionBundleResult, type SessionBundleSliceRule, type SessionBundleVerification, type SessionBundleView, type SessionCorruptionReceipt, type SessionEnvironment, type SessionFact, type SessionFacts, type SessionFactsReport, type SessionIdentityConflict, type SessionIdentityIndex, type SessionIntegrity, type SessionRef, type SessionRelationship, type SessionRole, type SessionSeedGroup, type SessionSelection, type SessionSelectionRow, type SessionSourceFile, type SessionTextLeak, type SessionTextLeakReport, type SessionTurnFact, type SessionWorkflow, SessionWorkflowError, type SessionWorkflowErrorCode, type SessionWorkflowIssue, type SessionWorkflowSession, type SessionWorkflowSummary, type SkippedAnalysis, type SkippedInputFile, type SourceRecordReference, type SpanInput, type SpanNodeStatus, type SpanRedaction, type SpanRunNode, type SpanRunSnapshot, type SpanStructure, type SpawnedChildResolution, type SpendRoll, type SteeringChainReport, type SteeringEdge, type SubagentSpawnFact, TRACES_REDACTION_RULES, TRACE_QUESTION_PREVIEW_HEAD_CHARS, type TraceAgenticRoute, type TraceAgenticRouteReason, type TraceAgenticRouteReasonCode, type TraceContextIndex, type TraceContextRoot, type TraceEvidenceExportOptions, type TraceEvidenceExportResult, type TraceEvidenceFormatOption, type TraceEvidenceInputFormat, type TraceEvidenceRow, type TraceExecutionOptions, type TraceFindingPacket, type TraceImprovementArtifacts, type TraceImprovementOptions, type TraceImprovementResult, type TraceIndexedFile, type TraceInspectionArea, type TraceInspectionFinding, type TraceInspectionReport, type TraceInspectionSeverity, type TraceInvestigationOptions, type TraceInvestigationResult, type TraceLiveAction, type TraceLiveActionKind, type TraceLiveAnalysisContext, type TraceLiveAnalyst, type TraceLiveBatch, type TraceLiveEvidence, type TraceLiveFinding, type TraceLiveLoop, type TraceLiveSeverity, type TraceQuestion, type TraceQuestionAnswer, type TraceQuestionCitation, type TraceQuestionFailureKind, type TraceQuestionTrace, type TraceQuestionsArtifacts, type TraceQuestionsOptions, type TraceQuestionsResult, type TraceQuestionsTotals, type TraceSessionIndex, type TraceSessionIndexRow, type TraceStoreInvestigationOptions, type TraceStoreInvestigationResult, type TraceStreamEvent, type TraceStreamOptions, type TraceStreamReplayOptions, type TraceStreamSession, type TracesConfig, type TriggerPair, type TurnActorCount, type UnavailableCapabilities, type UploadBackend, type UploadIdentity, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, type UserPromptInput, type WatchRunTargetOptions, ZERO_ROLL, type ZaiChatCallerOptions, addSpend, alreadyUploaded, analysisEngineFromEnv, analyzeAdoption, analyzeLiveBatch, analyzeLoopConvergence, analyzeReactions, analyzeSpans, analyzeSteeringChain, answerSchemaErrors, applyRedactor, assembleSessionBundle, assertAnswerSchema, assertSandboxReachable, buildPolicyEvidenceRecord, buildSessionFactsReport, buildSessionIndex, buildSessionIndexFromRows, buildSpanRunTree, buildTraceFindingPacket, capText, chatTrajectoryToSpans, classifyEvidenceOnlyPath, classifyFailureFollowUps, classifyLiveActions, classifyReaction, claudeActor, codexActor, collectPolicyEvidence, collectSessionIndex, collectSessionSelection, collectSessionWorkflow, collectSessions, commandAnalyzer, commandRedactor, commandTextFromInput, computeSessionFacts, conformanceOfSpans, connectors, countSkillRunsJsonl, createBundleSourceReader, createFindingRejectionTally, cwdMatchesSelection, cwdMatchesSelectionResolved, defaultTraceLiveAnalysts, describeSessionRelationship, engineCallReservationFloorUsd, environmentAttributes, environmentFromSpanAttributes, equivalentGitCwds, executeUpload, exportTraceEvidenceFile, exportTraceEvidenceRows, exportTraceEvidenceText, externalFailureMessage, fileRunContextSupervisorRunReader, fileSha256, findContextRoot, findSessionTextLeaks, findSpanSourceFiles, findingRejection, findingRejectionDetail, forEachTreeNode, formatFindingRejections, haloAnalyzer, hashSessionBundleFiles, hodoscopeAnalyzer, httpJsonTransport, importCodeTraceBench, incompleteInputsNote, indexSessionIdsByTrace, inferRole, inferSpanKind, inspectSessionIndex, int, isBridgeMismatchError, isChatTrajectory, isChatTrajectoryMessage, isFileRunContextDir, isInheritedSpan, isSynthesizedSpan, knownHarnesses, listAdapters, listSessionBundleFiles, loadState, loadTraceQuestionsFile, loadTracesConfig, locateSessions, looksLikeAgentPrompt, mergeRolls, mergeTracesConfig, ms, normalizeAnalystCitations, normalizeRemote, normalizeTraceQuestions, otlpRowToSpan, outboundHash, parseIsoToEpochMs, parseJsonAnswer, parseSession, parseSince, planTraceAgenticRoute, planUpload, primeAnalyzer, projectSessionBundle, readCoordinationLog, readFileRunContext, readJsonFile, readJsonl, readOtlpInput, readPullRequests, readRunContextJournal, readRunContextSnapshot, readRunWatchSnapshot, readSessionBundleManifest, readSessionIndexFile, readSpanRunSnapshot, recordSessionCorruption, redactSpans, renderConformance, renderInspectionReport, renderRunContextSnapshot, renderRunWatchSnapshot, renderSessionFacts, renderSessionFactsContext, renderSpanRunSnapshot, renderTraceQuestionInstructions, renderTraceQuestionPrompt, renderTraceQuestionsReport, renderValidation, resolveAdapter, resolveOtlpInputFiles, resolveRepoAttrs, resolveRunContextDir, resolveRunWatchTarget, resolveSessionRepoAttrs, runCommand, runExternalAnalyzers, runPipelines, runTotals, runTraceImprovement, runTraceInvestigation, runTraceQuestions, runTraceStoreInvestigation, sandboxReplayBackend, saveReport, saveState, scanSessions, selectAdapters, serializeInspectionReport, serializePolicyEvidence, serializeSessionIndex, serializeSpans, serializeTraceStreamEvent, sessionFactsContext, sessionHash, sessionIdFromAttributes, sessionIntegrityAttributes, sessionTextSignatures, shellCommands, skippedAnalyses, span, spanRunTotals, stampEnvironmentAttrs, stampRepoAttrs, stampSessionIdentity, stampSessionIntegrity, stampSpanWorkdirRepoAttrs, streamSessions, summarizeOtlpExecution, summarizeSpanExecution, summarizeSpanStructure, takeJsonl, textIsSynthetic, toOpenInferenceSpan, toRuntimeStore, toTraceSpanEvents, tokens, totalAgenticFailureMessage, totalFindingRejections, traceAgenticKinds, traceCitationsInText, traceStreamEventsFromSpans, unavailableCapabilities, uploadKey, usd, userPromptSpan, validateOtlpSpans, validationExitCode, verifyFindingsInSandbox, verifySessionBundle, watchRunTarget, watchSessions, withSessionFactsContext, wrap, writeHodoscopeInput, writeInspectionReportFile, writeOtlpFile, writePolicyEvidenceFile, writeSessionBundleManifest, writeSessionIndexFile, writeTraceEvidenceExportFile, writeTraceImprovementArtifacts, writeTraceQuestionsArtifacts, zaiChatCaller, zeroPricedCost };
|