@agentskit/harness 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -32
- package/CONTRIBUTING.md +60 -12
- package/MANIFESTO.md +23 -0
- package/README.md +199 -142
- package/dist/cli.js +796 -221
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +739 -121
- package/dist/index.js +1179 -303
- package/dist/index.js.map +1 -1
- package/docs/ADR-0025-portable-orchestration-controls.md +27 -0
- package/docs/ORGANIZATION.md +37 -0
- package/package.json +41 -34
- package/docs/ADR-0025-ci-dogfood.md +0 -22
- package/docs/ADR-0026-ci-evidence-artifact.md +0 -22
- package/docs/ADR-0027-portable-evidence.md +0 -19
- package/docs/ADR-0028-effective-metrics.md +0 -20
- package/docs/ADR-0029-honest-ci-preparation.md +0 -20
- package/docs/ADR-0030-agentskit-os-benchmark-bridge.md +0 -20
- package/docs/ADR-0031-real-provider-baseline.md +0 -18
- package/docs/ADR-0032-harness-equivalent-benchmark.md +0 -25
- package/docs/ADR-0033-portable-agent-gate.md +0 -25
- package/docs/ADR-0034-measurement-quality-gates.md +0 -25
- package/docs/ADR-0035-reproducible-benchmark-samples.md +0 -20
- package/docs/ADR-0036-comparable-baseline-samples.md +0 -20
- package/docs/ADR-0037-replicated-baseline-collection.md +0 -27
- package/docs/ADR-0038-end-to-end-benchmark-boundary.md +0 -28
- package/docs/ADR-0039-artifact-and-protocol-metrics.md +0 -39
- package/docs/ADR-0040-benchmark-corpus-surfaces.md +0 -32
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ declare const LEGAL_TRANSITIONS: {
|
|
|
3
3
|
readonly CLARIFYING: readonly ["PLANNED", "BLOCKED", "CANCELLED"];
|
|
4
4
|
readonly PLANNED: readonly ["IMPLEMENTING", "CLARIFYING", "STALE", "CANCELLED"];
|
|
5
5
|
readonly IMPLEMENTING: readonly ["VERIFYING", "CLARIFYING", "STALE", "CANCELLED"];
|
|
6
|
-
readonly VERIFYING: readonly ["AWAITING_HUMAN_APPROVAL", "BLOCKED", "STALE", "CANCELLED"];
|
|
6
|
+
readonly VERIFYING: readonly ["AWAITING_HUMAN_APPROVAL", "COMPLETE", "BLOCKED", "STALE", "CANCELLED"];
|
|
7
7
|
readonly AWAITING_HUMAN_APPROVAL: readonly ["AWAITING_AUTHORIZATION", "COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"];
|
|
8
8
|
readonly AWAITING_AUTHORIZATION: readonly ["COMPLETE", "BLOCKED", "IMPLEMENTING", "STALE", "CANCELLED"];
|
|
9
9
|
readonly COMPLETE: readonly ["STALE", "SUPERSEDED"];
|
|
@@ -13,7 +13,7 @@ declare const LEGAL_TRANSITIONS: {
|
|
|
13
13
|
readonly SUPERSEDED: readonly [];
|
|
14
14
|
};
|
|
15
15
|
|
|
16
|
-
type HarnessErrorCode = 'HARNESS_ERROR' | 'INVALID_CONFIG' | 'INVALID_INPUT' | 'INVALID_STATE' | 'POLICY_BLOCKED' | 'CLARIFYING' | 'STALE' | 'WORKTREE_DIRTY' | 'ACTIVE_RUN' | 'NO_RUN' | 'HUMAN_APPROVAL_REQUIRED';
|
|
16
|
+
type HarnessErrorCode = 'HARNESS_ERROR' | 'INVALID_CONFIG' | 'INVALID_INPUT' | 'INVALID_STATE' | 'POLICY_BLOCKED' | 'CLARIFYING' | 'STALE' | 'WORKTREE_DIRTY' | 'ACTIVE_RUN' | 'NO_RUN' | 'HUMAN_APPROVAL_REQUIRED' | 'GIT_REQUIRED';
|
|
17
17
|
declare class HarnessError extends Error {
|
|
18
18
|
readonly code: HarnessErrorCode;
|
|
19
19
|
constructor(message: string, code?: HarnessErrorCode);
|
|
@@ -69,6 +69,11 @@ interface DockerRuntimeEvidence {
|
|
|
69
69
|
readonly cpus: string;
|
|
70
70
|
readonly pidsLimit: number;
|
|
71
71
|
}
|
|
72
|
+
declare const createConfiguredToolRuntime: ({ runtime, process, docker }: {
|
|
73
|
+
readonly runtime: RuntimeConfig;
|
|
74
|
+
readonly process: Parameters<typeof createProcessToolRuntime>[0];
|
|
75
|
+
readonly docker: Parameters<typeof createDockerToolRuntime>[0];
|
|
76
|
+
}) => ToolRuntime;
|
|
72
77
|
type ToolExecutionResult = {
|
|
73
78
|
readonly status: 'completed';
|
|
74
79
|
readonly resultHash: string;
|
|
@@ -106,6 +111,14 @@ declare const HARNESS_EVENT_SCHEMA_VERSION: 1;
|
|
|
106
111
|
declare const EVENT_LOG_GENESIS: "GENESIS";
|
|
107
112
|
declare const HARNESS_EVENT_TYPES: readonly ["run.created", "state.transitioned", "context.attached", "verification.completed", "approval.recorded", "authorization.recorded", "session.started", "session.resumed", "agent.turn.started", "policy.evaluated", "tool.approval.requested", "tool.approval.recorded", "tool.requested", "tool.execution.started", "tool.recovery.recorded", "tool.blocked", "tool.completed", "tool.failed", "session.ended"];
|
|
108
113
|
type HarnessEventType = typeof HARNESS_EVENT_TYPES[number];
|
|
114
|
+
interface HarnessEventContext {
|
|
115
|
+
readonly operationId: string;
|
|
116
|
+
readonly runId?: string;
|
|
117
|
+
readonly sessionId?: string;
|
|
118
|
+
readonly turnId?: string;
|
|
119
|
+
readonly actionId?: string;
|
|
120
|
+
readonly traceId?: string;
|
|
121
|
+
}
|
|
109
122
|
interface HarnessEventPayloads {
|
|
110
123
|
readonly 'run.created': {
|
|
111
124
|
readonly project: string;
|
|
@@ -238,6 +251,7 @@ type HarnessEvent<K extends HarnessEventType = HarnessEventType> = K extends Har
|
|
|
238
251
|
readonly at: string;
|
|
239
252
|
readonly sourceRevision: string;
|
|
240
253
|
readonly configHash: string;
|
|
254
|
+
readonly correlation?: HarnessEventContext;
|
|
241
255
|
readonly previousHash?: string;
|
|
242
256
|
readonly eventHash?: string;
|
|
243
257
|
readonly sessionId?: string;
|
|
@@ -382,7 +396,9 @@ interface VerificationCheck {
|
|
|
382
396
|
readonly category: CheckCategory;
|
|
383
397
|
readonly command: string;
|
|
384
398
|
readonly required: boolean;
|
|
399
|
+
readonly reason?: string;
|
|
385
400
|
readonly timeoutMs: number;
|
|
401
|
+
readonly dependsOn?: readonly string[];
|
|
386
402
|
readonly execution?: 'real';
|
|
387
403
|
readonly capabilities?: readonly string[];
|
|
388
404
|
readonly evidence: 'structured';
|
|
@@ -397,16 +413,25 @@ interface BenchmarkBinding {
|
|
|
397
413
|
readonly taskId: string;
|
|
398
414
|
readonly mode: 'harness';
|
|
399
415
|
}
|
|
416
|
+
interface RuntimeConfig {
|
|
417
|
+
readonly kind: 'process' | 'docker';
|
|
418
|
+
}
|
|
419
|
+
type AutonomyMode = 'controlled' | 'yolo';
|
|
400
420
|
interface VerificationConfig {
|
|
401
421
|
readonly schemaVersion: 1;
|
|
402
422
|
readonly project: string;
|
|
403
423
|
readonly root?: string;
|
|
404
424
|
readonly stateDir?: string;
|
|
405
425
|
readonly profile: string;
|
|
426
|
+
readonly autonomy: AutonomyMode;
|
|
427
|
+
readonly runtime: RuntimeConfig;
|
|
406
428
|
readonly contract: TaskContract;
|
|
407
429
|
readonly surfaces: Readonly<Record<SurfaceName, SurfaceRequirement>>;
|
|
408
430
|
readonly checks: readonly VerificationCheck[];
|
|
409
431
|
readonly tracking: TrackingConfig;
|
|
432
|
+
readonly verification?: {
|
|
433
|
+
readonly maxConcurrency?: number;
|
|
434
|
+
};
|
|
410
435
|
readonly budget?: {
|
|
411
436
|
readonly maxDurationMs?: number;
|
|
412
437
|
};
|
|
@@ -446,14 +471,14 @@ interface StructuredEvidence {
|
|
|
446
471
|
interface CheckResult {
|
|
447
472
|
readonly id: string;
|
|
448
473
|
readonly category: CheckCategory;
|
|
449
|
-
readonly status: 'pending' | 'passed' | 'failed';
|
|
474
|
+
readonly status: 'pending' | 'passed' | 'failed' | 'not-applicable';
|
|
450
475
|
readonly exitCode?: number;
|
|
451
476
|
readonly durationMs?: number;
|
|
452
477
|
readonly evidence?: StructuredEvidence;
|
|
453
478
|
readonly failures?: readonly string[];
|
|
454
479
|
}
|
|
455
480
|
interface RunOutcome extends ContractOutcome {
|
|
456
|
-
readonly status: 'pending' | 'passed' | 'failed';
|
|
481
|
+
readonly status: 'pending' | 'passed' | 'failed' | 'not-applicable';
|
|
457
482
|
}
|
|
458
483
|
interface EvidenceReference {
|
|
459
484
|
readonly checkId: string;
|
|
@@ -471,18 +496,11 @@ interface VerificationRun {
|
|
|
471
496
|
readonly sourceRevision: string;
|
|
472
497
|
readonly sourceStatusHash: string;
|
|
473
498
|
readonly baseline: SourceSnapshot;
|
|
474
|
-
|
|
475
|
-
readonly contractApproval?: {
|
|
499
|
+
readonly contractApproval: {
|
|
476
500
|
readonly actor: 'human';
|
|
477
501
|
readonly at: string;
|
|
478
502
|
readonly contractHash: string;
|
|
479
503
|
};
|
|
480
|
-
/** An automated preparation is never an approval and cannot complete a run. */
|
|
481
|
-
readonly contractPreparation?: {
|
|
482
|
-
readonly actor: 'ci';
|
|
483
|
-
readonly at: string;
|
|
484
|
-
readonly contractHash: string;
|
|
485
|
-
};
|
|
486
504
|
readonly checks: readonly CheckResult[];
|
|
487
505
|
readonly outcomes: readonly RunOutcome[];
|
|
488
506
|
readonly transitions: readonly StateTransition[];
|
|
@@ -493,9 +511,13 @@ interface VerificationRun {
|
|
|
493
511
|
readonly benchmark?: BenchmarkBinding;
|
|
494
512
|
readonly supersedes?: string;
|
|
495
513
|
readonly dirtyBaselineAuthorized?: boolean;
|
|
514
|
+
readonly autonomy: AutonomyMode;
|
|
496
515
|
readonly metrics?: {
|
|
497
516
|
readonly totalDurationMs: number;
|
|
517
|
+
readonly wallDurationMs?: number;
|
|
518
|
+
readonly peakConcurrency?: number;
|
|
498
519
|
readonly budgetExceeded: boolean;
|
|
520
|
+
readonly machine?: MachineMetrics;
|
|
499
521
|
};
|
|
500
522
|
readonly humanApproval?: {
|
|
501
523
|
readonly actor: 'human';
|
|
@@ -513,6 +535,27 @@ interface VerificationRun {
|
|
|
513
535
|
readonly verificationDigest?: string;
|
|
514
536
|
};
|
|
515
537
|
}
|
|
538
|
+
interface MachineSample {
|
|
539
|
+
readonly at: string;
|
|
540
|
+
readonly cpus: number;
|
|
541
|
+
readonly load1: number;
|
|
542
|
+
readonly load1PerCpuPercent: number;
|
|
543
|
+
readonly memoryUsedPercent: number;
|
|
544
|
+
readonly rssBytes: number;
|
|
545
|
+
readonly swapUsedPercent?: number;
|
|
546
|
+
readonly memoryPressure?: 'normal' | 'warning' | 'critical';
|
|
547
|
+
readonly interferingProcesses?: readonly string[];
|
|
548
|
+
}
|
|
549
|
+
interface MachineMetrics {
|
|
550
|
+
readonly sampleIntervalMs: number;
|
|
551
|
+
readonly samples: readonly MachineSample[];
|
|
552
|
+
readonly peakLoad1PerCpuPercent: number;
|
|
553
|
+
readonly peakMemoryUsedPercent: number;
|
|
554
|
+
readonly peakRssBytes: number;
|
|
555
|
+
readonly pressureEvents: number;
|
|
556
|
+
readonly throttleEvents: number;
|
|
557
|
+
readonly minimumEffectiveConcurrency: number;
|
|
558
|
+
}
|
|
516
559
|
interface RunReconciliation {
|
|
517
560
|
readonly status: 'verified';
|
|
518
561
|
readonly runId: string;
|
|
@@ -584,54 +627,467 @@ interface DocBridgeContextProviderOptions {
|
|
|
584
627
|
}
|
|
585
628
|
declare const createDocBridgeContextProvider: ({ root, indexPath }: DocBridgeContextProviderOptions) => ContextProvider;
|
|
586
629
|
|
|
630
|
+
interface DiscoveryOption {
|
|
631
|
+
readonly id: string;
|
|
632
|
+
readonly summary: string;
|
|
633
|
+
readonly impact: string;
|
|
634
|
+
}
|
|
635
|
+
interface DiscoveryAmbiguity {
|
|
636
|
+
readonly id: string;
|
|
637
|
+
readonly question: string;
|
|
638
|
+
readonly material: boolean;
|
|
639
|
+
readonly options: readonly DiscoveryOption[];
|
|
640
|
+
readonly recommendedOptionId: string;
|
|
641
|
+
readonly assumptionId?: string;
|
|
642
|
+
}
|
|
643
|
+
interface ApprovedAssumption {
|
|
644
|
+
readonly id: string;
|
|
645
|
+
readonly policyId: string;
|
|
646
|
+
readonly resolution: string;
|
|
647
|
+
}
|
|
648
|
+
interface DiscoveryInput {
|
|
649
|
+
readonly issueId: string;
|
|
650
|
+
readonly sourceRevision: string;
|
|
651
|
+
readonly contractHash: string;
|
|
652
|
+
readonly contextHash?: string;
|
|
653
|
+
readonly ambiguities: readonly DiscoveryAmbiguity[];
|
|
654
|
+
readonly approvedAssumptions?: readonly ApprovedAssumption[];
|
|
655
|
+
}
|
|
656
|
+
interface DecisionPacket {
|
|
657
|
+
readonly issueId: string;
|
|
658
|
+
readonly contractHash: string;
|
|
659
|
+
readonly sourceRevision: string;
|
|
660
|
+
readonly contextHash?: string;
|
|
661
|
+
readonly decisions: readonly {
|
|
662
|
+
readonly id: string;
|
|
663
|
+
readonly question: string;
|
|
664
|
+
readonly options: readonly DiscoveryOption[];
|
|
665
|
+
readonly recommendedOptionId: string;
|
|
666
|
+
}[];
|
|
667
|
+
}
|
|
668
|
+
interface DiscoveryDecisionLogEntry {
|
|
669
|
+
readonly ambiguityId: string;
|
|
670
|
+
readonly kind: 'human-decision-required' | 'approved-assumption';
|
|
671
|
+
readonly detail: string;
|
|
672
|
+
readonly policyId?: string;
|
|
673
|
+
}
|
|
674
|
+
interface DiscoveryResult {
|
|
675
|
+
readonly version: 1;
|
|
676
|
+
readonly issueId: string;
|
|
677
|
+
readonly sourceRevision: string;
|
|
678
|
+
readonly contractHash: string;
|
|
679
|
+
readonly contextHash?: string;
|
|
680
|
+
readonly status: 'ready' | 'awaiting-decision';
|
|
681
|
+
readonly packet?: DecisionPacket;
|
|
682
|
+
readonly decisionLog: readonly DiscoveryDecisionLogEntry[];
|
|
683
|
+
readonly digest: string;
|
|
684
|
+
}
|
|
685
|
+
interface DiscoveryCurrentInput {
|
|
686
|
+
readonly sourceRevision: string;
|
|
687
|
+
readonly contractHash: string;
|
|
688
|
+
readonly contextHash?: string;
|
|
689
|
+
}
|
|
690
|
+
interface DiscoveryCurrentResult {
|
|
691
|
+
readonly current: boolean;
|
|
692
|
+
readonly reasons: readonly ('source' | 'contract' | 'context')[];
|
|
693
|
+
}
|
|
694
|
+
declare const assessDiscovery: (input: DiscoveryInput) => DiscoveryResult;
|
|
695
|
+
declare const isDiscoveryCurrent: (result: DiscoveryResult, current: DiscoveryCurrentInput) => DiscoveryCurrentResult;
|
|
696
|
+
|
|
697
|
+
declare const WIP_STATES: readonly ["ready", "implementing", "blocked", "awaiting-decision", "awaiting-acceptance", "done", "cancelled"];
|
|
698
|
+
type WipState = typeof WIP_STATES[number];
|
|
699
|
+
interface WipEntry {
|
|
700
|
+
readonly issueId: string;
|
|
701
|
+
readonly state: WipState;
|
|
702
|
+
}
|
|
703
|
+
interface WipAssessmentInput {
|
|
704
|
+
readonly entries: readonly WipEntry[];
|
|
705
|
+
readonly candidate: {
|
|
706
|
+
readonly issueId: string;
|
|
707
|
+
readonly kind: 'new' | 'resume';
|
|
708
|
+
};
|
|
709
|
+
readonly maxInFlight?: number;
|
|
710
|
+
}
|
|
711
|
+
interface WipAssessment {
|
|
712
|
+
readonly decision: 'admit' | 'hold';
|
|
713
|
+
readonly inFlight: readonly WipEntry[];
|
|
714
|
+
readonly counts: Readonly<Record<WipState, number>>;
|
|
715
|
+
readonly reason: string;
|
|
716
|
+
}
|
|
717
|
+
declare const assessWip: ({ entries, candidate, maxInFlight }: WipAssessmentInput) => WipAssessment;
|
|
718
|
+
|
|
719
|
+
interface RuntimeExperimentCandidate {
|
|
720
|
+
readonly runtime: string;
|
|
721
|
+
readonly sourceRevision: string;
|
|
722
|
+
readonly contractHash: string;
|
|
723
|
+
readonly provider: string;
|
|
724
|
+
readonly model: string;
|
|
725
|
+
readonly configurationHash: string;
|
|
726
|
+
readonly hardGatesPassed: boolean;
|
|
727
|
+
readonly humanMinutes: number;
|
|
728
|
+
readonly durationMs: number;
|
|
729
|
+
readonly cost: number;
|
|
730
|
+
}
|
|
731
|
+
interface RuntimeExperimentResult {
|
|
732
|
+
readonly decision: 'selected' | 'blocked';
|
|
733
|
+
readonly selected?: RuntimeExperimentCandidate;
|
|
734
|
+
readonly eligible: readonly RuntimeExperimentCandidate[];
|
|
735
|
+
readonly reason: string;
|
|
736
|
+
}
|
|
737
|
+
declare const selectRuntime: (candidates: readonly RuntimeExperimentCandidate[]) => RuntimeExperimentResult;
|
|
738
|
+
|
|
739
|
+
type CriterionStatus = 'passed' | 'failed' | 'pending' | 'not-applicable';
|
|
740
|
+
interface GateCriterion {
|
|
741
|
+
readonly id: string;
|
|
742
|
+
readonly gate: 'G2' | 'G3' | 'G4' | 'G5';
|
|
743
|
+
readonly status: CriterionStatus;
|
|
744
|
+
readonly reason?: string;
|
|
745
|
+
}
|
|
746
|
+
interface GateBinding {
|
|
747
|
+
readonly candidateRevision: string;
|
|
748
|
+
readonly contractHash: string;
|
|
749
|
+
readonly configHash: string;
|
|
750
|
+
}
|
|
751
|
+
interface GateAssessment {
|
|
752
|
+
readonly gate: 'G2' | 'G3' | 'G4' | 'G5';
|
|
753
|
+
readonly decision: 'approved' | 'blocked' | 'awaiting-acceptance';
|
|
754
|
+
readonly reasons: readonly string[];
|
|
755
|
+
readonly binding: GateBinding;
|
|
756
|
+
readonly digest: string;
|
|
757
|
+
}
|
|
758
|
+
declare const assessPreflight: ({ criteria, repairAttempts, implementerId, reviewerId, reviewKind, reviewApproved, binding: current }: {
|
|
759
|
+
readonly criteria: readonly GateCriterion[];
|
|
760
|
+
readonly repairAttempts?: number;
|
|
761
|
+
readonly implementerId: string;
|
|
762
|
+
readonly reviewerId?: string;
|
|
763
|
+
readonly reviewKind?: "adversarial";
|
|
764
|
+
readonly reviewApproved: boolean;
|
|
765
|
+
readonly binding: GateBinding;
|
|
766
|
+
}) => GateAssessment;
|
|
767
|
+
interface PullRequestDraft {
|
|
768
|
+
readonly issueId: string;
|
|
769
|
+
readonly candidateRevision: string;
|
|
770
|
+
readonly contractHash: string;
|
|
771
|
+
readonly configHash: string;
|
|
772
|
+
readonly g2Digest: string;
|
|
773
|
+
readonly evidence: readonly string[];
|
|
774
|
+
readonly documentation: readonly string[];
|
|
775
|
+
readonly risk: string;
|
|
776
|
+
readonly rollback: string;
|
|
777
|
+
readonly pendingCriteria: readonly string[];
|
|
778
|
+
}
|
|
779
|
+
declare const composePullRequest: ({ draft, g2, remote }: {
|
|
780
|
+
readonly draft: PullRequestDraft;
|
|
781
|
+
readonly g2: GateAssessment;
|
|
782
|
+
readonly remote?: {
|
|
783
|
+
readonly state: "missing" | "confirmed" | "uncertain";
|
|
784
|
+
readonly url?: string;
|
|
785
|
+
readonly candidateRevision?: string;
|
|
786
|
+
};
|
|
787
|
+
}) => {
|
|
788
|
+
readonly decision: "create" | "reuse" | "blocked";
|
|
789
|
+
readonly body?: string;
|
|
790
|
+
readonly reason: string;
|
|
791
|
+
readonly idempotencyKey: string;
|
|
792
|
+
};
|
|
793
|
+
declare const assessIntegration: ({ g2, candidateRevision, evidenceRevision, contractHash, configHash, ci }: {
|
|
794
|
+
readonly g2: GateAssessment;
|
|
795
|
+
readonly candidateRevision: string;
|
|
796
|
+
readonly evidenceRevision: string;
|
|
797
|
+
readonly contractHash: string;
|
|
798
|
+
readonly configHash: string;
|
|
799
|
+
readonly ci: CriterionStatus;
|
|
800
|
+
}) => GateAssessment;
|
|
801
|
+
declare const assessWorktreeCleanup: ({ branch, candidateRevision, contractHash, configHash, remoteBranchRevision, remotePr, integration }: {
|
|
802
|
+
readonly branch: string;
|
|
803
|
+
readonly candidateRevision: string;
|
|
804
|
+
readonly contractHash: string;
|
|
805
|
+
readonly configHash: string;
|
|
806
|
+
readonly remoteBranchRevision?: string;
|
|
807
|
+
readonly remotePr: "confirmed" | "missing" | "uncertain";
|
|
808
|
+
readonly integration: GateAssessment;
|
|
809
|
+
}) => {
|
|
810
|
+
readonly decision: "clean" | "preserve";
|
|
811
|
+
readonly reason: string;
|
|
812
|
+
};
|
|
813
|
+
interface RepositoryProfile {
|
|
814
|
+
readonly deploy: string;
|
|
815
|
+
readonly rollback: string;
|
|
816
|
+
readonly urls: readonly string[];
|
|
817
|
+
readonly featureFlag: string;
|
|
818
|
+
readonly syntheticTenant: string;
|
|
819
|
+
readonly observability: readonly string[];
|
|
820
|
+
readonly sensitivePaths: readonly string[];
|
|
821
|
+
readonly owners: readonly string[];
|
|
822
|
+
readonly approvedBy: string;
|
|
823
|
+
}
|
|
824
|
+
interface ProductionEvidence {
|
|
825
|
+
readonly tenant: string;
|
|
826
|
+
readonly realFlow: string;
|
|
827
|
+
readonly logs: readonly string[];
|
|
828
|
+
readonly metrics: readonly string[];
|
|
829
|
+
}
|
|
830
|
+
declare const assessProduction: ({ profile, integration, artifact, isolated, acceptanceArtifact, lowRisk, observationMinutes, technicalPassed, evidence, containmentPreauthorized, containmentAction, linkedDefect }: {
|
|
831
|
+
readonly profile: RepositoryProfile;
|
|
832
|
+
readonly integration: GateAssessment;
|
|
833
|
+
readonly artifact: string;
|
|
834
|
+
readonly isolated: boolean;
|
|
835
|
+
readonly acceptanceArtifact?: string;
|
|
836
|
+
readonly lowRisk?: boolean;
|
|
837
|
+
readonly observationMinutes: number;
|
|
838
|
+
readonly technicalPassed: boolean;
|
|
839
|
+
readonly evidence: ProductionEvidence;
|
|
840
|
+
readonly containmentPreauthorized: boolean;
|
|
841
|
+
readonly containmentAction?: string;
|
|
842
|
+
readonly linkedDefect?: string;
|
|
843
|
+
}) => GateAssessment;
|
|
844
|
+
declare const assessAcceptance: ({ production, acceptanceRequired, accepted, notApplicableReason, materialChange }: {
|
|
845
|
+
readonly production: GateAssessment;
|
|
846
|
+
readonly acceptanceRequired: boolean;
|
|
847
|
+
readonly accepted: boolean;
|
|
848
|
+
readonly notApplicableReason?: string;
|
|
849
|
+
readonly materialChange: boolean;
|
|
850
|
+
}) => GateAssessment;
|
|
851
|
+
|
|
852
|
+
interface PilotEntry {
|
|
853
|
+
readonly issueId: string;
|
|
854
|
+
readonly classification: 'normal' | 'incident' | 'sensitive';
|
|
855
|
+
readonly status: 'included' | 'excluded' | 'aborted';
|
|
856
|
+
readonly reason?: string;
|
|
857
|
+
}
|
|
858
|
+
interface PilotManifest {
|
|
859
|
+
readonly policyHash: string;
|
|
860
|
+
readonly baselineReference: string;
|
|
861
|
+
readonly entries: readonly PilotEntry[];
|
|
862
|
+
}
|
|
863
|
+
interface PilotAssessment {
|
|
864
|
+
readonly decision: 'ready' | 'blocked';
|
|
865
|
+
readonly included: readonly string[];
|
|
866
|
+
readonly reasons: readonly string[];
|
|
867
|
+
readonly digest: string;
|
|
868
|
+
}
|
|
869
|
+
declare const assessPilot: (manifest: PilotManifest) => PilotAssessment;
|
|
870
|
+
|
|
871
|
+
declare const IMPROVEMENT_CYCLE_STEPS: readonly ["adversarial-review", "g2-preflight", "baseline-record", "pilot-execution", "comparison"];
|
|
872
|
+
type ImprovementCycleStep = typeof IMPROVEMENT_CYCLE_STEPS[number];
|
|
873
|
+
type CycleStepStatus = 'passed' | 'failed' | 'blocked' | 'pending';
|
|
874
|
+
interface CycleStepResult {
|
|
875
|
+
readonly step: ImprovementCycleStep;
|
|
876
|
+
readonly status: CycleStepStatus;
|
|
877
|
+
readonly reason?: string;
|
|
878
|
+
}
|
|
879
|
+
interface CycleIterationMetrics {
|
|
880
|
+
readonly durationMs?: number;
|
|
881
|
+
readonly humanMinutes?: number;
|
|
882
|
+
readonly attempts?: number;
|
|
883
|
+
readonly escapedIncomplete?: number;
|
|
884
|
+
}
|
|
885
|
+
interface ImprovementCycleIteration {
|
|
886
|
+
readonly iteration: number;
|
|
887
|
+
readonly steps: readonly CycleStepResult[];
|
|
888
|
+
readonly adjustment?: string;
|
|
889
|
+
readonly metrics?: CycleIterationMetrics;
|
|
890
|
+
}
|
|
891
|
+
interface ImprovementCycleInput {
|
|
892
|
+
readonly cycleId: string;
|
|
893
|
+
readonly maxIterations: number;
|
|
894
|
+
readonly iterations: readonly ImprovementCycleIteration[];
|
|
895
|
+
}
|
|
896
|
+
interface CycleMatrixRow {
|
|
897
|
+
readonly iteration: number;
|
|
898
|
+
readonly passedSteps: number;
|
|
899
|
+
readonly totalSteps: number;
|
|
900
|
+
readonly passRate: number;
|
|
901
|
+
readonly statuses: Readonly<Record<ImprovementCycleStep, CycleStepStatus>>;
|
|
902
|
+
readonly adjustment?: string;
|
|
903
|
+
readonly metrics?: CycleIterationMetrics;
|
|
904
|
+
}
|
|
905
|
+
interface ImprovementCycleAssessment {
|
|
906
|
+
readonly type: 'agentskit-harness-improvement-cycle';
|
|
907
|
+
readonly cycleId: string;
|
|
908
|
+
readonly decision: 'complete' | 'repeat' | 'blocked';
|
|
909
|
+
readonly nextIteration?: number;
|
|
910
|
+
readonly reasons: readonly string[];
|
|
911
|
+
readonly matrix: readonly CycleMatrixRow[];
|
|
912
|
+
readonly digest: string;
|
|
913
|
+
}
|
|
914
|
+
declare const assessImprovementCycle: (input: ImprovementCycleInput) => ImprovementCycleAssessment;
|
|
915
|
+
|
|
916
|
+
type EvalExpectation = string | ((output: string) => boolean);
|
|
917
|
+
interface AgentEvalCase {
|
|
918
|
+
readonly id: string;
|
|
919
|
+
readonly input: string;
|
|
920
|
+
readonly expected: EvalExpectation;
|
|
921
|
+
}
|
|
922
|
+
interface AgentEvalSuite {
|
|
923
|
+
readonly name: string;
|
|
924
|
+
readonly cases: readonly AgentEvalCase[];
|
|
925
|
+
}
|
|
926
|
+
interface AgentEvalReport {
|
|
927
|
+
readonly suite: string;
|
|
928
|
+
readonly total: number;
|
|
929
|
+
readonly passed: number;
|
|
930
|
+
readonly failed: number;
|
|
931
|
+
readonly accuracy: number;
|
|
932
|
+
readonly failures: readonly string[];
|
|
933
|
+
}
|
|
934
|
+
declare const runAgentEval: ({ suite, agent, concurrency }: {
|
|
935
|
+
readonly suite: AgentEvalSuite;
|
|
936
|
+
readonly agent: (input: string) => Promise<string>;
|
|
937
|
+
readonly concurrency?: number;
|
|
938
|
+
}) => Promise<AgentEvalReport>;
|
|
939
|
+
declare const assessAgentEval: (report: AgentEvalReport, minimumAccuracy: number) => {
|
|
940
|
+
readonly status: "passed" | "blocked";
|
|
941
|
+
readonly reason: string;
|
|
942
|
+
readonly report: AgentEvalReport;
|
|
943
|
+
};
|
|
944
|
+
|
|
945
|
+
interface LlmCacheKeyInput {
|
|
946
|
+
readonly sourceRevision: string;
|
|
947
|
+
readonly contractHash: string;
|
|
948
|
+
readonly configHash: string;
|
|
949
|
+
readonly provider: string;
|
|
950
|
+
readonly model: string;
|
|
951
|
+
readonly systemPromptHash: string;
|
|
952
|
+
readonly inputHash: string;
|
|
953
|
+
readonly contextHash?: string;
|
|
954
|
+
readonly toolSchemaHash?: string;
|
|
955
|
+
readonly operation: 'context' | 'read-only';
|
|
956
|
+
}
|
|
957
|
+
interface LlmCacheStats {
|
|
958
|
+
readonly hits: number;
|
|
959
|
+
readonly misses: number;
|
|
960
|
+
readonly invalidations: number;
|
|
961
|
+
}
|
|
962
|
+
declare const validateCacheableOperation: (operation: unknown) => "context" | "read-only";
|
|
963
|
+
declare const createLlmCacheKey: (input: LlmCacheKeyInput) => string;
|
|
964
|
+
interface LlmCache<T> {
|
|
965
|
+
getOrCompute(key: string, compute: () => Promise<T>): Promise<T>;
|
|
966
|
+
invalidate(key?: string): void;
|
|
967
|
+
stats(): LlmCacheStats;
|
|
968
|
+
}
|
|
969
|
+
declare const createLlmCache: <T>() => LlmCache<T>;
|
|
970
|
+
|
|
971
|
+
interface TokenUsage {
|
|
972
|
+
readonly inputTokens: number;
|
|
973
|
+
readonly outputTokens: number;
|
|
974
|
+
readonly totalTokens: number;
|
|
975
|
+
readonly cacheReadTokens?: number;
|
|
976
|
+
readonly cacheWriteTokens?: number;
|
|
977
|
+
}
|
|
978
|
+
interface MemoryUsage {
|
|
979
|
+
readonly reads: number;
|
|
980
|
+
readonly writes: number;
|
|
981
|
+
readonly relevantHits: number;
|
|
982
|
+
readonly staleHits: number;
|
|
983
|
+
}
|
|
984
|
+
interface CacheUsage {
|
|
985
|
+
readonly hits: number;
|
|
986
|
+
readonly misses: number;
|
|
987
|
+
readonly invalidations: number;
|
|
988
|
+
readonly tokensSaved?: number;
|
|
989
|
+
}
|
|
990
|
+
interface ParallelismUsage {
|
|
991
|
+
readonly tasks: number;
|
|
992
|
+
readonly peakConcurrency: number;
|
|
993
|
+
readonly criticalPathMs: number;
|
|
994
|
+
readonly queueWaitMs?: number;
|
|
995
|
+
}
|
|
996
|
+
interface OptimizationObservation {
|
|
997
|
+
readonly sourceRevision: string;
|
|
998
|
+
readonly contractHash: string;
|
|
999
|
+
readonly configHash: string;
|
|
1000
|
+
readonly provider: string;
|
|
1001
|
+
readonly model: string;
|
|
1002
|
+
readonly durationMs: number;
|
|
1003
|
+
readonly accuracy?: number;
|
|
1004
|
+
readonly tokens?: TokenUsage;
|
|
1005
|
+
readonly memory?: MemoryUsage;
|
|
1006
|
+
readonly cache?: CacheUsage;
|
|
1007
|
+
readonly parallelism?: ParallelismUsage;
|
|
1008
|
+
}
|
|
1009
|
+
declare const validateOptimizationObservation: (observation: OptimizationObservation) => OptimizationObservation;
|
|
1010
|
+
interface OptimizationComparison {
|
|
1011
|
+
readonly comparable: boolean;
|
|
1012
|
+
readonly reason: string;
|
|
1013
|
+
readonly digest: string;
|
|
1014
|
+
readonly durationDeltaMs?: number;
|
|
1015
|
+
readonly tokenDelta?: number;
|
|
1016
|
+
readonly accuracyDelta?: number;
|
|
1017
|
+
readonly cacheHitRateDelta?: number;
|
|
1018
|
+
readonly memoryRelevantHitRateDelta?: number;
|
|
1019
|
+
readonly peakConcurrencyDelta?: number;
|
|
1020
|
+
}
|
|
1021
|
+
declare const compareOptimization: (baseline: OptimizationObservation, candidate: OptimizationObservation) => OptimizationComparison;
|
|
1022
|
+
|
|
1023
|
+
declare const MEMORY_SCOPES: readonly ["issue", "project", "global"];
|
|
1024
|
+
type MemoryScope = typeof MEMORY_SCOPES[number];
|
|
1025
|
+
interface AgentMemoryRecord {
|
|
1026
|
+
readonly id: string;
|
|
1027
|
+
readonly scope: MemoryScope;
|
|
1028
|
+
readonly summary: string;
|
|
1029
|
+
readonly source: string;
|
|
1030
|
+
readonly sourceRevision: string;
|
|
1031
|
+
readonly contentHash: string;
|
|
1032
|
+
readonly approved: true;
|
|
1033
|
+
}
|
|
1034
|
+
interface AgentMemoryHit {
|
|
1035
|
+
readonly record: AgentMemoryRecord;
|
|
1036
|
+
readonly relevant: boolean;
|
|
1037
|
+
readonly stale: boolean;
|
|
1038
|
+
}
|
|
1039
|
+
interface AgentMemoryAdapter {
|
|
1040
|
+
readonly id: string;
|
|
1041
|
+
readonly version: string;
|
|
1042
|
+
remember(record: AgentMemoryRecord): Promise<void>;
|
|
1043
|
+
recall(input: {
|
|
1044
|
+
readonly query: string;
|
|
1045
|
+
readonly issueId?: string;
|
|
1046
|
+
readonly project?: string;
|
|
1047
|
+
readonly sourceRevision?: string;
|
|
1048
|
+
}): Promise<readonly AgentMemoryHit[]>;
|
|
1049
|
+
}
|
|
1050
|
+
interface AgentMemoryKvStore {
|
|
1051
|
+
get(key: string): Promise<unknown>;
|
|
1052
|
+
set(key: string, value: unknown): Promise<void>;
|
|
1053
|
+
}
|
|
1054
|
+
declare const validateMemoryRecord: (record: AgentMemoryRecord) => AgentMemoryRecord;
|
|
1055
|
+
/** Minimal deterministic adapter for replay/tests; production uses @agentskit/memory through the same seam. */
|
|
1056
|
+
declare const createInMemoryMemoryAdapter: (options?: {
|
|
1057
|
+
readonly id?: string;
|
|
1058
|
+
readonly version?: string;
|
|
1059
|
+
}) => AgentMemoryAdapter;
|
|
1060
|
+
/** Bridges AgentsKit's KV memory stores without making the Harness depend on a backend. */
|
|
1061
|
+
declare const createKvMemoryAdapter: (store: AgentMemoryKvStore, options?: {
|
|
1062
|
+
readonly id?: string;
|
|
1063
|
+
readonly version?: string;
|
|
1064
|
+
}) => AgentMemoryAdapter;
|
|
1065
|
+
|
|
1066
|
+
interface WorkflowNode<T> {
|
|
1067
|
+
readonly id: string;
|
|
1068
|
+
readonly dependsOn?: readonly string[];
|
|
1069
|
+
/** Nodes sharing a mutation key are serialized even when otherwise independent. */
|
|
1070
|
+
readonly mutationKey?: string;
|
|
1071
|
+
readonly run: () => Promise<T>;
|
|
1072
|
+
}
|
|
1073
|
+
interface WorkflowResult<T> {
|
|
1074
|
+
readonly results: Readonly<Record<string, T>>;
|
|
1075
|
+
readonly order: readonly string[];
|
|
1076
|
+
readonly peakConcurrency: number;
|
|
1077
|
+
readonly criticalPathMs: number;
|
|
1078
|
+
}
|
|
1079
|
+
declare const runWorkflow: <T>(nodes: readonly WorkflowNode<T>[], options: {
|
|
1080
|
+
readonly maxConcurrency: number;
|
|
1081
|
+
readonly currentConcurrency?: () => number;
|
|
1082
|
+
}) => Promise<WorkflowResult<T>>;
|
|
1083
|
+
|
|
587
1084
|
declare const BENCHMARK_SCHEMA_VERSION: 1;
|
|
588
1085
|
type BenchmarkObservationStatus = 'passed' | 'failed' | 'blocked' | 'not-run';
|
|
589
1086
|
type BenchmarkImprovementDirection = 'improved' | 'regressed' | 'unchanged' | 'unavailable';
|
|
590
|
-
type BenchmarkConfidence = 'insufficient' | 'directional' | 'reliable';
|
|
591
|
-
interface BenchmarkPolicy {
|
|
592
|
-
readonly minComparableTasks: number;
|
|
593
|
-
readonly maxDurationRegressionRate: number;
|
|
594
|
-
readonly minCompletedRunsPerTask: number;
|
|
595
|
-
readonly minBaselineSamplesPerTask: number;
|
|
596
|
-
readonly requireZeroEscapedIncomplete: boolean;
|
|
597
|
-
}
|
|
598
|
-
interface BenchmarkQualityGate {
|
|
599
|
-
readonly status: 'passed' | 'failed' | 'insufficient-data';
|
|
600
|
-
readonly confidence: BenchmarkConfidence;
|
|
601
|
-
readonly comparableTaskCount: number;
|
|
602
|
-
readonly policy: BenchmarkPolicy;
|
|
603
|
-
readonly durationRegressionTaskIds: readonly string[];
|
|
604
|
-
readonly escapedIncompleteTaskIds: readonly string[];
|
|
605
|
-
readonly reasons: readonly string[];
|
|
606
|
-
}
|
|
607
1087
|
interface BenchmarkTask {
|
|
608
1088
|
readonly id: string;
|
|
609
1089
|
readonly title: string;
|
|
610
1090
|
readonly acceptanceCriteria: readonly string[];
|
|
611
|
-
/** Product/runtime surfaces exercised by the task. */
|
|
612
|
-
readonly surfaces?: readonly SurfaceName[];
|
|
613
|
-
readonly kind?: string;
|
|
614
|
-
readonly prompt?: BenchmarkTaskFile;
|
|
615
|
-
readonly source?: BenchmarkTaskSource;
|
|
616
|
-
readonly scope?: BenchmarkTaskScope;
|
|
617
|
-
}
|
|
618
|
-
interface BenchmarkTaskFile {
|
|
619
|
-
readonly path: string;
|
|
620
|
-
readonly sha256: string;
|
|
621
|
-
}
|
|
622
|
-
interface BenchmarkTaskSource {
|
|
623
|
-
readonly repository: string;
|
|
624
|
-
readonly path: string;
|
|
625
|
-
readonly revision: string;
|
|
626
|
-
}
|
|
627
|
-
interface BenchmarkSuiteSource {
|
|
628
|
-
readonly repository: string;
|
|
629
|
-
readonly revision: string;
|
|
630
|
-
readonly taskDefinition: string;
|
|
631
|
-
}
|
|
632
|
-
interface BenchmarkTaskScope {
|
|
633
|
-
readonly read: readonly string[];
|
|
634
|
-
readonly write: readonly string[];
|
|
635
1091
|
}
|
|
636
1092
|
interface BenchmarkObservation {
|
|
637
1093
|
readonly taskId: string;
|
|
@@ -641,11 +1097,6 @@ interface BenchmarkObservation {
|
|
|
641
1097
|
readonly recordedAt: string;
|
|
642
1098
|
readonly attempts?: number;
|
|
643
1099
|
readonly durationMs?: number;
|
|
644
|
-
readonly durationSamplesMs?: readonly number[];
|
|
645
|
-
/** Fraction of repeated samples whose task artifact passed acceptance validation. */
|
|
646
|
-
readonly artifactAcceptanceRate?: number;
|
|
647
|
-
/** Fraction of repeated samples whose verification protocol completed. */
|
|
648
|
-
readonly protocolCompletionRate?: number;
|
|
649
1100
|
readonly reviewMinutes?: number;
|
|
650
1101
|
readonly escapedIncomplete?: number;
|
|
651
1102
|
readonly evidence?: readonly BenchmarkObservationEvidence[];
|
|
@@ -661,10 +1112,8 @@ interface BenchmarkManifest {
|
|
|
661
1112
|
readonly schemaVersion: typeof BENCHMARK_SCHEMA_VERSION;
|
|
662
1113
|
readonly suiteId: string;
|
|
663
1114
|
readonly name: string;
|
|
664
|
-
readonly provenance?: BenchmarkSuiteSource;
|
|
665
1115
|
readonly tasks: readonly BenchmarkTask[];
|
|
666
1116
|
readonly observations: readonly BenchmarkObservation[];
|
|
667
|
-
readonly policy?: BenchmarkPolicy;
|
|
668
1117
|
}
|
|
669
1118
|
interface BenchmarkRun {
|
|
670
1119
|
readonly runId: string;
|
|
@@ -688,26 +1137,20 @@ interface BenchmarkRun {
|
|
|
688
1137
|
readonly total: number;
|
|
689
1138
|
readonly attached: number;
|
|
690
1139
|
};
|
|
691
|
-
/** Optional artifact outcome emitted by structured benchmark evidence. */
|
|
692
|
-
readonly artifactAcceptanceRate?: number;
|
|
693
1140
|
readonly escapedIncomplete?: number;
|
|
694
1141
|
readonly humanApproved: boolean;
|
|
695
1142
|
readonly humanReviewMinutes?: number;
|
|
696
1143
|
readonly authorized: boolean;
|
|
1144
|
+
readonly machine?: MachineMetrics;
|
|
697
1145
|
readonly benchmark?: BenchmarkBinding;
|
|
698
1146
|
}
|
|
699
1147
|
interface BenchmarkComparison {
|
|
700
1148
|
readonly taskId: string;
|
|
701
1149
|
readonly title: string;
|
|
702
|
-
readonly comparability: 'comparable' | 'missing-baseline' | 'baseline-not-run' | 'baseline-evidence-missing' | '
|
|
1150
|
+
readonly comparability: 'comparable' | 'missing-baseline' | 'baseline-not-run' | 'baseline-evidence-missing' | 'harness-not-run' | 'harness-not-complete';
|
|
703
1151
|
readonly comparable: boolean;
|
|
704
|
-
readonly baselineDeliveryComplete: boolean;
|
|
705
1152
|
readonly baseline?: BenchmarkObservation;
|
|
706
1153
|
readonly baselineEvidenceCoverageRate: number | null;
|
|
707
|
-
readonly baselineSampleCount: number;
|
|
708
|
-
readonly baselineArtifactAcceptanceRate: number | null;
|
|
709
|
-
readonly baselineProtocolCompletionRate: number | null;
|
|
710
|
-
readonly baselineMedianDurationMs?: number;
|
|
711
1154
|
readonly improvement: {
|
|
712
1155
|
readonly durationRate: number | null;
|
|
713
1156
|
readonly duration: BenchmarkImprovementDirection;
|
|
@@ -715,36 +1158,21 @@ interface BenchmarkComparison {
|
|
|
715
1158
|
readonly attempts: BenchmarkImprovementDirection;
|
|
716
1159
|
readonly reviewRate: number | null;
|
|
717
1160
|
readonly review: BenchmarkImprovementDirection;
|
|
718
|
-
readonly artifactAcceptanceRate: number | null;
|
|
719
|
-
readonly artifactAcceptance: BenchmarkImprovementDirection;
|
|
720
|
-
readonly artifactAcceptanceDelta: number | null;
|
|
721
|
-
readonly protocolCompletionRate: number | null;
|
|
722
|
-
readonly protocolCompletion: BenchmarkImprovementDirection;
|
|
723
|
-
readonly protocolCompletionDelta: number | null;
|
|
724
1161
|
readonly escapedIncompleteRate: number | null;
|
|
725
1162
|
readonly escapedIncomplete: BenchmarkImprovementDirection;
|
|
726
1163
|
};
|
|
727
1164
|
readonly harness: {
|
|
728
1165
|
readonly attempts: number;
|
|
729
|
-
readonly retryCount: number;
|
|
730
|
-
readonly completedRuns: number;
|
|
731
|
-
readonly durationSamplesMs: readonly number[];
|
|
732
|
-
readonly medianDurationMs?: number;
|
|
733
1166
|
readonly latestState: RunState | 'NOT_RUN';
|
|
734
1167
|
readonly latestRunId?: string;
|
|
735
1168
|
readonly latestDurationMs?: number;
|
|
736
1169
|
readonly checkPassRate: number | null;
|
|
737
1170
|
readonly outcomePassRate: number | null;
|
|
738
1171
|
readonly evidenceCoverageRate: number | null;
|
|
739
|
-
readonly artifactAcceptanceRate?: number;
|
|
740
|
-
readonly artifactAcceptanceSampleCount: number;
|
|
741
|
-
readonly protocolCompletionRate: number | null;
|
|
742
|
-
readonly protocolCompletionSampleCount: number;
|
|
743
1172
|
readonly humanApproved: boolean;
|
|
744
1173
|
readonly escapedIncomplete?: number;
|
|
745
1174
|
readonly humanReviewMinutes?: number;
|
|
746
1175
|
};
|
|
747
|
-
readonly confidence: BenchmarkConfidence;
|
|
748
1176
|
readonly durationDeltaMs?: number;
|
|
749
1177
|
readonly attemptDelta?: number;
|
|
750
1178
|
readonly reviewDeltaMinutes?: number;
|
|
@@ -759,12 +1187,6 @@ interface BenchmarkSummary {
|
|
|
759
1187
|
readonly firstAttemptRuns: number;
|
|
760
1188
|
readonly humanApprovedRuns: number;
|
|
761
1189
|
readonly authorizedRuns: number;
|
|
762
|
-
readonly effectiveRunCount: number;
|
|
763
|
-
readonly effectiveCompleteRuns: number;
|
|
764
|
-
readonly effectiveCompletionRate: number | null;
|
|
765
|
-
readonly effectiveCheckPassRate: number | null;
|
|
766
|
-
readonly effectiveOutcomePassRate: number | null;
|
|
767
|
-
readonly effectiveEvidenceCoverageRate: number | null;
|
|
768
1190
|
readonly checkPassRate: number | null;
|
|
769
1191
|
readonly outcomePassRate: number | null;
|
|
770
1192
|
readonly evidenceCoverageRate: number | null;
|
|
@@ -788,7 +1210,6 @@ interface BenchmarkReport {
|
|
|
788
1210
|
readonly comparableTaskCount: number;
|
|
789
1211
|
};
|
|
790
1212
|
readonly comparisons: readonly BenchmarkComparison[];
|
|
791
|
-
readonly qualityGate: BenchmarkQualityGate;
|
|
792
1213
|
}
|
|
793
1214
|
interface BenchmarkObservationInput {
|
|
794
1215
|
readonly taskId: string;
|
|
@@ -797,9 +1218,6 @@ interface BenchmarkObservationInput {
|
|
|
797
1218
|
readonly recordedAt?: string;
|
|
798
1219
|
readonly attempts?: number;
|
|
799
1220
|
readonly durationMs?: number;
|
|
800
|
-
readonly durationSamplesMs?: readonly number[];
|
|
801
|
-
readonly artifactAcceptanceRate?: number;
|
|
802
|
-
readonly protocolCompletionRate?: number;
|
|
803
1221
|
readonly reviewMinutes?: number;
|
|
804
1222
|
readonly escapedIncomplete?: number;
|
|
805
1223
|
readonly evidence?: readonly BenchmarkObservationEvidence[];
|
|
@@ -810,33 +1228,6 @@ declare const loadBenchmarkManifest: (path: string) => BenchmarkManifest;
|
|
|
810
1228
|
declare const recordBenchmarkObservation: (path: string, input: BenchmarkObservationInput) => BenchmarkManifest;
|
|
811
1229
|
declare const benchmarkRuns: (stateDir: string, manifest?: BenchmarkManifest) => BenchmarkReport;
|
|
812
1230
|
|
|
813
|
-
type ExternalCodingBenchmarkStatus = 'ok' | 'partial' | 'fail' | 'timeout';
|
|
814
|
-
interface ExternalCodingBenchmarkRow {
|
|
815
|
-
readonly providerId: string;
|
|
816
|
-
readonly status: ExternalCodingBenchmarkStatus;
|
|
817
|
-
readonly completenessScore: number;
|
|
818
|
-
readonly fileEditCount: number;
|
|
819
|
-
readonly summary: string;
|
|
820
|
-
readonly durationMs?: number;
|
|
821
|
-
readonly inputTokens?: number;
|
|
822
|
-
readonly outputTokens?: number;
|
|
823
|
-
readonly costUsd?: number;
|
|
824
|
-
readonly successPassed?: boolean;
|
|
825
|
-
}
|
|
826
|
-
interface ExternalCodingBenchmarkReport {
|
|
827
|
-
readonly kind: string;
|
|
828
|
-
readonly prompt: string;
|
|
829
|
-
readonly dryRun: boolean;
|
|
830
|
-
readonly isolateWorktrees: boolean;
|
|
831
|
-
readonly repoRoot: string;
|
|
832
|
-
readonly rows: readonly ExternalCodingBenchmarkRow[];
|
|
833
|
-
}
|
|
834
|
-
/**
|
|
835
|
-
* Validates the stable report shape emitted by AgentsKit OS coding benchmarks.
|
|
836
|
-
* Provider heuristics remain observations; this function never grants human acceptance.
|
|
837
|
-
*/
|
|
838
|
-
declare const validateExternalCodingBenchmarkReport: (value: unknown) => ExternalCodingBenchmarkReport;
|
|
839
|
-
|
|
840
1231
|
interface PolicyRule {
|
|
841
1232
|
readonly id: string;
|
|
842
1233
|
readonly effect: 'allow' | 'block' | 'approve';
|
|
@@ -915,6 +1306,233 @@ interface SessionRecorder {
|
|
|
915
1306
|
}
|
|
916
1307
|
declare const createSessionRecorder: ({ stateDir, run, adapter, policy, runtime, sessionId, resume }: AgentSessionOptions) => SessionRecorder;
|
|
917
1308
|
|
|
1309
|
+
interface MachineThresholds {
|
|
1310
|
+
readonly warningPercent: number;
|
|
1311
|
+
readonly criticalPercent: number;
|
|
1312
|
+
}
|
|
1313
|
+
declare const sampleMachine: () => MachineSample;
|
|
1314
|
+
declare const summarizeMachine: (samples: readonly MachineSample[], sampleIntervalMs?: number, limits?: Partial<MachineThresholds>) => MachineMetrics;
|
|
1315
|
+
declare const adaptiveConcurrency: (configured: number, sample: MachineSample, limits?: Partial<MachineThresholds>) => number;
|
|
1316
|
+
declare const createMachineMonitor: (sampleIntervalMs?: number, options?: {
|
|
1317
|
+
readonly sample?: () => MachineSample;
|
|
1318
|
+
readonly thresholds?: Partial<MachineThresholds>;
|
|
1319
|
+
}) => {
|
|
1320
|
+
readonly sample: () => MachineSample;
|
|
1321
|
+
readonly observeConcurrency: (value: number) => void;
|
|
1322
|
+
readonly markThrottle: () => void;
|
|
1323
|
+
readonly stop: () => MachineMetrics;
|
|
1324
|
+
};
|
|
1325
|
+
|
|
1326
|
+
interface CoordinationIdentity {
|
|
1327
|
+
readonly tracker: string;
|
|
1328
|
+
readonly repository: string;
|
|
1329
|
+
readonly issue: string;
|
|
1330
|
+
readonly worktree: string;
|
|
1331
|
+
readonly branch: string;
|
|
1332
|
+
}
|
|
1333
|
+
interface DispatchLease extends CoordinationIdentity {
|
|
1334
|
+
readonly key: string;
|
|
1335
|
+
readonly leaseId: string;
|
|
1336
|
+
readonly owner: string;
|
|
1337
|
+
readonly claimedAt: string;
|
|
1338
|
+
}
|
|
1339
|
+
interface ClaimResult {
|
|
1340
|
+
readonly decision: 'claimed' | 'already-claimed';
|
|
1341
|
+
readonly lease: DispatchLease;
|
|
1342
|
+
}
|
|
1343
|
+
interface DispatchRecord extends DispatchLease {
|
|
1344
|
+
readonly action: 'dispatch' | 'release' | 'recover';
|
|
1345
|
+
readonly at: string;
|
|
1346
|
+
readonly idempotencyKey?: string;
|
|
1347
|
+
readonly commandDigest?: string;
|
|
1348
|
+
readonly reason?: string;
|
|
1349
|
+
}
|
|
1350
|
+
interface DispatchLedger {
|
|
1351
|
+
claim(identity: CoordinationIdentity & {
|
|
1352
|
+
readonly owner: string;
|
|
1353
|
+
}): ClaimResult;
|
|
1354
|
+
recordDispatch(input: {
|
|
1355
|
+
readonly lease: DispatchLease;
|
|
1356
|
+
readonly idempotencyKey: string;
|
|
1357
|
+
readonly commandDigest: string;
|
|
1358
|
+
}): {
|
|
1359
|
+
readonly decision: 'recorded' | 'duplicate';
|
|
1360
|
+
readonly record: DispatchRecord;
|
|
1361
|
+
};
|
|
1362
|
+
release(lease: DispatchLease, reason?: string): DispatchRecord;
|
|
1363
|
+
recover(key: string, input: {
|
|
1364
|
+
readonly actor: string;
|
|
1365
|
+
readonly maxAgeMs?: number;
|
|
1366
|
+
readonly reason: string;
|
|
1367
|
+
}): DispatchRecord;
|
|
1368
|
+
active(): readonly DispatchLease[];
|
|
1369
|
+
records(): readonly DispatchRecord[];
|
|
1370
|
+
}
|
|
1371
|
+
declare const createDispatchLedger: (stateDir: string) => DispatchLedger;
|
|
1372
|
+
|
|
1373
|
+
type FailureClass = 'quota' | 'timeout' | 'policy' | 'validation' | 'external' | 'unknown';
|
|
1374
|
+
interface FailureClassification {
|
|
1375
|
+
readonly class: FailureClass;
|
|
1376
|
+
readonly retryable: boolean;
|
|
1377
|
+
readonly reason: string;
|
|
1378
|
+
}
|
|
1379
|
+
interface RecoveryPolicy {
|
|
1380
|
+
readonly maxAttempts: number;
|
|
1381
|
+
readonly baseDelayMs: number;
|
|
1382
|
+
readonly maxDelayMs: number;
|
|
1383
|
+
readonly timeoutMs?: number;
|
|
1384
|
+
}
|
|
1385
|
+
interface RecoveryObservation {
|
|
1386
|
+
readonly attempt: number;
|
|
1387
|
+
readonly failure: FailureClassification;
|
|
1388
|
+
readonly delayMs: number;
|
|
1389
|
+
}
|
|
1390
|
+
interface RecoveryResult<T> {
|
|
1391
|
+
readonly value?: T;
|
|
1392
|
+
readonly status: 'completed' | 'failed';
|
|
1393
|
+
readonly attempts: number;
|
|
1394
|
+
readonly observations: readonly RecoveryObservation[];
|
|
1395
|
+
readonly failure?: FailureClassification;
|
|
1396
|
+
}
|
|
1397
|
+
declare const classifyFailure: (error: unknown) => FailureClassification;
|
|
1398
|
+
declare const recoveryDelayMs: (attempt: number, policy: Pick<RecoveryPolicy, "baseDelayMs" | "maxDelayMs">) => number;
|
|
1399
|
+
declare const runWithRecovery: <T>(operation: (signal: AbortSignal, attempt: number) => Promise<T>, options: RecoveryPolicy & {
|
|
1400
|
+
readonly sleep?: (delayMs: number) => Promise<void>;
|
|
1401
|
+
readonly onObservation?: (observation: RecoveryObservation) => void;
|
|
1402
|
+
}) => Promise<RecoveryResult<T>>;
|
|
1403
|
+
|
|
1404
|
+
interface ChangedFile {
|
|
1405
|
+
readonly path: string;
|
|
1406
|
+
readonly status?: string;
|
|
1407
|
+
}
|
|
1408
|
+
interface FilePreflightPlan {
|
|
1409
|
+
readonly files: readonly string[];
|
|
1410
|
+
readonly codeFiles: readonly string[];
|
|
1411
|
+
readonly testFiles: readonly string[];
|
|
1412
|
+
readonly docsOnly: boolean;
|
|
1413
|
+
readonly checks: readonly ('lint' | 'typecheck' | 'test')[];
|
|
1414
|
+
}
|
|
1415
|
+
declare const validateSafeCommand: (command: string) => {
|
|
1416
|
+
readonly valid: true;
|
|
1417
|
+
readonly command: string;
|
|
1418
|
+
};
|
|
1419
|
+
declare const planFilePreflight: (files: readonly ChangedFile[], options?: {
|
|
1420
|
+
readonly testRoots?: readonly string[];
|
|
1421
|
+
readonly includeTests?: boolean;
|
|
1422
|
+
}) => FilePreflightPlan;
|
|
1423
|
+
|
|
1424
|
+
declare const BLOCK_STATUSES: readonly ["todo", "picked", "development", "validation", "pr-open", "merged", "post-merge", "done", "blocked", "scope-cut"];
|
|
1425
|
+
type BlockStatus = typeof BLOCK_STATUSES[number];
|
|
1426
|
+
interface BlockManifest {
|
|
1427
|
+
readonly schemaVersion: 1;
|
|
1428
|
+
readonly id: string;
|
|
1429
|
+
readonly title: string;
|
|
1430
|
+
readonly tracker: string;
|
|
1431
|
+
readonly repository: string;
|
|
1432
|
+
readonly acceptanceCriteria: readonly string[];
|
|
1433
|
+
readonly dependencies: readonly string[];
|
|
1434
|
+
readonly wave: number;
|
|
1435
|
+
readonly status: BlockStatus;
|
|
1436
|
+
readonly budget?: {
|
|
1437
|
+
readonly maxMinutes?: number;
|
|
1438
|
+
readonly maxAttempts?: number;
|
|
1439
|
+
};
|
|
1440
|
+
readonly humanGates?: readonly string[];
|
|
1441
|
+
readonly sourceHash?: string;
|
|
1442
|
+
}
|
|
1443
|
+
interface BlockAssessment {
|
|
1444
|
+
readonly status: 'ready' | 'blocked';
|
|
1445
|
+
readonly manifestHash: string;
|
|
1446
|
+
readonly blockers: readonly string[];
|
|
1447
|
+
readonly next: readonly string[];
|
|
1448
|
+
}
|
|
1449
|
+
declare const validateBlockManifest: (value: unknown) => BlockManifest;
|
|
1450
|
+
declare const assessBlock: (manifest: BlockManifest, completedDependencies?: readonly string[]) => BlockAssessment;
|
|
1451
|
+
|
|
1452
|
+
declare const LEARNING_STATUSES: readonly ["proposed", "promoted", "rejected"];
|
|
1453
|
+
type LearningStatus = typeof LEARNING_STATUSES[number];
|
|
1454
|
+
interface LearningRecord {
|
|
1455
|
+
readonly id: string;
|
|
1456
|
+
readonly source: string;
|
|
1457
|
+
readonly category: 'worked' | 'problem' | 'adjustment' | 'other';
|
|
1458
|
+
readonly text: string;
|
|
1459
|
+
readonly status: LearningStatus;
|
|
1460
|
+
readonly recordedAt: string;
|
|
1461
|
+
}
|
|
1462
|
+
declare const parseRetro: (markdown: string, source: string, recordedAt?: string) => readonly LearningRecord[];
|
|
1463
|
+
declare const promoteLearnings: (records: readonly LearningRecord[], input: {
|
|
1464
|
+
readonly actor: string;
|
|
1465
|
+
readonly ids: readonly string[];
|
|
1466
|
+
readonly status?: "promoted" | "rejected";
|
|
1467
|
+
}) => readonly LearningRecord[];
|
|
1468
|
+
|
|
1469
|
+
interface StatusBlock {
|
|
1470
|
+
readonly id: string;
|
|
1471
|
+
readonly status: BlockStatus;
|
|
1472
|
+
readonly owner?: string;
|
|
1473
|
+
readonly issue?: string;
|
|
1474
|
+
readonly branch?: string;
|
|
1475
|
+
readonly revision?: string;
|
|
1476
|
+
readonly blockers?: readonly string[];
|
|
1477
|
+
}
|
|
1478
|
+
interface StatusSnapshot {
|
|
1479
|
+
readonly schemaVersion: 1;
|
|
1480
|
+
readonly generatedAt: string;
|
|
1481
|
+
readonly sourceRevision: string;
|
|
1482
|
+
readonly blocks: readonly StatusBlock[];
|
|
1483
|
+
readonly machine?: MachineMetrics;
|
|
1484
|
+
readonly metrics?: Readonly<Record<string, number>>;
|
|
1485
|
+
readonly next?: string;
|
|
1486
|
+
readonly digest: string;
|
|
1487
|
+
}
|
|
1488
|
+
declare const createStatusSnapshot: (input: Omit<StatusSnapshot, "schemaVersion" | "digest">) => StatusSnapshot;
|
|
1489
|
+
declare const validateStatusSnapshot: (value: unknown) => StatusSnapshot;
|
|
1490
|
+
|
|
1491
|
+
declare const MODEL_ROLES: readonly ["orchestrator", "reviewer", "builder", "watcher"];
|
|
1492
|
+
type ModelRole = typeof MODEL_ROLES[number];
|
|
1493
|
+
interface ModelBinding {
|
|
1494
|
+
readonly role: ModelRole;
|
|
1495
|
+
readonly provider: string;
|
|
1496
|
+
readonly model: string;
|
|
1497
|
+
readonly maxTokens?: number;
|
|
1498
|
+
}
|
|
1499
|
+
interface ModelPolicy {
|
|
1500
|
+
readonly bindings: readonly ModelBinding[];
|
|
1501
|
+
readonly digest: string;
|
|
1502
|
+
}
|
|
1503
|
+
declare const createModelPolicy: (bindings: readonly ModelBinding[]) => ModelPolicy;
|
|
1504
|
+
declare const modelFor: (policy: ModelPolicy, role: ModelRole) => ModelBinding;
|
|
1505
|
+
|
|
1506
|
+
interface OrcaDispatchInput {
|
|
1507
|
+
readonly repository: string;
|
|
1508
|
+
readonly worktree: string;
|
|
1509
|
+
readonly branch: string;
|
|
1510
|
+
readonly baseBranch: string;
|
|
1511
|
+
readonly goalFile: string;
|
|
1512
|
+
readonly agent?: string;
|
|
1513
|
+
}
|
|
1514
|
+
interface OrcaDispatchPlan {
|
|
1515
|
+
readonly argv: readonly string[];
|
|
1516
|
+
readonly commandDigest: string;
|
|
1517
|
+
readonly idempotencyKey: string;
|
|
1518
|
+
}
|
|
1519
|
+
declare const createOrcaDispatchPlan: (input: OrcaDispatchInput) => OrcaDispatchPlan;
|
|
1520
|
+
|
|
1521
|
+
interface TrackingTransition {
|
|
1522
|
+
readonly tracker: string;
|
|
1523
|
+
readonly issue: string;
|
|
1524
|
+
readonly from?: string;
|
|
1525
|
+
readonly to: string;
|
|
1526
|
+
readonly reason: string;
|
|
1527
|
+
readonly idempotencyKey: string;
|
|
1528
|
+
}
|
|
1529
|
+
interface TrackingAdapter {
|
|
1530
|
+
readonly id: string;
|
|
1531
|
+
transition(input: Omit<TrackingTransition, 'idempotencyKey'>): Promise<TrackingTransition>;
|
|
1532
|
+
}
|
|
1533
|
+
declare const createTrackingTransition: (input: Omit<TrackingTransition, "idempotencyKey">) => TrackingTransition;
|
|
1534
|
+
declare const createTrackingAdapter: (id: string, handler: (input: TrackingTransition) => Promise<void> | void) => TrackingAdapter;
|
|
1535
|
+
|
|
918
1536
|
declare const EVIDENCE_BUNDLE_SCHEMA_VERSION: 1;
|
|
919
1537
|
interface EvidenceBundleFile {
|
|
920
1538
|
readonly path: string;
|
|
@@ -965,4 +1583,4 @@ declare const verifyEvidenceBundle: (path: string, { trustedKeys }?: {
|
|
|
965
1583
|
}) => EvidenceBundleVerification;
|
|
966
1584
|
declare const readEvidenceTrustStore: (path: string) => readonly TrustedEvidenceKey[];
|
|
967
1585
|
|
|
968
|
-
export { type AgentAdapter, type
|
|
1586
|
+
export { type AgentAdapter, type AgentEvalCase, type AgentEvalReport, type AgentEvalSuite, type AgentMemoryAdapter, type AgentMemoryHit, type AgentMemoryKvStore, type AgentMemoryRecord, type AgentSessionOptions, type ApprovedAssumption, type AutonomyMode, BENCHMARK_SCHEMA_VERSION, BLOCK_STATUSES, type BenchmarkBinding, type BenchmarkComparison, type BenchmarkImprovementDirection, type BenchmarkManifest, type BenchmarkObservation, type BenchmarkObservationEvidence, type BenchmarkObservationInput, type BenchmarkObservationStatus, type BenchmarkReport, type BenchmarkRun, type BenchmarkSummary, type BenchmarkTask, type BlockAssessment, type BlockManifest, type BlockStatus, CHECK_CATEGORIES, CONTEXT_PROVIDER_SLOT, type CacheUsage, type ChangedFile, type CheckCategory, type CheckResult, type ClaimResult, type ContextProvider, type ContextQuery, type ContextReference, type ContextSnapshot, type ContractOutcome, type ContractScope, type CoordinationIdentity, type CriterionStatus, type CycleIterationMetrics, type CycleMatrixRow, type CycleStepResult, type CycleStepStatus, type DecisionPacket, type DiscoveryAmbiguity, type DiscoveryCurrentInput, type DiscoveryCurrentResult, type DiscoveryDecisionLogEntry, type DiscoveryInput, type DiscoveryOption, type DiscoveryResult, type DispatchLease, type DispatchLedger, type DispatchRecord, type Disposer, type DockerMount, type DockerRuntimeEvidence, type DockerToolDefinition, EVENT_LOG_GENESIS, EVIDENCE_BUNDLE_SCHEMA_VERSION, type EvalExpectation, type EventLogLock, type EventLogLockRecovery, type EventLogLockStatus, type EventLogVerification, type EventStore, type EvidenceArtifact, type EvidenceBundle, type EvidenceBundleFile, type EvidenceBundleSignature, type EvidenceBundleVerification, type EvidenceReference, type FailureClass, type FailureClassification, FileEventStore, type FilePreflightPlan, type GateAssessment, type GateBinding, type GateCriterion, HARNESS_EVENT_SCHEMA_VERSION, HARNESS_EVENT_TYPES, HARNESS_PLUGIN_API_VERSION, HarnessError, type HarnessEvent, type HarnessEventContext, type HarnessEventInput, type HarnessEventListener, type HarnessEventPayloads, type HarnessEventType, type HarnessPlugin, type HarnessPluginContext, IMPROVEMENT_CYCLE_STEPS, type ImprovementCycleAssessment, type ImprovementCycleInput, type ImprovementCycleIteration, type ImprovementCycleStep, LEARNING_STATUSES, LEGAL_TRANSITIONS, type LearningRecord, type LearningStatus, type LlmCache, type LlmCacheKeyInput, type LlmCacheStats, type LoadedConfig, MEMORY_SCOPES, MODEL_ROLES, type MachineMetrics, type MachineSample, type MachineThresholds, type MemoryScope, type MemoryUsage, type ModelBinding, type ModelPolicy, type ModelRole, type OptimizationComparison, type OptimizationObservation, type OrcaDispatchInput, type OrcaDispatchPlan, type ParallelismUsage, type PilotAssessment, type PilotEntry, type PilotManifest, type PluginContribution, type PluginRegistry, type PluginSlot, type PolicyDecision, type PolicyGate, type PolicyRequest, type PolicyRule, type ProcessToolDefinition, type ProductionEvidence, type PullRequestDraft, RUN_STATES, type RecoveryObservation, type RecoveryPolicy, type RecoveryResult, type RepositoryProfile, type RunOutcome, type RunReconciliation, type RunState, type RuntimeConfig, type RuntimeExperimentCandidate, type RuntimeExperimentResult, STATES, SURFACE_NAMES, type SessionRecorder, type SourceSnapshot, type StateTransition, type StatusBlock, type StatusSnapshot, type StructuredEvidence, type SurfaceName, type SurfaceRequirement, type TaskContract, type TokenUsage, type ToolDefinition, type ToolExecutionRequest, type ToolExecutionResult, type ToolRuntime, type TrackingAdapter, type TrackingConfig, type TrackingTransition, type TrustedEvidenceKey, type VerificationCheck, type VerificationConfig, type VerificationRun, WIP_STATES, type WipAssessment, type WipAssessmentInput, type WipEntry, type WipState, type WorkflowNode, type WorkflowResult, adaptiveConcurrency, approveRun, approvedDecision, assertHuman, assessAcceptance, assessAgentEval, assessBlock, assessDiscovery, assessImprovementCycle, assessIntegration, assessPilot, assessPreflight, assessProduction, assessWip, assessWorktreeCleanup, authorizeRun, benchmarkRuns, cancelRun, classifyFailure, cleanTaskArtifacts, compareOptimization, composePullRequest, createConfiguredToolRuntime, createDispatchLedger, createDocBridgeContextProvider, createDockerToolRuntime, createInMemoryMemoryAdapter, createKvMemoryAdapter, createLlmCache, createLlmCacheKey, createMachineMonitor, createModelPolicy, createOrcaDispatchPlan, createPluginRegistry, createPluginSlot, createPolicyGate, createProcessToolRuntime, createSessionRecorder, createStatusSnapshot, createToolRuntime, createTrackingAdapter, createTrackingTransition, exportEvidenceBundle, hashContextSnapshot, hashContextSnapshots, inspectEventLogLock, isDiscoveryCurrent, loadBenchmarkManifest, loadConfig, loadLatestRun, modelFor, parseRetro, planFilePreflight, planRun, promoteLearnings, readContextSnapshots, readEvidenceTrustStore, reconcileRun, recordBenchmarkObservation, recoverEventLogLock, recoveryDelayMs, retryRun, runAgentEval, runWithRecovery, runWorkflow, sampleMachine, selectRuntime, startRun, summarizeMachine, transition, validateBenchmarkManifest, validateBlockManifest, validateCacheableOperation, validateConfig, validateContextSnapshot, validateContextSnapshots, validateMemoryRecord, validateOptimizationObservation, validateSafeCommand, validateStatusSnapshot, verifyEvidenceBundle, verifyRun };
|