@tangle-network/agent-eval 0.147.0 → 0.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +7 -0
- package/dist/analyst/index.d.ts +3 -3
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-BTiysiYU.js → benchmark-command-gufocQ-N.js} +2 -2
- package/dist/{benchmark-command-BTiysiYU.js.map → benchmark-command-gufocQ-N.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-DCDdhuv2.js → campaign-BcfXzmPM.js} +4 -4
- package/dist/{campaign-DCDdhuv2.js.map → campaign-BcfXzmPM.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/contract/index.d.ts +3 -3
- package/dist/contract/index.js +3 -3
- package/dist/{define-agent-eval-D52ClbX2.d.ts → define-agent-eval-BWuk2O_N.d.ts} +3 -4
- package/dist/define-agent-eval-BWuk2O_N.d.ts.map +1 -0
- package/dist/{define-agent-eval-BqWFz3sK.js → define-agent-eval-CvZQW4u9.js} +2 -2
- package/dist/define-agent-eval-CvZQW4u9.js.map +1 -0
- package/dist/experiment/index.d.ts +117 -1
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +217 -1
- package/dist/experiment/index.js.map +1 -1
- package/dist/fuzz.d.ts +1 -1
- package/dist/{index-BjBjxiVv.d.ts → index-BCDyf_kT.d.ts} +12 -358
- package/dist/index-BCDyf_kT.d.ts.map +1 -0
- package/dist/{index-gtjtfLEJ.d.ts → index-aHDnhBC7.d.ts} +2 -2
- package/dist/{index-gtjtfLEJ.d.ts.map → index-aHDnhBC7.d.ts.map} +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +8 -15
- package/dist/index.js.map +1 -1
- package/dist/{llm-judge-DrzsVS5k.js → llm-judge-DWq1Ptco.js} +259 -5
- package/dist/llm-judge-DWq1Ptco.js.map +1 -0
- package/dist/{matrix-C-2Qx1Zr.d.ts → matrix-Bc111FKv.d.ts} +17 -1
- package/dist/{matrix-C-2Qx1Zr.d.ts.map → matrix-Bc111FKv.d.ts.map} +1 -1
- package/dist/{multi-layer-verifier-DIguZc8Z.d.ts → multi-layer-verifier-BUaQ4C17.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DIguZc8Z.d.ts.map → multi-layer-verifier-BUaQ4C17.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/multishot/index.js +16 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-Dtx60bUQ.js → produced-state-C0oJ4vr-.js} +2 -2
- package/dist/{produced-state-Dtx60bUQ.js.map → produced-state-C0oJ4vr-.js.map} +1 -1
- package/dist/{provenance-LrOEOHQb.d.ts → provenance-DA-Pmyfv.d.ts} +441 -2
- package/dist/provenance-DA-Pmyfv.d.ts.map +1 -0
- package/dist/{reward-hacking-Bu8ev6PR.d.ts → reward-hacking-62tojkQd.d.ts} +2 -2
- package/dist/{reward-hacking-Bu8ev6PR.d.ts.map → reward-hacking-62tojkQd.d.ts.map} +1 -1
- package/dist/rl.d.ts +2 -2
- package/dist/{skillopt-optimization-method-BoC1Qccx.d.ts → skillopt-optimization-method-ChCTN4FD.d.ts} +3 -3
- package/dist/{skillopt-optimization-method-BoC1Qccx.d.ts.map → skillopt-optimization-method-ChCTN4FD.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-C_UrqZs2.js → skillopt-optimization-method-Donmq4sq.js} +2 -2
- package/dist/{skillopt-optimization-method-C_UrqZs2.js.map → skillopt-optimization-method-Donmq4sq.js.map} +1 -1
- package/docs/feature-guide.md +1 -1
- package/docs/multishot-golden-records.md +12 -1
- package/docs/search-history-receipts.md +139 -0
- package/package.json +2 -2
- package/dist/define-agent-eval-BqWFz3sK.js.map +0 -1
- package/dist/define-agent-eval-D52ClbX2.d.ts.map +0 -1
- package/dist/index-BjBjxiVv.d.ts.map +0 -1
- package/dist/llm-judge-DrzsVS5k.js.map +0 -1
- package/dist/provenance-LrOEOHQb.d.ts.map +0 -1
|
@@ -8,11 +8,10 @@ import { A as LabeledScenarioWrite, D as LabeledScenarioSampleArgs, E as Labeled
|
|
|
8
8
|
import { d as CompletionVerdict, f as CorrectnessChecker, h as ProducedState, n as BackendIntegrityReport, s as RuntimeEventLike, u as CompletionRequirement } from "./backend-integrity-Bz8nSrRE.js";
|
|
9
9
|
import { g as PairedArmsComparison } from "./pre-registration-zFSLEiFU.js";
|
|
10
10
|
import { d as PairedBootstrapResult } from "./paired-promotion-decision-CGzg0cI_.js";
|
|
11
|
-
import {
|
|
12
|
-
import "./skillopt-optimization-method-
|
|
11
|
+
import { Cn as CampaignStorage, xn as PlanCampaignRunOptions, yn as CampaignRunPlan } from "./provenance-DA-Pmyfv.js";
|
|
12
|
+
import "./skillopt-optimization-method-ChCTN4FD.js";
|
|
13
13
|
import "./promotion-policy-CewdjfCJ.js";
|
|
14
14
|
import "./statistical-heldout-uxSpFEjm.js";
|
|
15
|
-
import { D as LedgerHash, m as LedgerTrustedHeadRemoval, p as LedgerTrustedHead } from "./index-B8Ui1mr1.js";
|
|
16
15
|
import { AgentProfile } from "@tangle-network/agent-interface";
|
|
17
16
|
//#region src/campaign/analyst-surface.d.ts
|
|
18
17
|
interface TraceAnalystScenario extends Scenario {
|
|
@@ -307,6 +306,14 @@ interface CrossSurfaceInteractionReport<TRow extends CrossSurfaceTaskRow = Cross
|
|
|
307
306
|
*/
|
|
308
307
|
declare function analyzeCrossSurfaceInteractions<TRow extends CrossSurfaceTaskRow>(input: AnalyzeCrossSurfaceInteractionsInput<TRow>): CrossSurfaceInteractionReport<TRow>;
|
|
309
308
|
//#endregion
|
|
309
|
+
//#region src/campaign/search-ledger-errors.d.ts
|
|
310
|
+
/** Base error for invalid search-ledger input or operations. */
|
|
311
|
+
declare class SearchLedgerError extends ValidationError {}
|
|
312
|
+
/** Error raised when durable search-ledger data fails an integrity check. */
|
|
313
|
+
declare class SearchLedgerIntegrityError extends SearchLedgerError {}
|
|
314
|
+
/** Error raised when an event identifier is reused with different content. */
|
|
315
|
+
declare class SearchLedgerConflictError extends SearchLedgerError {}
|
|
316
|
+
//#endregion
|
|
310
317
|
//#region src/campaign/fixtures.d.ts
|
|
311
318
|
type EvalFixtureValidationMode = 'vitest' | 'none';
|
|
312
319
|
interface EvalFixtureFile {
|
|
@@ -944,359 +951,6 @@ interface CampaignBreakdown {
|
|
|
944
951
|
* on: mean score per judge dimension + per-scenario composite. */
|
|
945
952
|
declare function campaignBreakdown<TArtifact, TScenario extends Scenario>(campaign: CampaignResult<TArtifact, TScenario>): CampaignBreakdown;
|
|
946
953
|
//#endregion
|
|
947
|
-
//#region src/campaign/search-ledger-errors.d.ts
|
|
948
|
-
/** Base error for invalid search-ledger input or operations. */
|
|
949
|
-
declare class SearchLedgerError extends ValidationError {}
|
|
950
|
-
/** Error raised when durable search-ledger data fails an integrity check. */
|
|
951
|
-
declare class SearchLedgerIntegrityError extends SearchLedgerError {}
|
|
952
|
-
/** Error raised when an event identifier is reused with different content. */
|
|
953
|
-
declare class SearchLedgerConflictError extends SearchLedgerError {}
|
|
954
|
-
//#endregion
|
|
955
|
-
//#region src/campaign/search-ledger.d.ts
|
|
956
|
-
declare const SEARCH_LEDGER_SCHEMA: 'tangle.search-ledger.v1';
|
|
957
|
-
type SearchLedgerHash = LedgerHash;
|
|
958
|
-
type SearchSurfaceKind = 'prompt' | 'tool-contract' | 'runtime-config' | 'memory' | 'knowledge' | 'agent-profile' | 'code' | 'deployment';
|
|
959
|
-
/** Content-addressed artifact or receipt. Mutable paths are locators only; the
|
|
960
|
-
* digest and byte length bind the exact bytes used by the search. */
|
|
961
|
-
interface SearchArtifactRef {
|
|
962
|
-
role: string;
|
|
963
|
-
uri: string;
|
|
964
|
-
sha256: SearchLedgerHash;
|
|
965
|
-
byteLength: number;
|
|
966
|
-
}
|
|
967
|
-
/** Repository, dataset, or package source pinned to an immutable commit or
|
|
968
|
-
* content digest. Branches, tags, and bare package versions are rejected. */
|
|
969
|
-
interface SearchSourceRef {
|
|
970
|
-
uri: string;
|
|
971
|
-
revision: string;
|
|
972
|
-
}
|
|
973
|
-
interface SearchModelIdentity {
|
|
974
|
-
provider: string;
|
|
975
|
-
snapshot: string;
|
|
976
|
-
}
|
|
977
|
-
interface SearchCandidateSurface {
|
|
978
|
-
surfaceId: string;
|
|
979
|
-
kind: SearchSurfaceKind;
|
|
980
|
-
artifact: SearchArtifactRef;
|
|
981
|
-
}
|
|
982
|
-
interface SearchCandidateLineage {
|
|
983
|
-
/** Existing `LineageNode.id`; this ledger references rather than embeds it. */
|
|
984
|
-
lineageNodeId: string;
|
|
985
|
-
parentCandidateIds: string[];
|
|
986
|
-
generation: number;
|
|
987
|
-
proposer: string;
|
|
988
|
-
proposerSource: SearchSourceRef;
|
|
989
|
-
}
|
|
990
|
-
type SearchOperationKind = 'candidate-generation' | 'analysis' | 'selection' | 'judge' | 'other';
|
|
991
|
-
interface SearchPlannedTask {
|
|
992
|
-
taskId: string;
|
|
993
|
-
source: SearchSourceRef;
|
|
994
|
-
benchmark: SearchSourceRef;
|
|
995
|
-
/** Maximum transport attempts for this task and candidate. Only an explicit
|
|
996
|
-
* passed/failed outcome satisfies the planned denominator. */
|
|
997
|
-
maxAttempts: number;
|
|
998
|
-
}
|
|
999
|
-
interface SearchPlannedOperation {
|
|
1000
|
-
operationId: string;
|
|
1001
|
-
kind: SearchOperationKind;
|
|
1002
|
-
}
|
|
1003
|
-
interface SearchCandidateSlot {
|
|
1004
|
-
slotId: string;
|
|
1005
|
-
/** Planned candidate-generation call that must either produce this slot or
|
|
1006
|
-
* fail before the slot can be closed. Several slots may share one batched call. */
|
|
1007
|
-
generationOperationId: string;
|
|
1008
|
-
}
|
|
1009
|
-
interface SearchPlan {
|
|
1010
|
-
/** Stable slots and their proposer calls are frozen before search begins. */
|
|
1011
|
-
candidateSlots: SearchCandidateSlot[];
|
|
1012
|
-
/** Every task applies to every successfully registered candidate. */
|
|
1013
|
-
tasks: SearchPlannedTask[];
|
|
1014
|
-
/** Non-task spend slots: proposal, analysis, selection, extra judges, etc. */
|
|
1015
|
-
operations: SearchPlannedOperation[];
|
|
1016
|
-
}
|
|
1017
|
-
type SearchTokenAccounting = {
|
|
1018
|
-
status: 'known';
|
|
1019
|
-
inputTokens: number;
|
|
1020
|
-
outputTokens: number;
|
|
1021
|
-
cachedTokens: number;
|
|
1022
|
-
} | {
|
|
1023
|
-
status: 'unknown';
|
|
1024
|
-
reason: string;
|
|
1025
|
-
};
|
|
1026
|
-
type SearchCostAccounting = {
|
|
1027
|
-
status: 'known';
|
|
1028
|
-
usd: number;
|
|
1029
|
-
source: 'provider' | 'pricing-table' | 'free';
|
|
1030
|
-
} | {
|
|
1031
|
-
status: 'unknown';
|
|
1032
|
-
/** Known spend may still be a lower bound when one call was unpriced. */
|
|
1033
|
-
knownLowerBoundUsd: number;
|
|
1034
|
-
reason: string;
|
|
1035
|
-
};
|
|
1036
|
-
interface SearchAttemptAccounting {
|
|
1037
|
-
tokens: SearchTokenAccounting;
|
|
1038
|
-
cost: SearchCostAccounting;
|
|
1039
|
-
}
|
|
1040
|
-
interface SearchFailureReason {
|
|
1041
|
-
code: string;
|
|
1042
|
-
message: string;
|
|
1043
|
-
}
|
|
1044
|
-
type SearchTaskOutcome = {
|
|
1045
|
-
status: 'passed';
|
|
1046
|
-
score: number;
|
|
1047
|
-
metrics: Record<string, number>;
|
|
1048
|
-
} | {
|
|
1049
|
-
status: 'failed';
|
|
1050
|
-
score: number;
|
|
1051
|
-
metrics: Record<string, number>;
|
|
1052
|
-
failure: SearchFailureReason;
|
|
1053
|
-
} | {
|
|
1054
|
-
status: 'errored';
|
|
1055
|
-
metrics: Record<string, number>;
|
|
1056
|
-
error: SearchFailureReason & {
|
|
1057
|
-
retryable: boolean;
|
|
1058
|
-
};
|
|
1059
|
-
};
|
|
1060
|
-
type SearchSurfaceEffect = {
|
|
1061
|
-
status: 'measured';
|
|
1062
|
-
metric: string;
|
|
1063
|
-
baselineValue: number;
|
|
1064
|
-
candidateValue: number;
|
|
1065
|
-
delta: number;
|
|
1066
|
-
} | {
|
|
1067
|
-
status: 'not-measured';
|
|
1068
|
-
reason: string;
|
|
1069
|
-
};
|
|
1070
|
-
/** Per-attempt proof that a declared candidate surface was or was not active,
|
|
1071
|
-
* plus measured effect when the experiment supports attribution. */
|
|
1072
|
-
interface SearchSurfaceEvidence {
|
|
1073
|
-
surfaceId: string;
|
|
1074
|
-
fired: boolean;
|
|
1075
|
-
firingCount: number;
|
|
1076
|
-
effect: SearchSurfaceEffect;
|
|
1077
|
-
evidence: SearchArtifactRef[];
|
|
1078
|
-
}
|
|
1079
|
-
interface SearchLedgerEventBase {
|
|
1080
|
-
eventId: string;
|
|
1081
|
-
occurredAt: string;
|
|
1082
|
-
artifacts: SearchArtifactRef[];
|
|
1083
|
-
}
|
|
1084
|
-
interface SearchPlannedEvent extends SearchLedgerEventBase {
|
|
1085
|
-
kind: 'search-planned';
|
|
1086
|
-
plan: SearchPlan;
|
|
1087
|
-
}
|
|
1088
|
-
interface SearchCandidateRegisteredEvent extends SearchLedgerEventBase {
|
|
1089
|
-
kind: 'candidate-registered';
|
|
1090
|
-
slotId: string;
|
|
1091
|
-
generationOperationId: string;
|
|
1092
|
-
candidateId: string;
|
|
1093
|
-
lineage: SearchCandidateLineage;
|
|
1094
|
-
surfaces: SearchCandidateSurface[];
|
|
1095
|
-
}
|
|
1096
|
-
interface SearchCandidateSlotClosedEvent extends SearchLedgerEventBase {
|
|
1097
|
-
kind: 'candidate-slot-closed';
|
|
1098
|
-
slotId: string;
|
|
1099
|
-
generationOperationId: string;
|
|
1100
|
-
reason: SearchFailureReason;
|
|
1101
|
-
}
|
|
1102
|
-
interface SearchTaskAttemptedEvent extends SearchLedgerEventBase {
|
|
1103
|
-
kind: 'task-attempted';
|
|
1104
|
-
candidateId: string;
|
|
1105
|
-
runId: string;
|
|
1106
|
-
attemptIndex: number;
|
|
1107
|
-
task: {
|
|
1108
|
-
taskId: string;
|
|
1109
|
-
source: SearchSourceRef;
|
|
1110
|
-
};
|
|
1111
|
-
identity: {
|
|
1112
|
-
model: SearchModelIdentity;
|
|
1113
|
-
agent: SearchSourceRef;
|
|
1114
|
-
benchmark: SearchSourceRef;
|
|
1115
|
-
};
|
|
1116
|
-
outcome: SearchTaskOutcome;
|
|
1117
|
-
accounting: SearchAttemptAccounting;
|
|
1118
|
-
surfaceEvidence: SearchSurfaceEvidence[];
|
|
1119
|
-
}
|
|
1120
|
-
interface SearchOperationRecordedEvent extends SearchLedgerEventBase {
|
|
1121
|
-
kind: 'search-operation-recorded';
|
|
1122
|
-
operationId: string;
|
|
1123
|
-
operationKind: SearchOperationKind;
|
|
1124
|
-
execution: {
|
|
1125
|
-
kind: 'model';
|
|
1126
|
-
model: SearchModelIdentity;
|
|
1127
|
-
source: SearchSourceRef;
|
|
1128
|
-
} | {
|
|
1129
|
-
kind: 'deterministic';
|
|
1130
|
-
source: SearchSourceRef;
|
|
1131
|
-
};
|
|
1132
|
-
outcome: {
|
|
1133
|
-
status: 'completed';
|
|
1134
|
-
} | {
|
|
1135
|
-
status: 'partial';
|
|
1136
|
-
failure: SearchFailureReason;
|
|
1137
|
-
} | {
|
|
1138
|
-
status: 'failed';
|
|
1139
|
-
failure: SearchFailureReason;
|
|
1140
|
-
};
|
|
1141
|
-
accounting: SearchAttemptAccounting;
|
|
1142
|
-
}
|
|
1143
|
-
interface SearchCandidateDecidedEvent extends SearchLedgerEventBase {
|
|
1144
|
-
kind: 'candidate-decided';
|
|
1145
|
-
candidateId: string;
|
|
1146
|
-
decision: {
|
|
1147
|
-
status: 'selected';
|
|
1148
|
-
} | {
|
|
1149
|
-
status: 'rejected';
|
|
1150
|
-
reason: SearchFailureReason;
|
|
1151
|
-
};
|
|
1152
|
-
}
|
|
1153
|
-
interface SearchCompletedEvent extends SearchLedgerEventBase {
|
|
1154
|
-
kind: 'search-completed';
|
|
1155
|
-
result: {
|
|
1156
|
-
status: 'selected';
|
|
1157
|
-
candidateId: string;
|
|
1158
|
-
} | {
|
|
1159
|
-
status: 'all-rejected';
|
|
1160
|
-
reason: SearchFailureReason;
|
|
1161
|
-
};
|
|
1162
|
-
}
|
|
1163
|
-
type SearchLedgerEvent = SearchPlannedEvent | SearchCandidateRegisteredEvent | SearchCandidateSlotClosedEvent | SearchTaskAttemptedEvent | SearchOperationRecordedEvent | SearchCandidateDecidedEvent | SearchCompletedEvent;
|
|
1164
|
-
interface SearchLedgerEntry {
|
|
1165
|
-
schema: typeof SEARCH_LEDGER_SCHEMA;
|
|
1166
|
-
campaignId: string;
|
|
1167
|
-
sequence: number;
|
|
1168
|
-
previousHash: SearchLedgerHash | null;
|
|
1169
|
-
event: SearchLedgerEvent;
|
|
1170
|
-
entryHash: SearchLedgerHash;
|
|
1171
|
-
}
|
|
1172
|
-
type SearchAccountingAudit = {
|
|
1173
|
-
status: 'known';
|
|
1174
|
-
inputTokens: number;
|
|
1175
|
-
outputTokens: number;
|
|
1176
|
-
cachedTokens: number;
|
|
1177
|
-
costUsd: number;
|
|
1178
|
-
} | {
|
|
1179
|
-
status: 'partial';
|
|
1180
|
-
knownInputTokens: number;
|
|
1181
|
-
knownOutputTokens: number;
|
|
1182
|
-
knownCachedTokens: number;
|
|
1183
|
-
knownCostUsd: number;
|
|
1184
|
-
unknownTokenEventIds: string[];
|
|
1185
|
-
unknownCostEventIds: string[];
|
|
1186
|
-
};
|
|
1187
|
-
interface SearchLedgerAudit {
|
|
1188
|
-
campaignId: string;
|
|
1189
|
-
eventCount: number;
|
|
1190
|
-
candidateCount: number;
|
|
1191
|
-
closedCandidateSlotCount: number;
|
|
1192
|
-
attemptCount: number;
|
|
1193
|
-
operationCount: number;
|
|
1194
|
-
outcomes: {
|
|
1195
|
-
passed: number;
|
|
1196
|
-
failed: number;
|
|
1197
|
-
errored: number;
|
|
1198
|
-
};
|
|
1199
|
-
operationOutcomes: {
|
|
1200
|
-
completed: number;
|
|
1201
|
-
partial: number;
|
|
1202
|
-
failed: number;
|
|
1203
|
-
};
|
|
1204
|
-
decisions: {
|
|
1205
|
-
selected: number;
|
|
1206
|
-
rejected: number;
|
|
1207
|
-
pending: number;
|
|
1208
|
-
};
|
|
1209
|
-
expected: {
|
|
1210
|
-
candidateSlots: number;
|
|
1211
|
-
taskOutcomes: number;
|
|
1212
|
-
operations: number;
|
|
1213
|
-
missingCandidateSlots: string[];
|
|
1214
|
-
missingTaskOutcomes: string[];
|
|
1215
|
-
missingOperations: string[];
|
|
1216
|
-
};
|
|
1217
|
-
status: 'in-progress' | 'selected' | 'all-rejected';
|
|
1218
|
-
selectedCandidateId: string | null;
|
|
1219
|
-
accounting: SearchAccountingAudit;
|
|
1220
|
-
headHash: SearchLedgerHash | null;
|
|
1221
|
-
}
|
|
1222
|
-
interface SearchLedgerReplay {
|
|
1223
|
-
entries: SearchLedgerEntry[];
|
|
1224
|
-
plan: SearchPlannedEvent | null;
|
|
1225
|
-
candidates: SearchCandidateRegisteredEvent[];
|
|
1226
|
-
closedCandidateSlots: SearchCandidateSlotClosedEvent[];
|
|
1227
|
-
attempts: SearchTaskAttemptedEvent[];
|
|
1228
|
-
operations: SearchOperationRecordedEvent[];
|
|
1229
|
-
decisions: SearchCandidateDecidedEvent[];
|
|
1230
|
-
completion: SearchCompletedEvent | null;
|
|
1231
|
-
audit: SearchLedgerAudit;
|
|
1232
|
-
}
|
|
1233
|
-
interface SearchLedgerAppendResult {
|
|
1234
|
-
entry: SearchLedgerEntry;
|
|
1235
|
-
/** False when the exact event was already durably present. */
|
|
1236
|
-
appended: boolean;
|
|
1237
|
-
replay: SearchLedgerReplay;
|
|
1238
|
-
}
|
|
1239
|
-
/** Validate and return a canonical copy. Arrays whose order is not semantic are
|
|
1240
|
-
* sorted so retries from different processes produce byte-identical events. */
|
|
1241
|
-
declare function validateSearchLedgerEvent(input: unknown): SearchLedgerEvent;
|
|
1242
|
-
/**
|
|
1243
|
-
* How this ledger uses its trusted head — the `(sequence, entryHash)` pin kept
|
|
1244
|
-
* in the sibling `<path>.head` file that a hash chain needs to prove entries
|
|
1245
|
-
* were not deleted from the end. `ledger-core/trusted-head.ts` holds the threat
|
|
1246
|
-
* model.
|
|
1247
|
-
*
|
|
1248
|
-
* - `pin` (default): every append records the new head, and a pin that is
|
|
1249
|
-
* present is verified on every read.
|
|
1250
|
-
* - `require`: additionally refuses to read a non-empty ledger whose pin is
|
|
1251
|
-
* gone, so deleting the sibling file cannot downgrade the guarantee. Only for
|
|
1252
|
-
* ledgers written under `pin` from their first entry.
|
|
1253
|
-
* - `off`: chain verification only. Truncation to a valid shorter prefix is
|
|
1254
|
-
* undetectable.
|
|
1255
|
-
*/
|
|
1256
|
-
type SearchLedgerTrustedHeadMode = 'pin' | 'require' | 'off';
|
|
1257
|
-
interface OpenSearchLedgerOptions {
|
|
1258
|
-
path: string;
|
|
1259
|
-
campaignId: string;
|
|
1260
|
-
trustedHead?: SearchLedgerTrustedHeadMode;
|
|
1261
|
-
}
|
|
1262
|
-
interface SearchLedger {
|
|
1263
|
-
readonly path: string;
|
|
1264
|
-
readonly campaignId: string;
|
|
1265
|
-
/** Sibling file holding this ledger's trusted head. */
|
|
1266
|
-
readonly trustedHeadPath: string;
|
|
1267
|
-
append(event: SearchLedgerEvent): Promise<SearchLedgerAppendResult>;
|
|
1268
|
-
replay(): Promise<SearchLedgerReplay>;
|
|
1269
|
-
/** The pinned head, or null when this ledger has never been pinned. */
|
|
1270
|
-
trustedHead(): Promise<LedgerTrustedHead | null>;
|
|
1271
|
-
/** Pin the current verified head: how a ledger written under `off`, or one
|
|
1272
|
-
* whose pin file was removed, acquires a pin without rewriting a byte. */
|
|
1273
|
-
pinTrustedHead(): Promise<LedgerTrustedHead>;
|
|
1274
|
-
/** Discard this ledger's pin, reporting what was discarded. Deleting or
|
|
1275
|
-
* rebuilding the ledger file leaves a pin naming history the file no longer
|
|
1276
|
-
* carries, and every later read is refused because that is exactly the
|
|
1277
|
-
* deletion the pin exists to catch; clearing is the supported way to abandon
|
|
1278
|
-
* that history on purpose. It gives up the deletion guarantee for every entry
|
|
1279
|
-
* the pin covered. */
|
|
1280
|
-
clearTrustedHead(): Promise<LedgerTrustedHeadRemoval>;
|
|
1281
|
-
}
|
|
1282
|
-
/** Open a durable filesystem search ledger. Construction performs no I/O; the
|
|
1283
|
-
* first `append` or `replay` validates the complete existing file. */
|
|
1284
|
-
declare function openSearchLedger(options: OpenSearchLedgerOptions): SearchLedger;
|
|
1285
|
-
/** Append-only file-backed search ledger with idempotent writes and replay. */
|
|
1286
|
-
declare class FileSearchLedger implements SearchLedger {
|
|
1287
|
-
readonly path: string;
|
|
1288
|
-
readonly campaignId: string;
|
|
1289
|
-
readonly trustedHeadPath: string;
|
|
1290
|
-
private readonly trustedHeadMode;
|
|
1291
|
-
private readonly journal;
|
|
1292
|
-
constructor(path: string, campaignId: string, trustedHead?: SearchLedgerTrustedHeadMode);
|
|
1293
|
-
replay(): Promise<SearchLedgerReplay>;
|
|
1294
|
-
append(input: SearchLedgerEvent): Promise<SearchLedgerAppendResult>;
|
|
1295
|
-
trustedHead(): Promise<LedgerTrustedHead | null>;
|
|
1296
|
-
pinTrustedHead(): Promise<LedgerTrustedHead>;
|
|
1297
|
-
clearTrustedHead(): Promise<LedgerTrustedHeadRemoval>;
|
|
1298
|
-
}
|
|
1299
|
-
//#endregion
|
|
1300
954
|
//#region src/campaign/single-run-lock.d.ts
|
|
1301
955
|
/**
|
|
1302
956
|
* Single-run lock for evaluations that share one mutable environment.
|
|
@@ -1510,5 +1164,5 @@ declare function verifyCodeSurface(surface: CodeSurface, worktreeDir?: string):
|
|
|
1510
1164
|
* identity against the checkout at `worktreeRef`. */
|
|
1511
1165
|
declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
|
|
1512
1166
|
//#endregion
|
|
1513
|
-
export {
|
|
1514
|
-
//# sourceMappingURL=index-
|
|
1167
|
+
export { ScoreboardRow as $, CrossSurfaceCandidateOutcome as $t, compareRankKeys as A, EvalFixtureLoadOptions as At, ProfileMatrixCoverage as B, SearchLedgerConflictError as Bt, surfaceHash as C, BuildTraceAnalystSurfaceDispatchOptions as Cn, UngroundedLiteralReport as Ct, CampaignBreakdown as D, traceAnalystQualityJudge as Dn, neutralizationGate as Dt, acquireSingleRunLock as E, buildTraceAnalystSurfaceDispatch as En, NeutralizationGateOptions as Et, resolveRunDir as F, PlanEvalFixtureRunOptions as Ft, SegmentedProfileMatrixError as G, CrossSurfaceAdditionDecision as Gt, ProfileMatrixRow as H, SearchLedgerIntegrityError as Ht, tangleTracesRoot as I, discoverEvalFixtures as It, runProfileMatrixSegment as J, CrossSurfaceBestSingleSelection as Jt, createProfileMatrixPlan as K, CrossSurfaceAdditionRejectionReason as Kt, CreateProfileMatrixPlanOptions as L, loadEvalFixture as Lt, ScenarioSignal as M, EvalFixtureScenario as Mt, scoreDiscrimination as N, EvalFixtureValidationMode as Nt, campaignBreakdown as O, EvalFixture as Ot, selectDiscriminative as P, LoadEvalFixtureScenariosOptions as Pt, ScoreboardRenderOptions as Q, CrossSurfaceCandidateEvidence as Qt, FinalizeProfileMatrixOptions as R, loadEvalFixtureScenarios as Rt, surfaceContentHash as S, CrossSurfaceTaskRow as Sn, ScoredRollout as St, SingleRunLockOptions as T, TraceAnalystScenario as Tn, rolloutArgumentDiff as Tt, ProfileMatrixSegmentResult as U, analyzeCrossSurfaceInteractions as Ut, ProfileMatrixPlan as V, SearchLedgerError as Vt, RunProfileMatrixSegmentOptions as W, AnalyzeCrossSurfaceInteractionsInput as Wt, PlaybackDriver as X, CrossSurfaceCandidate as Xt, PlaybackContext as Y, CrossSurfaceBootstrapPolicy as Yt, PlaybackStep as Z, CrossSurfaceCandidateComparison as Zt, assertCodeSurfaceIdentity as _, CrossSurfacePairwiseEntry as _n, FsLabeledScenarioStoreOptions as _t, WorktreeAdapterError as a, CrossSurfaceEligibility as an, scoreUserStory as at, componentSurfaceIdentityMaterial as b, CrossSurfaceSelectionPolicy as bn, RolloutArgumentDiffOptions as bt, verifyCodeSurface as c, CrossSurfaceInteractionAwareSelection as cn, ProfileDispatchFn as ct, PhoenixEvaluationResultLike as d, CrossSurfaceInteractionReport as dn, RunProfileMatrixOptions as dt, CrossSurfaceCandidateSummary as en, ScoreboardSummary as et, PhoenixEvaluatorLike as f, CrossSurfaceInteractionTask as fn, RunProfileMatrixResult as ft, isTransientTransportFailure as g, CrossSurfacePairIncompatibilityReason as gn, FsLabeledScenarioStore as gt, TransientFailureOptions as h, CrossSurfacePairEvidence as hn, neutralizeText as ht, WorktreeAdapter as i, CrossSurfaceDistribution as in, renderScoreboardMarkdown as it, DiscriminationScore as j, EvalFixtureRunPlan as jt, campaignMeanComposite as k, EvalFixtureFile as kt, AutoevalsScoreLike as l, CrossSurfaceInteractionEffect as ln, ProfileMatrixError as lt, phoenixEvaluatorJudge as m, CrossSurfacePairCompatibility as mn, runProfileMatrix as mt, GitWorktreeAdapterOptions as n, CrossSurfaceComponentEvidence as nn, UserStoryVerdict as nt, gitWorktreeAdapter as o, CrossSurfaceEvidenceBreakdown as on, scoreboardSummary as ot, autoevalsScorerJudge as p, CrossSurfaceNaiveStackSelection as pn, ScenarioRollup as pt, finalizeProfileMatrix as q, CrossSurfaceAttemptCompleteness as qt, Worktree as r, CrossSurfaceCompositionStep as rn, makePlaybackDispatch as rt, resolveWorktreePath as s, CrossSurfaceIneligibilityReason as sn, userStoryScoreboard as st, CodeSurfaceVerification as t, CrossSurfaceComponent as tn, UserStory as tt, AutoevalsScorerLike as u, CrossSurfaceInteractionPath as un, ProfileSummary as ut, assertComponentSurface as v, CrossSurfaceRankedSingle as vn, LabeledScenarioStoreError as vt, SingleRunLock as w, TraceAnalystArtifact as wn, classifyUngroundedLiterals as wt, renderSurfaceDiff as x, CrossSurfaceSelections as xn, RolloutCall as xt, codeSurfaceIdentityMaterial as y, CrossSurfaceRelativeCost as yn, RolloutArgumentDiff as yt, FinalizedProfileMatrixResult as z, planEvalFixtureRun as zt };
|
|
1168
|
+
//# sourceMappingURL=index-BCDyf_kT.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-BCDyf_kT.d.ts","names":[],"sources":["../src/campaign/analyst-surface.ts","../src/campaign/cross-surface-types.ts","../src/campaign/cross-surface-interaction.ts","../src/campaign/search-ledger-errors.ts","../src/campaign/fixtures.ts","../src/campaign/gates/neutralization-gate.ts","../src/campaign/grounded-reflection.ts","../src/campaign/labeled-store/fs-adapter.ts","../src/campaign/neutralize.ts","../src/campaign/presets/run-profile-matrix.ts","../src/campaign/presets/playback.ts","../src/campaign/presets/segmented-profile-matrix.ts","../src/campaign/run-dir.ts","../src/campaign/scenario-selection.ts","../src/campaign/score-utils.ts","../src/campaign/single-run-lock.ts","../src/campaign/surface-identity.ts","../src/campaign/transient-failure.ts","../src/campaign/upstream-evaluators.ts","../src/campaign/worktree/index.ts"],"mappings":";;;;;;;;;;;;;;;;UAWiB,6BAA6B;EAC5C;EACA,YAAY;EACZ,YAAY;EACZ,yBAAyB;EACzB,kBAAkB;;UAGH;EACf,mBAAmB;EACnB,QAAQ;EACR,MAAM;;UAGS;EACf,QAAQ;IACN;IACA,YAAY;IACZ;IACA,QAAQ;MACN,QAAQ;;iBAGE,iCACd,SAAS,2CAET,SAAS,gBACT,UAAU,sBACV,SAAS,oBACN,QAAQ;iBAgBG,4BAA4B,YAC1C,sBACA;;;;KCtDU;;UAGK;EACf;EACA;;EAEA;;;UAIe;EACf;EACA;EACA;EACA;;;UAIe;EACf;;EAEA;;EAEA;;;;;;;UAQe;EACf;EACA;;EAEA;EACA,cAAc;EACd;EACA;;;;;;EAMA,MAAM;EACN,mBAAmB;;EAEnB;;UAGe;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;;EAEA,kCAAkC;;EAElC;;UAGe,qCACf,aAAa,sBAAsB;EAEnC,qBAAqB;EACrB,qBAAqB;EACrB,eAAe;EACf;;EAEA;;EAEA;;EAEA;;EAEA;EACA,WAAW;EACX,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;UAGe;EACf,aAAa;EACb;EACA;EACA;EACA;;KAGU;UAWK;EACf;EACA,SAAS;;UAGM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX,SAAS;EACT,OAAO;EACP,OAAO,eAAe;EACtB,QAAQ;EACR,QAAQ;;EAER,sBAAsB;;EAEtB,aAAa;;UAGE;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,QAAQ;EACR,cAAc,eAAe;;UAGd;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;EACA;;UAGe;EACf,SAAS;EACT;EACA;EACA;EACA;EACA,eAAe;EACf,gBAAgB;;KAGN;UAYK;EACf;EACA,SAAS;EACT;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,4BAA4B,iCAAiC;EAC7D,wBAAwB,eAAe;EACvC,QAAQ;EACR,QAAQ;EACR,aAAa;EACb,eAAe;;UAGA;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA,SAAS;;UAGM;;EAEf;EACA;;KAGU;UAaK;EACf;EACA;EACA;EACA;EACA;EACA,uBAAuB;EACvB;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,YAAY;EACZ;;;UAIe;EACf;EACA;EACA;EACA;EACA,OAAO;;UAGQ;;EAEf;;EAEA;EACA;;EAEA;EACA;;EAEA,gBAAgB;;EAEhB,OAAO;;UAGQ;EACf,YAAY;EACZ,YAAY;EACZ,kBAAkB;;UAGH,8BACf,aAAa,sBAAsB;EAEnC;EACA;EACA;EACA;;EAEA,MAAM;EACN,iBAAiB;EACjB,iBAAiB;EACjB,YAAY;EACZ,UAAU;EACV,YAAY;;;;;;;;;iBCnRE,gCAAgC,aAAa,qBAC3D,OAAO,qCAAqC,QAC3C,8BAA8B;;;;cClDpB,0BAA0B;;cAG1B,mCAAmC;;cAGnC,kCAAkC;;;KCDnC;UAEK;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;EACP;;UAGe,4BAA4B;EAC3C;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf,aAAa;;EAEb;;UAGe,wCAAwC;EACvD;;UAGe,0BAA0B,6BACjC,KACN,uBAAuB,qBAAqB;EAG9C;EACA,aAAa;EACb;EACA;EACA,UAAU;;KAGA,qBAAqB;EAC/B,UAAU,MAAM,KAAK;;;iBAIP,qBAAqB;;;;;iBA4BrB,gBACd,kBACA,cACA,UAAS,yBACR;;iBA4Ca,yBACd,kBACA,UAAS,kCACR;;;;;iBAuBa,mBAAmB,qBACjC,SAAS,0BAA0B,aAClC;;;UChJc,0BAA0B,kBAAkB,WAAW;EACtE,WAAW;;;;;EAKX;;;;;;iBAuBc,mBAAmB,WAAW,kBAAkB,UAC9D,SAAS,0BAA0B,aAClC,KAAK,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UC9BF;WACN;WACA,MAAM,SAAS;;;UAIT;;WAEN;;WAEA;WACA,gBAAgB;;UAGV;;WAEN;;WAEA;;UAGM;;WAEN;;WAEA,eAAe;;WAEf,eAAe;;;;;;;;iBASV,oBACd,mBAAmB,iBACnB,OAAM,6BACL;UAsCc;;WAEN;;WAEA;;;;;;;;;iBAUK,2BACd,cACA,MAAM,KAAK,0DACV;;;UCjFc;;EAEf;;;EAGA;;EAEA;;;cAIW,kCAAkC;WAE3B;EADlB,YACkB,cAChB;;;;;;cAiBS,kCAAkC;mBAIhB;mBAHZ;mBACA;EAEjB,YAA6B,SAAS;EAKhC,QAAQ,OAAO,uBAAuB;EAYtC,OAAO,MAAM,4BAA4B,QAAQ;EA8CjD,QAAQ;IACZ;IACA;IACA,UAAU;IACV,SAAS,OAAO;;UAkCV;UAiCA;UAmBA;UAkBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBCtNM,eAAe;;;;;;cC8ClB,2BAA2B;EACtC,YAAY;;;;;KAQF,kBAAkB,kBAAkB,UAAU,cACxD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;UAEI,wBAAwB,kBAAkB,UAAU;;EAEnE,UAAU;;EAEV,WAAW;;EAEX,UAAU,kBAAkB,WAAW;;EAEvC,SAAS,YAAY,WAAW;;;EAGhC;;;EAGA;;;EAGA;;;;EAIA;;EAEA,WAAW;;EAEX;;EAEA;;;;;;;;EAQA;;;EAGA;;EAEA;;;EAGA;;EAEA;;EAEA,eAAe;EACf,gBAAgB;;;EAGhB,UAAU;;EAEV,YAAY;;;EAGZ,aAAa,UAAU;;;;EAIvB;;;;;;;EAOA,cACE,UAAU,WACV,UAAU;IACL;IAAgB;;;;;;;EAMvB,aAAa;IAAS,SAAS;IAAc,UAAU;IAAW;;;EAElE;;EAEA;;UAGe;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA,gBAAgB;;;EAGhB,WAAW;;UAGI;EACf;EACA;;UAGe,uBAAuB,WAAW,kBAAkB;EACnE;EACA;;;;EAIA,SAAS;EACT,WAAW,eAAe;EAC1B,YAAY,eAAe;;EAE3B,YAAY,eAAe;;EAE3B,WAAW;;EAEX,WAAW,eAAe,eAAe,WAAW;;;;;iBAoOhC,iBAAiB,kBAAkB,UAAU,WACjE,MAAM,wBAAwB,WAAW,aACxC,QAAQ,uBAAuB,WAAW;;;;;UC9Z5B;;EAEf;;EAEA,UAAU;;;;;;;UAQK,kBAAkB;;EAEjC;;EAEA,OAAO;;EAEP,cAAc;;;UAIC,wBAAwB;EACvC,SAAS;;;;;;;;;;;UAYM,eAAe,eAAe,YAAY;EACzD,IAAI,OAAO,QAAQ,KAAK,kBAAkB,iBAAiB;;;;;;;iBAQ7C,qBAAqB,eAAe,WAClD,QAAQ,eAAe,UACtB,kBAAkB,QAAQ;;UAQZ,yBAAyB;EACxC;;;;;;;;iBASoB,eACpB,OAAO,WACP,OAAO,eACP,kBAAkB,qBACjB,QAAQ;;UAUM;EACf;EACA;EACA;EACA;EACA;EACA;;;;;;;iBAQc,oBAAoB,mBAAmB,qBAAqB;;UAkB3D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;iBAIc,kBAAkB,eAAe,kBAAkB;UAwBlD;;EAEf;;EAEA,OAAO;;EAEP;;;;;;;;;iBAkBc,yBACd,eAAe,iBACf,OAAM;;;cC1KK,oCAAoC;EAC/C,YAAY;;UAKG;EACf;EACA;EACA;EACA;EACA;;UAGe,kBAAkB,kBAAkB,UAAU;WACpD;WACA;WACA;WACA;WACA,mBAAmB;WACnB,oBAAoB;WACpB,iBAAiB,YAAY,WAAW;WACxC;WACA;WACA,UAAU,YAAY,wBAAwB,WAAW;WACzD;;WAEA;WACA,WAAW,YAAY,wBAAwB,WAAW;WAC1D;WACA;WACA,aAAa,UAAU;WACvB,cACP,UAAU,WACV,UAAU;IACL;IAAgB;;WACd,eAAe;;UAGT,+BAA+B,kBAAkB,UAAU,mBAClE,KACN,wBAAwB,WAAW;;EAgBrC;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe,+BAA+B,kBAAkB,UAAU;EAC1E,MAAM,kBAAkB,WAAW;;EAEnC;;EAEA,yBAAyB;EACzB,UAAU,kBAAkB,WAAW;EACvC;EACA,UAAU;EACV;EACA;EACA;EACA,eAAe,wBAAwB,WAAW;EAClD,gBAAgB,wBAAwB,WAAW;EACnD,YAAY;;UAGG,2BAA2B,WAAW,kBAAkB;EACvE;EACA;EACA,QAAQ,uBAAuB,WAAW;EAC1C,UAAU;;UAGK,6BAA6B,kBAAkB,UAAU;EACxE,MAAM,kBAAkB,WAAW;EACnC;EACA,UAAU;EACV;EACA;EACA,YAAY;;UAGG,6BAA6B,WAAW,kBAAkB,kBACjE,uBAAuB,WAAW;EAC1C,UAAU;;iBAGI,wBAAwB,kBAAkB,UAAU,WAClE,MAAM,+BAA+B,WAAW,aAC/C,kBAAkB,WAAW;iBA0HV,wBAAwB,kBAAkB,UAAU,WACxE,MAAM,+BAA+B,WAAW,aAC/C,QAAQ,2BAA2B,WAAW;iBAkD3B,sBAAsB,kBAAkB,UAAU,WACtE,MAAM,6BAA6B,WAAW,aAC7C,QAAQ,6BAA6B,WAAW;;;;;;;;iBCrTnC;;;;;iBAQA,cAAc,gBAAgB;;;;;;;;;;;;;;;;;UCD7B;EACf;;EAEA;;UAGe;EACf;;EAEA;;EAEA;EACA;;EAEA;;;;;;;;;;iBA4Cc,oBACd,SAAS,kBACT;EAAS;IACR;;;;;;;;;;;iBA0Ba,qBACd,SAAS,kBACT,WACA;EAAS;;;;;;;iBC3FK,sBAAsB,WAAW,kBAAkB,UACjE,UAAU,eAAe,WAAW;;;;iBA0CtB,gBAAgB,sBAAsB;UAWrC;;EAEf,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;;iBAK5D,kBAAkB,WAAW,kBAAkB,UAC7D,UAAU,eAAe,WAAW,aACnC;;;;;;;;;;;;;;;;;;UC5Dc;;WAEN;;WAEA;;WAEA;;WAEA;;UAGM;;EAEf;;;;;;;iBAwBc,qBAAqB,MAAM,uBAAuB;;;;iBCpDlD,0BAA0B,2BAA2B,WAAW;;iBAmChE,uBAAuB,2BAA2B,WAAW;;iBA8B7D,iCAAiC,SAAS;;;;iBAa1C,4BAA4B,SAAS;;iBAgBrC,mBAAmB,SAAS;;iBAW5B,YAAY,SAAS;;iBAKrB,kBACd,eAAe,gBACf,iBAAiB;;;;;;;;;;;;;;;;;;;;;UCpGF;;;;;;;WAON;;WAEA,yBAAyB;;;;;;iBAYpB,4BACd,oCACA,OAAM;;;UCnCS;EACf;EACA;EACA;;UAGe,qBAAqB,gBAAgB;EACpD;EACA;EACA;EACA,SACE,QAAQ,SACR,SAAS,4BACR,QAAQ;;UAGI;EACf;EACA;EACA,WAAW;;KAGD,oBAAoB,eAAe,4BAC7C,OAAO,QACP,SAAS,8BACN,qBAAqB,QAAQ;UAEjB;WACN,QAAQ;WACR;;KAGN,sBAAsB,WAAW,KACpC,iBAAiB;EAGjB;;UAGQ,qBAAqB,kBAAkB;EAC/C;EACA;EACA;EACA,aAAa,UAAU;;EAEvB,eAAe;;iBAGD,sBACd,gBAAgB,yBAChB,WACA,kBAAkB,WAAW,UAE7B,WAAW,qBAAqB,UAChC,SAAS,qBAAqB;EAC5B,SAAS;IAAS,UAAU;IAAW,UAAU;MAAc;EAC/D,WAAW,sBAAsB;IAElC,YAAY,WAAW;iBA8CV,qBACd,eAAe,yBACf,WACA,kBAAkB,WAAW,UAE7B,QAAQ,oBAAoB,SAC5B,SAAS,qBAAqB;EAC5B;EACA,SAAS;IAAS,UAAU;IAAW,UAAU;MAAc;;EAEzD;EAAc;;EACd;EAAa,UAAU,sBAAsB;KAEpD,YAAY,WAAW;;;KCxFrB,qBAAqB;KACrB,iBAAiB,SAAS;KAC1B,aAAa,gBAAgB,aAAa,MAAM,mBAAmB;UAcvD;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;UAGM;;EAEf,OAAO;IAAQ;IAAiB;MAAkB,QAAQ;;;EAG1D,SAAS,UAAU,UAAU,kBAAkB,QAAQ;;EAEvD,QAAQ,UAAU,WAAW;;;cAIlB,6BAA6B;WAG7B;EAFX,YACE,iBACS;;UAOI;;EAEf;;EAEA;;EAEA;;;;EAIA,MAAM;;UAobS;;EAEf;;EAEA;;EAEA;;;EAGA,YAAY;;;;;iBAoJE,mBAAmB,MAAM,4BAA4B;;;;iBA0FrD,kBACd,SAAS,aACT,uBACC;;;iBAOa,oBAAoB,SAAS,aAAa"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { o as Severity } from "./multi-layer-verifier-
|
|
1
|
+
import { o as Severity } from "./multi-layer-verifier-BUaQ4C17.js";
|
|
2
2
|
import { b as CustomTokenPricing, c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
|
|
3
3
|
import { A as LlmClientOptions } from "./types-DOhGq4S1.js";
|
|
4
4
|
import { i as AnalystFinding, n as AnalystContext } from "./types-BohHewKK.js";
|
|
@@ -449,4 +449,4 @@ declare const KNOWLEDGE_POISONING_KIND_SPEC: TraceAnalystDefinition;
|
|
|
449
449
|
declare const DEFAULT_TRACE_ANALYST_KINDS: readonly TraceAnalystDefinition[];
|
|
450
450
|
//#endregion
|
|
451
451
|
export { SemanticConceptJudgeInput as A, parseFindingSubject as C, ConceptFinding as D, createDspyRlmTraceEngine as E, RunScoreWeights as F, aggregateRunScore as I, clamp01 as L, SemanticConceptJudgeResult as M, runSemanticConceptJudge as N, ConceptSpec as O, RunScore as P, findingSubjectGrammarPromptFor as S, DspyRlmTraceEngineOptions as T, FINDING_SUBJECT_SYNTAX as _, FAILURE_MODE_KIND_SPEC as a, FindingSubjectStringSchema as b, emitControlIntegrityFindings as c, FindingsStore as d, PersistedFinding as f, FINDING_SUBJECT_KINDS as g, FINDING_SUBJECT_GRAMMAR_PROMPT as h, IMPROVEMENT_KIND_SPEC as i, SemanticConceptJudgeOptions as j, SEMANTIC_CONCEPT_JUDGE_VERSION as k, DiffPolicy as l, diffFindings as m, KNOWLEDGE_POISONING_KIND_SPEC as n, CONTROL_INTEGRITY_ANALYST as o, defaultIsMaterial as p, KNOWLEDGE_GAP_KIND_SPEC as r, ControlIntegrityAnalyst as s, DEFAULT_TRACE_ANALYST_KINDS as t, FindingsDiff as u, FindingSubject as v, renderFindingSubject as w, KIND_EXPECTED_SUBJECTS as x, FindingSubjectKind as y };
|
|
452
|
-
//# sourceMappingURL=index-
|
|
452
|
+
//# sourceMappingURL=index-aHDnhBC7.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-
|
|
1
|
+
{"version":3,"file":"index-aHDnhBC7.d.ts","names":[],"sources":["../src/run-score.ts","../src/semantic-concept-judge.ts","../src/analyst/trace-tool-callback.ts","../src/analyst/dspy-rlm-engine.ts","../src/analyst/finding-subject.ts","../src/analyst/findings-store.ts","../src/analyst/kinds/control-integrity.ts","../src/analyst/kinds/failure-mode.ts","../src/analyst/kinds/improvement.ts","../src/analyst/kinds/knowledge-gap.ts","../src/analyst/kinds/knowledge-poisoning.ts","../src/analyst/kinds/index.ts"],"mappings":";;;;;;;;;;;UAAiB;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;iBAiBc,kBAAkB,OAAO,UAAU,UAAS,QAAQ;iBAiBpD,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;KCLZ;UAEK;EACf;;EAEA;;EAEA;;EAEA,aAAa;;UAGE;EACf;EACA;;EAEA;EACA;EACA,UAAU;;UAGK;;EAEf;;EAEA;;EAEA,aAAa;IAAQ;IAAc;;;EAEnC,kBAAkB;;EAElB;EACA;;UAGe;EACf;EACA;;EAEA;EACA;EACA;EACA,UAAU;EACV;EACA;EACA;;EAEA;EACA;;;;;;;;KASU;UAQK;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,MAAM;EACN,aAAa;EACb;EACA,WAAW;EACX,SAAS;;;;;;EAMT,iBAAiB;;EAEjB,oBAAoB,QAAQ,OAAO;;cAKxB;;;;;;iBAkGS,wBACpB,OAAO,2BACP,UAAS,8BACR,QAAQ;;;KC5OC,0BAA0B;;;UCoBrB;;EAEf,MAAM;;EAEN;;EAEA,kBAAkB,aAAa;EAC/B;;EAEA,UAAU;;EAEV;;EAEA;;;;;;EAMA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,kBAAkB,QAAQ;;EAE1B;;;;;;;;;;EAUA;;EAEA,SAAS;;EAET;;;iBAIc,yBAAyB,SAAS,4BAA4B;;;;;;;;;;KCvClE;EAEN;EAAwB;EAAc;;EACtC;EAAyB;;EACzB;EAAuB;;EACvB;EAAyB;;EAGzB;EAAuB;;EACvB;EAAe;;EACf;EAAkB;EAAc;;EAChC;EAAkB;;EAClB;EAAa;EAAgB;;EAC7B;EAAc;;EACd;EAAkB;;EAClB;EAAkB;;EAClB;EAAwB;;EACxB;EAAuB;;EACvB;EAAc;;EACd;EAAa;EAAgB;;EAC7B;EAAgB;;EAChB;EAAqB;;EACrB;EAAuB;;EAEvB;EAA4B;;EAC5B;EAA2B;;EAE3B;EAAiB;;KAEX,qBAAqB;cAEpB,uBAAuB,cAAc;;;;;;;;;;;;;;;;;iBA2ClC,oBAAoB,iCAAiC;;;;;;;iBAoHrD,qBAAqB,GAAG;;;;;;;;;;cA8D3B,wBAAwB,SAAS,OAAO;cA0DxC;;;;;;;;;cAYA,wBAAwB,eAAe,cAAc;;iBAqElD,+BAA+B;;;;;;;;;;;;cAmBlC,4BAA0B,EAAA;;;;;;;;UC1atB,yBAAyB;EACxC;;cAGW;WAGiB;mBAFX;EAEjB,YAA4B;EAItB,OAAO,eAAe,UAAU,mBAAmB;;EAQzD,WAAW;;EAkBX,QAAQ,gBAAgB;;UAOT;;EAEf,UAAU;;EAEV,aAAa;;EAEb,WAAW;;;;;EAKX,SAAS;IAAQ,UAAU;IAAkB,SAAS;;;UAGvC;;;;;;;;EAQf,cAAc,UAAU,gBAAgB,SAAS;;;;;;iBAOnC,kBAAkB,GAAG,gBAAgB,GAAG;;;;;iBAWxC,aACd,UAAU,oBACV,SAAS,oBACT,SAAQ,aACP;;;;iBC7Fa,6BACd,OAAO,uBAAuB,mBAC9B,qBACC;;cA4BU,mCACA,oBAAoB,uBAAuB;WAE7C;WACA;WAEA;WACA;IAAS;IAAgC;;WACzC;WACA;aACP;aACA;;EAGI,QACJ,OAAO,uBAAuB,mBAC9B,KAAK,iBACJ,QAAQ;;cAUA,2BAAyB;;;cCHzB,wBAAwB;;;cC3BxB,uBAAuB;;;cCGvB,yBAAyB;;;cCPzB,+BAA+B;;;;;;;;;;;;cCP/B,sCAAsC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { a as JudgeError, c as ValidationError, i as ConfigError, n as AgentEvalErrorCode, s as NotFoundError, t as AgentEvalError } from "./errors-DEE6u6ot.js";
|
|
2
2
|
import { C as verifyAgentProfileCell, h as agentProfileCellKey, i as AgentProfileCellInput, l as AgentProfileJson, m as agentProfileCellHashMaterial, p as AgentProfileSourceInput, r as AgentProfileCell, s as AgentProfileDimensionValue, t as AGENT_PROFILE_KINDS, v as buildAgentProfileCell, x as toAgentProfileJson, y as groupRunsByAgentProfileCell } from "./agent-profile-cell-BkcRDikH.js";
|
|
3
3
|
import { C as defineEquivalenceCheck, S as buildEquivalenceRecord, _ as StrategyChecker, a as CheckerIdentity, b as VerificationStrategyProfile, c as EquivalenceCheckDefinition, d as EquivalenceCheckerInput, f as EquivalenceCheckerResult, g as EquivalenceRecord, h as EquivalenceProtocolError, i as equivalenceVerdict, l as EquivalenceCheckSpec, m as EquivalenceObligationStatus, n as VerdictCertification, o as CheckerOutcome, p as EquivalenceObligation, r as certificationEvidenceDigest, s as EquivalenceArm, t as DefaultVerdict, u as EquivalenceChecker, v as VERIFICATION_STRATEGIES, w as runEquivalenceCheck, x as VerificationStrategySource, y as VERIFICATION_STRATEGY_SOURCES } from "./verdict-E4eRNf7-.js";
|
|
4
|
-
import { a as MultiLayerVerifier, c as VerifyOptions, i as LayerStatus, l as gradeSemanticStatus, n as Layer, o as Severity, r as LayerResult, s as VerificationReport, t as Finding } from "./multi-layer-verifier-
|
|
5
|
-
import { A as SemanticConceptJudgeInput, D as ConceptFinding, E as createDspyRlmTraceEngine, F as RunScoreWeights, I as aggregateRunScore, L as clamp01, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, P as RunScore, T as DspyRlmTraceEngineOptions, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubject, y as FindingSubjectKind } from "./index-
|
|
4
|
+
import { a as MultiLayerVerifier, c as VerifyOptions, i as LayerStatus, l as gradeSemanticStatus, n as Layer, o as Severity, r as LayerResult, s as VerificationReport, t as Finding } from "./multi-layer-verifier-BUaQ4C17.js";
|
|
5
|
+
import { A as SemanticConceptJudgeInput, D as ConceptFinding, E as createDspyRlmTraceEngine, F as RunScoreWeights, I as aggregateRunScore, L as clamp01, M as SemanticConceptJudgeResult, N as runSemanticConceptJudge, O as ConceptSpec, P as RunScore, T as DspyRlmTraceEngineOptions, a as FAILURE_MODE_KIND_SPEC, d as FindingsStore, f as PersistedFinding, j as SemanticConceptJudgeOptions, k as SEMANTIC_CONCEPT_JUDGE_VERSION, l as DiffPolicy, m as diffFindings, t as DEFAULT_TRACE_ANALYST_KINDS, u as FindingsDiff, v as FindingSubject, y as FindingSubjectKind } from "./index-aHDnhBC7.js";
|
|
6
6
|
import { C as PendingCostCall, D as costForUsage, E as costForTokenPricing, O as modelPriceKey, S as PaidCallResult, T as RunPaidCallInput, _ as CostReservationExceededError, a as CostChannel, b as CustomTokenPricing, c as CostLedgerHandle, d as CostLedgerPersistenceError, f as CostLedgerSummary, g as CostReceiptInput, h as CostReceiptCaptureError, i as CostCeilingReachedError, l as CostLedgerOptions, m as CostReceipt, n as CostAccountingIncompleteError, o as CostLedger, p as CostProvenance, r as CostCallConflictError, s as CostLedgerFilter, t as ChannelRollup, u as CostLedgerPersistence, v as CostResult, w as PendingCostCallView, x as MaximumCharge, y as CostUsage } from "./cost-ledger-DbQdN3nO.js";
|
|
7
7
|
import { C as TraceEvent, O as isToolSpan, S as ToolSpan, T as isLlmSpan, _ as Span, a as FAILURE_CLASSES, c as JudgeSpan, f as Run, h as RunStatus, l as LlmSpan, n as BudgetLedgerEntry, o as FailureClass, r as BudgetSpec, s as GenericSpan, t as Artifact, w as isJudgeSpan } from "./schema-BtVldJ3T.js";
|
|
8
8
|
import { _ as validateRunRecord, a as RunRecord, c as RunTaskFailure, d as UNKNOWN_MODEL, f as isRunRecord, g as runTaskScore, h as roundTripRunRecord, i as RunOutcome, l as RunTerminalOutcome, m as parseRunRecordSafe, n as RunCostProvenance, o as RunRecordValidationError, p as modelHasSnapshot, r as RunJudgeMetadata, s as RunSplitTag, t as JudgeScoresRecord, u as RunTokenUsage } from "./run-record-DVV82Gwh.js";
|
|
@@ -35,10 +35,10 @@ import { o as InsightReport } from "./insight-report-BeT8KCgI.js";
|
|
|
35
35
|
import { C as analyzeRuns, b as AnalyzeRunsOptions, d as RawAnalystFinding, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-CIy18RTX.js";
|
|
36
36
|
import { o as MintedRolloutLine } from "./schema-Cef2cFmb.js";
|
|
37
37
|
import { i as BenchmarkEvaluation } from "./types-ColZlZtc.js";
|
|
38
|
-
import { A as RedTeamCategory, B as CanaryReport, F as scoreRedTeamOutput, I as CanaryAlert, L as CanaryEvaluation, M as RedTeamReport,
|
|
38
|
+
import { A as RedTeamCategory, B as CanaryReport, F as scoreRedTeamOutput, I as CanaryAlert, L as CanaryEvaluation, M as RedTeamReport, Mn as llmJudge, N as redTeamDataset, O as DEFAULT_RED_TEAM_CORPUS, P as redTeamReport, R as CanaryKind, V as runCanaries, _n as RunCampaignOptions, gn as CampaignCellFailureReceipt, j as RedTeamFinding, jn as LlmJudgeOptions, k as RedTeamCase, vn as runCampaign, z as CanaryOptions } from "./provenance-DA-Pmyfv.js";
|
|
39
39
|
import { a as paretoFrontier, i as dominates, n as Objective, r as ParetoResult } from "./pareto-BqNW3LJR.js";
|
|
40
40
|
import { n as SandboxDriver } from "./sandbox-harness-BlSOu4LX.js";
|
|
41
|
-
import { a as DefinedAgentEval, c as SelfImproveOptions, f as selfImprove, i as DefineAgentEvalOptions, o as defineAgentEval, u as SelfImproveResult } from "./define-agent-eval-
|
|
41
|
+
import { a as DefinedAgentEval, c as SelfImproveOptions, f as selfImprove, i as DefineAgentEvalOptions, o as defineAgentEval, u as SelfImproveResult } from "./define-agent-eval-BWuk2O_N.js";
|
|
42
42
|
import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-BhsrMupG.js";
|
|
43
43
|
import { n as TrajectoryStep, r as buildTrajectory, t as Trajectory } from "./trajectory-YC15QDYQ.js";
|
|
44
44
|
import { a as runCounterfactual, i as CounterfactualRunner, n as CounterfactualMutation, r as CounterfactualResult, t as CounterfactualContext } from "./counterfactual--bpysZF0.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
|
|
2
2
|
import { i as JudgeError, o as NotFoundError, r as ConfigError, s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
|
|
3
3
|
import { r as hashJson, t as canonicalize } from "./pre-registration-DakwTRXk.js";
|
|
4
|
-
import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-
|
|
4
|
+
import { a as CODING_HARNESSES, c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, n as completionVerdict, o as HARNESS_NATIVE_MODEL, r as createLlmCorrectnessChecker, s as agentProfileHash, t as extractProducedState, u as expandProfileAxes } from "./produced-state-C0oJ4vr-.js";
|
|
5
5
|
import { AGENT_PROFILE_KINDS, agentProfileCellHashMaterial, agentProfileCellKey, buildAgentProfileCell, groupRunsByAgentProfileCell, toAgentProfileJson, verifyAgentProfileCell } from "./profile-cell.js";
|
|
6
6
|
import { c as mulberry32 } from "./internal-BDHPCnjk.js";
|
|
7
7
|
import { a as spearmanR, i as ranks, n as partialCredit, o as weightedComposite, r as pearsonR, s as weightedMean, t as confidenceInterval } from "./descriptive-B5MwKfbf.js";
|
|
@@ -15,12 +15,12 @@ import { t as eProcess } from "./sequential-eprocess-CbUt2htw.js";
|
|
|
15
15
|
import { n as iqr, r as welchsTTest } from "./baseline-BhPRQBVn.js";
|
|
16
16
|
import { i as isLlmSpan, r as isJudgeSpan, s as isToolSpan, t as FAILURE_CLASSES } from "./schema-CRhEY1SO.js";
|
|
17
17
|
import { a as judgeSpans, c as runsForScenario, n as argHash } from "./query-Di7eEQ79.js";
|
|
18
|
-
import { i as analyzeRuns, o as checkCanaries, r as selfImprove, t as defineAgentEval } from "./define-agent-eval-
|
|
18
|
+
import { i as analyzeRuns, o as checkCanaries, r as selfImprove, t as defineAgentEval } from "./define-agent-eval-CvZQW4u9.js";
|
|
19
19
|
import { r as observedSplitScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
20
20
|
import { a as parseRunRecordSafe, c as validateRunRecord, i as modelHasSnapshot, n as UNKNOWN_MODEL, o as roundTripRunRecord, r as isRunRecord, s as runTaskScore, t as RunRecordValidationError } from "./run-record-D2lDdSAz.js";
|
|
21
21
|
import { a as summaryTable, n as gainHistogram, r as paretoChart } from "./summary-report-Blysd6Z2.js";
|
|
22
22
|
import { n as contentHash, r as fileVerdictCache, t as canonicalJson } from "./verdict-cache-mZf5FEiY.js";
|
|
23
|
-
import {
|
|
23
|
+
import { H as buildReflectionPrompt, J as DEFAULT_RED_TEAM_CORPUS, Q as runCanaries, U as parseReflectionResponse, X as redTeamReport, Y as redTeamDataset, Z as scoreRedTeamOutput, _ as paretoFrontier, at as assertRealBackend, et as runCampaign, g as dominates, it as BackendIntegrityError, k as surfaceContentHash, ot as summarizeBackendIntegrity, t as llmJudge } from "./llm-judge-DWq1Ptco.js";
|
|
24
24
|
import { a as resolveModelPricing, i as isModelPriced, n as estimateCost, r as estimateTokens, t as MODEL_PRICING } from "./metrics-Qv-cpptD.js";
|
|
25
25
|
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-B1qx30B4.js";
|
|
26
26
|
import { n as REDACTION_VERSION, r as redactString, t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
|
|
@@ -6293,11 +6293,9 @@ function softFail(expectedConcepts, start, error) {
|
|
|
6293
6293
|
/**
|
|
6294
6294
|
* Multi-layer verifier — ordered pipeline of verification layers.
|
|
6295
6295
|
*
|
|
6296
|
-
*
|
|
6297
|
-
*
|
|
6298
|
-
*
|
|
6299
|
-
* dependency-based skip, per-layer findings, soft-fail semantics, and
|
|
6300
|
-
* an aggregated `blendedScore` across all passed layers.
|
|
6296
|
+
* A DAG of layers (install → typecheck → build → lint → serve → semantic → …)
|
|
6297
|
+
* with dependency-based skip, per-layer findings, soft-fail semantics, and an
|
|
6298
|
+
* aggregated `blendedScore` across all passed layers.
|
|
6301
6299
|
*
|
|
6302
6300
|
* Use when you want:
|
|
6303
6301
|
* - ordered stages where a failing upstream stage skips downstream ones
|
|
@@ -6305,13 +6303,8 @@ function softFail(expectedConcepts, start, error) {
|
|
|
6305
6303
|
* - a single composite score across stages with per-stage weights
|
|
6306
6304
|
* - soft-fail stages whose failure doesn't abort the pipeline
|
|
6307
6305
|
*
|
|
6308
|
-
*
|
|
6309
|
-
*
|
|
6310
|
-
* - no inter-judge dependencies
|
|
6311
|
-
* - boolean `passed` per judge + overall
|
|
6312
|
-
*
|
|
6313
|
-
* Both primitives compose — JudgeRunner can be invoked as a single
|
|
6314
|
-
* layer inside a MultiLayerVerifier if that suits the caller.
|
|
6306
|
+
* A layer body is an arbitrary async function, so independent checks that need
|
|
6307
|
+
* no ordering run as one layer that resolves them in parallel.
|
|
6315
6308
|
*/
|
|
6316
6309
|
/**
|
|
6317
6310
|
* Grade a semantic-concept-style judge result into a single layer status.
|