@tangle-network/agent-eval 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/{analyze-runs-Da7P_Mc5.js → analyze-runs-ARGa3-a4.js} +2 -2
- package/dist/{analyze-runs-Da7P_Mc5.js.map → analyze-runs-ARGa3-a4.js.map} +1 -1
- package/dist/benchmarks/index.js +2 -2
- package/dist/builder-eval/index.d.ts +1 -1
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +2 -2
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-CqCWhH7v.js → campaign-VKtf8HRr.js} +3 -3
- package/dist/{campaign-CqCWhH7v.js.map → campaign-VKtf8HRr.js.map} +1 -1
- package/dist/contract/index.js +4 -4
- package/dist/{define-agent-eval-BT1SOjkZ.js → define-agent-eval-CGonw5ed.js} +3 -3
- package/dist/{define-agent-eval-BT1SOjkZ.js.map → define-agent-eval-CGonw5ed.js.map} +1 -1
- package/dist/diagnosis/index.js +1 -1
- package/dist/{eval-campaign-DVqiBoXR.js → eval-campaign-CkHn9y1S.js} +2 -2
- package/dist/{eval-campaign-DVqiBoXR.js.map → eval-campaign-CkHn9y1S.js.map} +1 -1
- package/dist/index.d.ts +2 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -6
- package/dist/index.js.map +1 -1
- package/dist/{judge-gate-C9sLLJ-v.js → judge-gate-DMlX38B7.js} +3 -9
- package/dist/judge-gate-DMlX38B7.js.map +1 -0
- package/dist/{llm-judge-BiUljZDR.js → llm-judge-CyDyqb6V.js} +3 -4
- package/dist/{llm-judge-BiUljZDR.js.map → llm-judge-CyDyqb6V.js.map} +1 -1
- package/dist/meta-eval/index.js +1 -1
- package/dist/openapi.json +1 -1
- package/dist/reporting.d.ts +9 -1
- package/dist/reporting.d.ts.map +1 -0
- package/dist/reporting.js +15 -2
- package/dist/reporting.js.map +1 -0
- package/dist/rl.js +1 -1
- package/dist/{sandbox-harness-BDr1GRm0.js → sandbox-harness-CzRlzJ5y.js} +29 -4
- package/dist/sandbox-harness-CzRlzJ5y.js.map +1 -0
- package/dist/{sandbox-harness-CdO35T2g.d.ts → sandbox-harness-K3LuyORm.d.ts} +8 -5
- package/dist/sandbox-harness-K3LuyORm.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-DpasRfXS.js → skillopt-optimization-method-O35aMdxU.js} +2 -2
- package/dist/{skillopt-optimization-method-DpasRfXS.js.map → skillopt-optimization-method-O35aMdxU.js.map} +1 -1
- package/dist/summary-report-0La5vejY.d.ts.map +1 -1
- package/dist/{summary-report-BstMpOfy.js → summary-report-CPkHWJWr.js} +40 -76
- package/dist/summary-report-CPkHWJWr.js.map +1 -0
- package/dist/wire/index.d.ts +2 -2
- package/docs/concepts.md +7 -0
- package/package.json +1 -1
- package/dist/judge-gate-C9sLLJ-v.js.map +0 -1
- package/dist/sandbox-harness-BDr1GRm0.js.map +0 -1
- package/dist/sandbox-harness-CdO35T2g.d.ts.map +0 -1
- package/dist/summary-report-BstMpOfy.js.map +0 -1
package/dist/meta-eval/index.js
CHANGED
|
@@ -6,7 +6,7 @@ import { s as runMetricExtractor } from "../query-CGXzO9d4.js";
|
|
|
6
6
|
import { n as callLlmJson } from "../llm-client-BdSiZ4qz.js";
|
|
7
7
|
import { a as validateObservationOptions, i as reduceOutcomeMetric, n as correlationSummary, r as hasVariation, t as assertUniqueObservationIds } from "../outcome-observations-ahbNtxuP.js";
|
|
8
8
|
import { a as rankedProbabilityScore, i as multiclassBrierScore, n as calibrationCurve, r as calibrationFromPairs, t as brierScore } from "../calibration-DLzckXnn.js";
|
|
9
|
-
import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, d as judgeSentinelReport, f as snapshotFromAgreement, g as withJudgeRetry, h as validateSentinelSnapshot, i as assertJudgeMayGate, l as evalHealthStamp, m as snapshotFromSentinelSet, n as JudgeGateRefusedError, o as measureJudgeAgreement, p as snapshotFromCalibration, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY, u as fileSentinelStore } from "../judge-gate-
|
|
9
|
+
import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, d as judgeSentinelReport, f as snapshotFromAgreement, g as withJudgeRetry, h as validateSentinelSnapshot, i as assertJudgeMayGate, l as evalHealthStamp, m as snapshotFromSentinelSet, n as JudgeGateRefusedError, o as measureJudgeAgreement, p as snapshotFromCalibration, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY, u as fileSentinelStore } from "../judge-gate-DMlX38B7.js";
|
|
10
10
|
import { t as computeInterval } from "../interval-Bd2f4CRT.js";
|
|
11
11
|
import { t as clusterBootstrapMean } from "../cluster-bootstrap-hNXESoc1.js";
|
|
12
12
|
import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-BdxXDN6h.js";
|
package/dist/openapi.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"openapi": "3.1.0",
|
|
3
3
|
"info": {
|
|
4
4
|
"title": "@tangle-network/agent-eval — wire protocol",
|
|
5
|
-
"version": "1.0.
|
|
5
|
+
"version": "1.0.6",
|
|
6
6
|
"description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
|
|
7
7
|
"contact": {
|
|
8
8
|
"name": "Tangle Network",
|
package/dist/reporting.d.ts
CHANGED
|
@@ -4,4 +4,12 @@ import { _ as evaluateReleaseConfidence, a as bootstrapCi, c as ReleaseConfidenc
|
|
|
4
4
|
import { a as RubricPredictiveValidityReport, i as RubricPredictiveValidityInput, o as RubricRanking, r as RubricOutcomePair, s as rubricPredictiveValidity } from "./rubric-predictive-validity-B3RzbwOx.js";
|
|
5
5
|
import { _ as gainHistogram, a as ResearchReportOptions, c as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as GainDistributionOptions, f as ParetoFigureSpec, g as SummaryTableRow, h as SummaryTableOptions, i as ResearchReportMethodology, l as GainDistributionBin, m as SummaryTable, n as ResearchReportCandidate, o as ResearchReportRecommendation, p as ParetoPoint, r as ResearchReportDecision, s as researchReport, t as ResearchReport, u as GainDistributionFigureSpec, v as summaryTable } from "./summary-report-0La5vejY.js";
|
|
6
6
|
import { a as PairedEvalueStep, c as SequentialDecision, i as PairedEvalueSequence, l as evaluateInterimReleaseConfidence, n as InterimReleaseConfidenceInput, r as PairedEvalueOptions, t as InterimReleaseConfidence, u as pairedEvalueSequence } from "./sequential-Bb_pdTqG.js";
|
|
7
|
-
|
|
7
|
+
/** A text report column; `none` leaves the cell unpadded. */
|
|
8
|
+
export interface ReportTableColumn {
|
|
9
|
+
header: string;
|
|
10
|
+
align?: 'left' | 'right' | 'none';
|
|
11
|
+
}
|
|
12
|
+
/** Render already-formatted measurements without changing values or missing-data labels. */
|
|
13
|
+
export declare function formatReportTable(columns: readonly ReportTableColumn[], rows: readonly (readonly string[])[]): string;
|
|
14
|
+
export { type BootstrapOptions, type BootstrapResult, type GainDistributionBin, type GainDistributionFigureSpec, type GainDistributionOptions, type InterimReleaseConfidence, type InterimReleaseConfidenceInput, type JudgeReplayGateArgs, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type ParetoFigureSpec, type ParetoPoint, RESEARCH_REPORT_HARD_PAIR_FLOOR, type ReleaseConfidenceAxis, type ReleaseConfidenceAxisName, type ReleaseConfidenceInput, type ReleaseConfidenceIssue, type ReleaseConfidenceMetrics, type ReleaseConfidenceScorecard, type ReleaseConfidenceStatus, type ReleaseConfidenceThresholds, type ReleaseTraceEvidence, type ResearchReport, type ResearchReportCandidate, type ResearchReportDecision, type ResearchReportMethodology, type ResearchReportOptions, type ResearchReportRecommendation, type RubricOutcomePair, type RubricPredictiveValidityInput, type RubricPredictiveValidityReport, type RubricRanking, type SequentialDecision, type SummaryTable, type SummaryTableOptions, type SummaryTableRow, type Verdict, benjaminiHochberg, bootstrapCi, evaluateInterimReleaseConfidence, evaluateReleaseConfidence, gainHistogram, judgeReplayGate, pairedBootstrap, pairedEvalueSequence, researchReport, rubricPredictiveValidity, summaryTable, wilcoxonSignedRank };
|
|
15
|
+
//# sourceMappingURL=reporting.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporting.d.ts","names":[],"sources":["../src/reporting.ts"],"mappings":";;;;;;;iBA0EiB;EACf;EACA;;;wBAIc,kBACd,kBAAkB,qBAClB"}
|
package/dist/reporting.js
CHANGED
|
@@ -1,8 +1,21 @@
|
|
|
1
|
-
import { i as summaryTable, l as RESEARCH_REPORT_HARD_PAIR_FLOOR, n as gainHistogram, t as researchReport } from "./summary-report-
|
|
1
|
+
import { i as summaryTable, l as RESEARCH_REPORT_HARD_PAIR_FLOOR, n as gainHistogram, t as researchReport } from "./summary-report-CPkHWJWr.js";
|
|
2
2
|
import { a as wilcoxonSignedRank } from "./paired-arms-C0Vd9bGG.js";
|
|
3
3
|
import { t as benjaminiHochberg } from "./multiplicity-DAd4xaDC.js";
|
|
4
4
|
import { r as pairedBootstrap } from "./paired-tests-kASEKpc-.js";
|
|
5
5
|
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Bjs5roiK.js";
|
|
6
6
|
import { n as bootstrapCi, r as judgeReplayGate, t as evaluateReleaseConfidence } from "./release-confidence--1iP-d3x.js";
|
|
7
7
|
import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-BdxXDN6h.js";
|
|
8
|
-
|
|
8
|
+
// src/reporting.ts
|
|
9
|
+
/** Render already-formatted measurements without changing values or missing-data labels. */
|
|
10
|
+
function formatReportTable(columns, rows) {
|
|
11
|
+
const table = [columns.map((column) => column.header), ...rows];
|
|
12
|
+
const widths = columns.map((_, index) => Math.max(...table.map((row) => (row[index] ?? "").length)));
|
|
13
|
+
return table.map((row) => columns.map((column, index) => {
|
|
14
|
+
const value = row[index] ?? "";
|
|
15
|
+
if (column.align === "none") return value;
|
|
16
|
+
return column.align === "right" ? value.padStart(widths[index]) : value.padEnd(widths[index]);
|
|
17
|
+
}).join(" ")).join("\n");
|
|
18
|
+
}
|
|
19
|
+
export { RESEARCH_REPORT_HARD_PAIR_FLOOR, benjaminiHochberg, bootstrapCi, evaluateInterimReleaseConfidence, evaluateReleaseConfidence, formatReportTable, gainHistogram, judgeReplayGate, pairedBootstrap, pairedEvalueSequence, researchReport, rubricPredictiveValidity, summaryTable, wilcoxonSignedRank };
|
|
20
|
+
|
|
21
|
+
//# sourceMappingURL=reporting.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporting.js","names":[],"sources":["../src/reporting.ts"],"sourcesContent":["export type {\n RubricOutcomePair,\n RubricPredictiveValidityInput,\n RubricPredictiveValidityReport,\n RubricRanking,\n} from './meta-eval/rubric-predictive-validity'\nexport { rubricPredictiveValidity } from './meta-eval/rubric-predictive-validity'\nexport type {\n BootstrapOptions,\n BootstrapResult,\n JudgeReplayGateArgs,\n Verdict,\n} from './promotion-gate'\nexport {\n bootstrapCi,\n judgeReplayGate,\n} from './promotion-gate'\nexport type {\n ReleaseConfidenceAxis,\n ReleaseConfidenceAxisName,\n ReleaseConfidenceInput,\n ReleaseConfidenceIssue,\n ReleaseConfidenceMetrics,\n ReleaseConfidenceScorecard,\n ReleaseConfidenceStatus,\n ReleaseConfidenceThresholds,\n ReleaseTraceEvidence,\n} from './release-confidence'\nexport { evaluateReleaseConfidence } from './release-confidence'\nexport type {\n InterimReleaseConfidence,\n InterimReleaseConfidenceInput,\n PairedEvalueOptions,\n PairedEvalueSequence,\n PairedEvalueStep,\n SequentialDecision,\n} from './sequential'\nexport {\n evaluateInterimReleaseConfidence,\n pairedEvalueSequence,\n} from './sequential'\nexport type {\n PairedBootstrapOptions,\n PairedBootstrapResult,\n} from './statistics'\nexport {\n benjaminiHochberg,\n pairedBootstrap,\n wilcoxonSignedRank,\n} from './statistics'\nexport type {\n GainDistributionBin,\n GainDistributionFigureSpec,\n GainDistributionOptions,\n ParetoFigureSpec,\n ParetoPoint,\n ResearchReport,\n ResearchReportCandidate,\n ResearchReportDecision,\n ResearchReportMethodology,\n ResearchReportOptions,\n ResearchReportRecommendation,\n SummaryTable,\n SummaryTableOptions,\n SummaryTableRow,\n} from './summary-report'\nexport {\n gainHistogram,\n RESEARCH_REPORT_HARD_PAIR_FLOOR,\n researchReport,\n summaryTable,\n} from './summary-report'\n\n/** A text report column; `none` leaves the cell unpadded. */\nexport interface ReportTableColumn {\n header: string\n align?: 'left' | 'right' | 'none'\n}\n\n/** Render already-formatted measurements without changing values or missing-data labels. */\nexport function formatReportTable(\n columns: readonly ReportTableColumn[],\n rows: readonly (readonly string[])[],\n): string {\n const table = [columns.map((column) => column.header), ...rows]\n const widths = columns.map((_, index) =>\n Math.max(...table.map((row) => (row[index] ?? '').length)),\n )\n return table\n .map((row) =>\n columns\n .map((column, index) => {\n const value = row[index] ?? ''\n if (column.align === 'none') return value\n return column.align === 'right'\n ? value.padStart(widths[index]!)\n : value.padEnd(widths[index]!)\n })\n .join(' '),\n )\n .join('\\n')\n}\n"],"mappings":";;;;;;;;;AAgFA,SAAgB,kBACd,SACA,MACQ;CACR,MAAM,QAAQ,CAAC,QAAQ,KAAK,WAAW,OAAO,MAAM,GAAG,GAAG,IAAI;CAC9D,MAAM,SAAS,QAAQ,KAAK,GAAG,UAC7B,KAAK,IAAI,GAAG,MAAM,KAAK,SAAS,IAAI,UAAU,GAAA,CAAI,MAAM,CAAC,CAC3D;CACA,OAAO,MACJ,KAAK,QACJ,QACG,KAAK,QAAQ,UAAU;EACtB,MAAM,QAAQ,IAAI,UAAU;EAC5B,IAAI,OAAO,UAAU,QAAQ,OAAO;EACpC,OAAO,OAAO,UAAU,UACpB,MAAM,SAAS,OAAO,MAAO,IAC7B,MAAM,OAAO,OAAO,MAAO;CACjC,CAAC,CAAC,CACD,KAAK,IAAI,CACd,CAAC,CACA,KAAK,IAAI;AACd"}
|
package/dist/rl.js
CHANGED
|
@@ -7,7 +7,7 @@ import { r as pairedBootstrap } from "./paired-tests-kASEKpc-.js";
|
|
|
7
7
|
import { t as decidePairedPromotion } from "./paired-promotion-decision-5Pl_qJLX.js";
|
|
8
8
|
import { n as campaignCellToRunRecord } from "./run-record-B-G2Tu11.js";
|
|
9
9
|
import { n as extractVerifiableReward, r as extractVerifiableRewardsFromRecords, t as detectRewardHacking } from "./reward-hacking-D-Vi9-fE.js";
|
|
10
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
10
|
+
import { t as runEvalCampaign } from "./eval-campaign-CkHn9y1S.js";
|
|
11
11
|
import { s as assertRewardGate } from "./schema-YUIXT8iW.js";
|
|
12
12
|
import { n as isTrainingLineEligible, o as toSftRows, t as isSplitEligible } from "./exporters-ByLcILYU.js";
|
|
13
13
|
import { t as mintRolloutRows } from "./mint-D9ZysYtY.js";
|
|
@@ -101,11 +101,33 @@ async function runSandboxScenario(spec, options = {}) {
|
|
|
101
101
|
assertions: [],
|
|
102
102
|
timings: []
|
|
103
103
|
};
|
|
104
|
+
const skipped = Symbol("scenario skipped");
|
|
105
|
+
let retiring = false;
|
|
104
106
|
let measurementFailures = 0;
|
|
105
107
|
const retirements = [];
|
|
106
108
|
const scope = {
|
|
107
109
|
signal,
|
|
108
|
-
check
|
|
110
|
+
check(...assertions) {
|
|
111
|
+
for (const value of assertions) report.assertions.push("status" in value ? value : {
|
|
112
|
+
...value[2],
|
|
113
|
+
name: value[0],
|
|
114
|
+
status: value[1] ? "pass" : "fail"
|
|
115
|
+
});
|
|
116
|
+
},
|
|
117
|
+
skip(name, message, metadata) {
|
|
118
|
+
if (retiring) throw new Error("Retirement cannot be skipped");
|
|
119
|
+
report.skipped = true;
|
|
120
|
+
report.metadata = {
|
|
121
|
+
skipReason: message,
|
|
122
|
+
...metadata
|
|
123
|
+
};
|
|
124
|
+
scope.check({
|
|
125
|
+
name,
|
|
126
|
+
status: "skip",
|
|
127
|
+
message
|
|
128
|
+
});
|
|
129
|
+
throw skipped;
|
|
130
|
+
},
|
|
109
131
|
defer: (retire) => retirements.push(retire),
|
|
110
132
|
async measure(name, run) {
|
|
111
133
|
const start = performance.now();
|
|
@@ -122,6 +144,7 @@ async function runSandboxScenario(spec, options = {}) {
|
|
|
122
144
|
measured.durationMs = Math.round(performance.now() - start);
|
|
123
145
|
}
|
|
124
146
|
} catch (error) {
|
|
147
|
+
if (error === skipped) throw error;
|
|
125
148
|
measured = {
|
|
126
149
|
name,
|
|
127
150
|
durationMs: Math.round(performance.now() - start),
|
|
@@ -129,6 +152,7 @@ async function runSandboxScenario(spec, options = {}) {
|
|
|
129
152
|
error: error instanceof Error ? error.message : String(error)
|
|
130
153
|
};
|
|
131
154
|
}
|
|
155
|
+
if (report.skipped && !retiring) throw skipped;
|
|
132
156
|
if (!measured.success) measurementFailures++;
|
|
133
157
|
report.timings.push(measured);
|
|
134
158
|
return measured;
|
|
@@ -173,8 +197,9 @@ async function runSandboxScenario(spec, options = {}) {
|
|
|
173
197
|
scope.check(...spec.expectations?.(state) ?? []);
|
|
174
198
|
report.metadata = spec.metadata?.(state);
|
|
175
199
|
} catch (error) {
|
|
176
|
-
report.error = error instanceof Error ? error.message : String(error);
|
|
200
|
+
if (error !== skipped) report.error = error instanceof Error ? error.message : String(error);
|
|
177
201
|
} finally {
|
|
202
|
+
retiring = true;
|
|
178
203
|
for (const retire of retirements.reverse()) {
|
|
179
204
|
const measured = await scope.measure("cleanup", retire);
|
|
180
205
|
if (!measured.success) scope.check({
|
|
@@ -184,9 +209,9 @@ async function runSandboxScenario(spec, options = {}) {
|
|
|
184
209
|
});
|
|
185
210
|
}
|
|
186
211
|
}
|
|
187
|
-
report.pass = !report.error && measurementFailures === 0 && report.assertions.every((a) => a.soft || a.status === "pass");
|
|
212
|
+
report.pass = !report.error && measurementFailures === 0 && report.assertions.every((a) => a.soft || a.status === "pass" || report.skipped && a.status === "skip");
|
|
188
213
|
return report;
|
|
189
214
|
}
|
|
190
215
|
export { runSandboxScenario as n, SandboxHarness as t };
|
|
191
216
|
|
|
192
|
-
//# sourceMappingURL=sandbox-harness-
|
|
217
|
+
//# sourceMappingURL=sandbox-harness-CzRlzJ5y.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sandbox-harness-CzRlzJ5y.js","names":["delay"],"sources":["../src/sandbox-harness.ts"],"sourcesContent":["import { setTimeout as delay } from 'node:timers/promises'\nimport { runBoundedProcess } from './bounded-process'\n\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n setupCommand?: string\n /** Run command (e.g. \"pnpm build\"). */\n runCommand?: string\n /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n testCommand?: string\n cwd?: string\n /** Max wall-clock per phase in ms. Default 10 minutes. */\n timeoutMs?: number\n /** Captured stdout + stderr cap, default 16 MiB; excess is dropped. */\n maxOutputBytes?: number\n image?: string\n env?: Record<string, string>\n /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n id: string\n parse(\n stdout: string,\n stderr: string,\n exitCode: number,\n ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n phase: 'setup' | 'run' | 'test'\n exitCode: number\n stdout: string\n stderr: string\n wallMs: number\n testsTotal?: number\n testsPassed?: number\n /** A timed-out phase always fails, even with exitCode 0. Absent/false means natural exit. */\n killedByTimeout?: boolean\n /** Captured output exceeded maxOutputBytes. */\n outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n id: string\n exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult>\n}\n\ninterface SubprocessSandboxDriverOptions {\n cwd?: string\n env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n id = 'subprocess'\n constructor(private options: SubprocessSandboxDriverOptions = {}) {}\n\n async exec(\n phase: SandboxResult['phase'],\n command: string,\n config: HarnessConfig,\n ): Promise<SandboxResult> {\n const result = await runBoundedProcess({\n command,\n shell: true,\n cwd: config.cwd ?? this.options.cwd,\n env: { ...process.env, ...this.options.env, ...config.env },\n envMode: 'replace',\n timeoutMs: config.timeoutMs ?? 10 * 60_000,\n maxOutputBytes: config.maxOutputBytes ?? 16 * 1024 * 1024,\n })\n // A timed-out phase never reaches the parser: the output of a killed\n // process is a fragment, and a fragment that happens to parse would\n // report a pass for a run that never finished.\n const parsed =\n !result.killedByTimeout && phase === 'test' && config.testParser\n ? config.testParser.parse(result.stdout, result.stderr, result.exitCode)\n : undefined\n return {\n phase,\n exitCode: result.exitCode,\n stdout: result.stdout,\n stderr: result.runnerError ? result.stderr + result.runnerError : result.stderr,\n wallMs: result.wallMs,\n testsTotal: parsed?.testsTotal,\n testsPassed: parsed?.testsPassed,\n killedByTimeout: result.killedByTimeout || undefined,\n outputTruncated: result.outputTruncated || undefined,\n }\n }\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n passed: boolean\n setup?: SandboxResult\n run?: SandboxResult\n test?: SandboxResult\n totalWallMs: number\n /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n score: number\n}\n\nexport class SandboxHarness {\n constructor(private driver: SandboxDriver = new SubprocessSandboxDriver()) {}\n\n async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n const handle = await emitter.sandbox({\n name: `sandbox(${this.driver.id})`,\n image: config.image,\n command: [config.setupCommand, config.runCommand, config.testCommand]\n .filter(Boolean)\n .join(' && '),\n })\n const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n try {\n for (const phase of ['setup', 'run', 'test'] as const) {\n const command = config[`${phase}Command`]\n if (!command) continue\n const measured = await this.driver.exec(phase, command, config)\n result[phase] = measured\n result.totalWallMs += measured.wallMs\n const passed = !measured.killedByTimeout && measured.exitCode === 0\n if (phase !== 'test' && !passed) {\n await handle.fail(\n measured.killedByTimeout\n ? `${phase} timed out after ${measured.wallMs}ms`\n : `${phase} failed (exit ${measured.exitCode})`,\n {\n exitCode: measured.exitCode,\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>,\n )\n return result\n }\n }\n const test = result.test\n result.passed = !test || (!test.killedByTimeout && test.exitCode === 0)\n result.score =\n test?.testsTotal !== undefined && test.testsTotal > 0\n ? (test.testsPassed ?? 0) / test.testsTotal\n : result.passed\n ? 1\n : 0\n await handle.end({\n exitCode: test?.exitCode,\n testsTotal: test?.testsTotal,\n testsPassed: test?.testsPassed,\n status: result.passed ? 'ok' : 'error',\n wallMs: result.totalWallMs,\n } as Partial<SandboxSpan>)\n } catch (err) {\n await handle.fail(err instanceof Error ? err : String(err))\n throw err\n }\n return result\n }\n}\n\n/** Portable integration report; products may retain extra timing and assertion fields. */\nexport interface HarnessAssertion {\n name: string\n status: 'pass' | 'fail' | 'skip'\n expected?: string\n actual?: string\n message?: string\n soft?: boolean\n}\nexport type HarnessExpectation =\n | HarnessAssertion\n | readonly [name: string, passed: boolean, details?: Omit<HarnessAssertion, 'name' | 'status'>]\nexport interface HarnessTiming {\n name: string\n durationMs: number\n success: boolean\n error?: string\n}\nexport interface HarnessReport {\n pass: boolean\n skipped?: boolean\n assertions: HarnessAssertion[]\n timings: HarnessTiming[]\n error?: string\n metadata?: Record<string, unknown>\n}\nexport interface HarnessResource<T> {\n name: string\n create(signal: AbortSignal): Promise<T>\n retire(resource: T): Promise<unknown>\n ready?: (resource: T, signal: AbortSignal) => Promise<unknown>\n onCreate?: (resource: T) => void\n}\nexport interface HarnessScope {\n signal: AbortSignal\n resource<T>(input: HarnessResource<T>): Promise<{ resource?: T; timing: HarnessTiming }>\n check(...assertions: HarnessExpectation[]): void\n skip(name: string, message: string, metadata?: Record<string, unknown>): never\n measure(name: string, run: () => Promise<unknown>): Promise<HarnessTiming>\n /** Register retirement immediately after allocation; cleanup ignores workload cancellation. */\n defer(retire: () => Promise<unknown>): void\n poll<T>(\n read: () => Promise<T>,\n ready: (value: T) => boolean,\n budget: { timeoutMs: number; intervalMs: number },\n ): Promise<T | undefined>\n}\nexport interface HarnessScenario<T> {\n setup(scope: HarnessScope): Promise<T>\n steps?: ReadonlyArray<{\n name: string\n /** Diagnostic measurements retain failures without gating the scenario. */\n advisory?: boolean\n when?: (state: T) => boolean\n run(state: T, scope: HarnessScope): Promise<unknown>\n expect?: (state: T, timing: HarnessTiming) => HarnessExpectation[]\n }>\n expectations?: (state: T) => HarnessExpectation[]\n metadata?: (state: T) => Record<string, unknown>\n /** Includes setup and steps. Transports must honor scope.signal. */\n budgetMs: number\n}\n\n/** Campaigns own whole-scenario retries; polling only repeats the declared read. */\nexport async function runSandboxScenario<T>(\n spec: HarnessScenario<T>,\n options: {\n signal?: AbortSignal\n measure?: HarnessScope['measure']\n } = {},\n): Promise<HarnessReport> {\n if (!(spec.budgetMs > 0 && Number.isSafeInteger(spec.budgetMs)))\n throw new Error('Scenario budgetMs must be positive and finite')\n const signal = AbortSignal.any([\n AbortSignal.timeout(spec.budgetMs),\n ...(options.signal ? [options.signal] : []),\n ])\n const report: HarnessReport = { pass: false, assertions: [], timings: [] }\n const skipped = Symbol('scenario skipped')\n let retiring = false\n let measurementFailures = 0\n const retirements: Array<() => Promise<unknown>> = []\n const scope: HarnessScope = {\n signal,\n check(...assertions) {\n for (const value of assertions)\n report.assertions.push(\n 'status' in value\n ? value\n : {\n ...value[2],\n name: value[0],\n status: value[1] ? 'pass' : 'fail',\n },\n )\n },\n skip(name, message, metadata) {\n if (retiring) throw new Error('Retirement cannot be skipped')\n report.skipped = true\n report.metadata = { skipReason: message, ...metadata }\n scope.check({ name, status: 'skip', message })\n throw skipped\n },\n defer: (retire) => retirements.push(retire),\n async measure(name, run) {\n const start = performance.now()\n let measured: HarnessTiming = { name, durationMs: 0, success: true }\n try {\n if (options.measure) measured = await options.measure(name, run)\n else {\n const result = (await run()) as { success?: boolean; error?: string } | undefined\n if (result?.success === false)\n throw new Error(result.error ?? 'measured operation returned success=false')\n measured.durationMs = Math.round(performance.now() - start)\n }\n } catch (error) {\n if (error === skipped) throw error\n measured = {\n name,\n durationMs: Math.round(performance.now() - start),\n success: false,\n error: error instanceof Error ? error.message : String(error),\n }\n }\n if (report.skipped && !retiring) throw skipped\n if (!measured.success) measurementFailures++\n report.timings.push(measured)\n return measured\n },\n async resource<T>(input: HarnessResource<T>) {\n let resource: T | undefined\n const timing = await scope.measure(input.name, async () => {\n const owned = await input.create(signal)\n resource = owned\n scope.defer(() => input.retire(owned))\n input.onCreate?.(owned)\n await input.ready?.(owned, signal)\n })\n return { resource, timing }\n },\n async poll(read, ready, budget) {\n if (!(budget.timeoutMs > 0 && budget.intervalMs > 0))\n throw new Error('Polling timeout and interval must be positive')\n const deadline = performance.now() + budget.timeoutMs\n while (performance.now() < deadline) {\n signal.throwIfAborted()\n const value = await read()\n if (ready(value)) return value\n await delay(\n Math.min(budget.intervalMs, Math.max(0, deadline - performance.now())),\n undefined,\n { signal },\n )\n }\n return undefined\n },\n }\n try {\n signal.throwIfAborted()\n const state = await spec.setup(scope)\n for (const step of spec.steps ?? []) {\n signal.throwIfAborted()\n if (step.when && !step.when(state)) continue\n const failuresBefore = measurementFailures\n const timing = await scope.measure(step.name, () => step.run(state, scope))\n if (step.advisory) measurementFailures = failuresBefore\n scope.check(...(step.expect?.(state, timing) ?? []))\n }\n signal.throwIfAborted()\n scope.check(...(spec.expectations?.(state) ?? []))\n report.metadata = spec.metadata?.(state)\n } catch (error) {\n if (error !== skipped) report.error = error instanceof Error ? error.message : String(error)\n } finally {\n retiring = true\n for (const retire of retirements.reverse()) {\n const measured = await scope.measure('cleanup', retire)\n if (!measured.success)\n scope.check({ name: 'cleanup', status: 'fail', message: measured.error })\n }\n }\n report.pass =\n !report.error &&\n measurementFailures === 0 &&\n report.assertions.every(\n (a) => a.soft || a.status === 'pass' || (report.skipped && a.status === 'skip'),\n )\n return report\n}\n"],"mappings":";;;AA6DA,IAAa,0BAAb,MAA8D;CAExC;CADpB,KAAK;CACL,YAAY,UAAkD,CAAC,GAAG;EAA9C,KAAA,UAAA;CAA+C;CAEnE,MAAM,KACJ,OACA,SACA,QACwB;EACxB,MAAM,SAAS,MAAM,kBAAkB;GACrC;GACA,OAAO;GACP,KAAK,OAAO,OAAO,KAAK,QAAQ;GAChC,KAAK;IAAE,GAAG,QAAQ;IAAK,GAAG,KAAK,QAAQ;IAAK,GAAG,OAAO;GAAI;GAC1D,SAAS;GACT,WAAW,OAAO,aAAa;GAC/B,gBAAgB,OAAO,kBAAkB;EAC3C,CAAC;EAID,MAAM,SACJ,CAAC,OAAO,mBAAmB,UAAU,UAAU,OAAO,aAClD,OAAO,WAAW,MAAM,OAAO,QAAQ,OAAO,QAAQ,OAAO,QAAQ,IACrE,KAAA;EACN,OAAO;GACL;GACA,UAAU,OAAO;GACjB,QAAQ,OAAO;GACf,QAAQ,OAAO,cAAc,OAAO,SAAS,OAAO,cAAc,OAAO;GACzE,QAAQ,OAAO;GACf,YAAY,QAAQ;GACpB,aAAa,QAAQ;GACrB,iBAAiB,OAAO,mBAAmB,KAAA;GAC3C,iBAAiB,OAAO,mBAAmB,KAAA;EAC7C;CACF;AACF;AAcA,IAAa,iBAAb,MAA4B;CACN;CAApB,YAAY,SAAgC,IAAI,wBAAwB,GAAG;EAAvD,KAAA,SAAA;CAAwD;CAE5E,MAAM,IAAI,QAAuB,SAAsD;EACrF,MAAM,SAAS,MAAM,QAAQ,QAAQ;GACnC,MAAM,WAAW,KAAK,OAAO,GAAG;GAChC,OAAO,OAAO;GACd,SAAS;IAAC,OAAO;IAAc,OAAO;IAAY,OAAO;GAAW,CAAC,CAClE,OAAO,OAAO,CAAC,CACf,KAAK,MAAM;EAChB,CAAC;EACD,MAAM,SAA+B;GAAE,QAAQ;GAAO,aAAa;GAAG,OAAO;EAAE;EAC/E,IAAI;GACF,KAAK,MAAM,SAAS;IAAC;IAAS;IAAO;GAAM,GAAY;IACrD,MAAM,UAAU,OAAO,GAAG,MAAM;IAChC,IAAI,CAAC,SAAS;IACd,MAAM,WAAW,MAAM,KAAK,OAAO,KAAK,OAAO,SAAS,MAAM;IAC9D,OAAO,SAAS;IAChB,OAAO,eAAe,SAAS;IAC/B,MAAM,SAAS,CAAC,SAAS,mBAAmB,SAAS,aAAa;IAClE,IAAI,UAAU,UAAU,CAAC,QAAQ;KAC/B,MAAM,OAAO,KACX,SAAS,kBACL,GAAG,MAAM,mBAAmB,SAAS,OAAO,MAC5C,GAAG,MAAM,gBAAgB,SAAS,SAAS,IAC/C;MACE,UAAU,SAAS;MACnB,QAAQ,OAAO;KACjB,CACF;KACA,OAAO;IACT;GACF;GACA,MAAM,OAAO,OAAO;GACpB,OAAO,SAAS,CAAC,QAAS,CAAC,KAAK,mBAAmB,KAAK,aAAa;GACrE,OAAO,QACL,MAAM,eAAe,KAAA,KAAa,KAAK,aAAa,KAC/C,KAAK,eAAe,KAAK,KAAK,aAC/B,OAAO,SACL,IACA;GACR,MAAM,OAAO,IAAI;IACf,UAAU,MAAM;IAChB,YAAY,MAAM;IAClB,aAAa,MAAM;IACnB,QAAQ,OAAO,SAAS,OAAO;IAC/B,QAAQ,OAAO;GACjB,CAAyB;EAC3B,SAAS,KAAK;GACZ,MAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;GAC1D,MAAM;EACR;EACA,OAAO;CACT;AACF;;AAkEA,eAAsB,mBACpB,MACA,UAGI,CAAC,GACmB;CACxB,IAAI,EAAE,KAAK,WAAW,KAAK,OAAO,cAAc,KAAK,QAAQ,IAC3D,MAAM,IAAI,MAAM,+CAA+C;CACjE,MAAM,SAAS,YAAY,IAAI,CAC7B,YAAY,QAAQ,KAAK,QAAQ,GACjC,GAAI,QAAQ,SAAS,CAAC,QAAQ,MAAM,IAAI,CAAC,CAC3C,CAAC;CACD,MAAM,SAAwB;EAAE,MAAM;EAAO,YAAY,CAAC;EAAG,SAAS,CAAC;CAAE;CACzE,MAAM,UAAU,OAAO,kBAAkB;CACzC,IAAI,WAAW;CACf,IAAI,sBAAsB;CAC1B,MAAM,cAA6C,CAAC;CACpD,MAAM,QAAsB;EAC1B;EACA,MAAM,GAAG,YAAY;GACnB,KAAK,MAAM,SAAS,YAClB,OAAO,WAAW,KAChB,YAAY,QACR,QACA;IACE,GAAG,MAAM;IACT,MAAM,MAAM;IACZ,QAAQ,MAAM,KAAK,SAAS;GAC9B,CACN;EACJ;EACA,KAAK,MAAM,SAAS,UAAU;GAC5B,IAAI,UAAU,MAAM,IAAI,MAAM,8BAA8B;GAC5D,OAAO,UAAU;GACjB,OAAO,WAAW;IAAE,YAAY;IAAS,GAAG;GAAS;GACrD,MAAM,MAAM;IAAE;IAAM,QAAQ;IAAQ;GAAQ,CAAC;GAC7C,MAAM;EACR;EACA,QAAQ,WAAW,YAAY,KAAK,MAAM;EAC1C,MAAM,QAAQ,MAAM,KAAK;GACvB,MAAM,QAAQ,YAAY,IAAI;GAC9B,IAAI,WAA0B;IAAE;IAAM,YAAY;IAAG,SAAS;GAAK;GACnE,IAAI;IACF,IAAI,QAAQ,SAAS,WAAW,MAAM,QAAQ,QAAQ,MAAM,GAAG;SAC1D;KACH,MAAM,SAAU,MAAM,IAAI;KAC1B,IAAI,QAAQ,YAAY,OACtB,MAAM,IAAI,MAAM,OAAO,SAAS,2CAA2C;KAC7E,SAAS,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;IAC5D;GACF,SAAS,OAAO;IACd,IAAI,UAAU,SAAS,MAAM;IAC7B,WAAW;KACT;KACA,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;KAChD,SAAS;KACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;IAC9D;GACF;GACA,IAAI,OAAO,WAAW,CAAC,UAAU,MAAM;GACvC,IAAI,CAAC,SAAS,SAAS;GACvB,OAAO,QAAQ,KAAK,QAAQ;GAC5B,OAAO;EACT;EACA,MAAM,SAAY,OAA2B;GAC3C,IAAI;GACJ,MAAM,SAAS,MAAM,MAAM,QAAQ,MAAM,MAAM,YAAY;IACzD,MAAM,QAAQ,MAAM,MAAM,OAAO,MAAM;IACvC,WAAW;IACX,MAAM,YAAY,MAAM,OAAO,KAAK,CAAC;IACrC,MAAM,WAAW,KAAK;IACtB,MAAM,MAAM,QAAQ,OAAO,MAAM;GACnC,CAAC;GACD,OAAO;IAAE;IAAU;GAAO;EAC5B;EACA,MAAM,KAAK,MAAM,OAAO,QAAQ;GAC9B,IAAI,EAAE,OAAO,YAAY,KAAK,OAAO,aAAa,IAChD,MAAM,IAAI,MAAM,+CAA+C;GACjE,MAAM,WAAW,YAAY,IAAI,IAAI,OAAO;GAC5C,OAAO,YAAY,IAAI,IAAI,UAAU;IACnC,OAAO,eAAe;IACtB,MAAM,QAAQ,MAAM,KAAK;IACzB,IAAI,MAAM,KAAK,GAAG,OAAO;IACzB,MAAMA,WACJ,KAAK,IAAI,OAAO,YAAY,KAAK,IAAI,GAAG,WAAW,YAAY,IAAI,CAAC,CAAC,GACrE,KAAA,GACA,EAAE,OAAO,CACX;GACF;EAEF;CACF;CACA,IAAI;EACF,OAAO,eAAe;EACtB,MAAM,QAAQ,MAAM,KAAK,MAAM,KAAK;EACpC,KAAK,MAAM,QAAQ,KAAK,SAAS,CAAC,GAAG;GACnC,OAAO,eAAe;GACtB,IAAI,KAAK,QAAQ,CAAC,KAAK,KAAK,KAAK,GAAG;GACpC,MAAM,iBAAiB;GACvB,MAAM,SAAS,MAAM,MAAM,QAAQ,KAAK,YAAY,KAAK,IAAI,OAAO,KAAK,CAAC;GAC1E,IAAI,KAAK,UAAU,sBAAsB;GACzC,MAAM,MAAM,GAAI,KAAK,SAAS,OAAO,MAAM,KAAK,CAAC,CAAE;EACrD;EACA,OAAO,eAAe;EACtB,MAAM,MAAM,GAAI,KAAK,eAAe,KAAK,KAAK,CAAC,CAAE;EACjD,OAAO,WAAW,KAAK,WAAW,KAAK;CACzC,SAAS,OAAO;EACd,IAAI,UAAU,SAAS,OAAO,QAAQ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;CAC7F,UAAU;EACR,WAAW;EACX,KAAK,MAAM,UAAU,YAAY,QAAQ,GAAG;GAC1C,MAAM,WAAW,MAAM,MAAM,QAAQ,WAAW,MAAM;GACtD,IAAI,CAAC,SAAS,SACZ,MAAM,MAAM;IAAE,MAAM;IAAW,QAAQ;IAAQ,SAAS,SAAS;GAAM,CAAC;EAC5E;CACF;CACA,OAAO,OACL,CAAC,OAAO,SACR,wBAAwB,KACxB,OAAO,WAAW,OACf,MAAM,EAAE,QAAQ,EAAE,WAAW,UAAW,OAAO,WAAW,EAAE,WAAW,MAC1E;CACF,OAAO;AACT"}
|
|
@@ -57,6 +57,7 @@ interface HarnessAssertion {
|
|
|
57
57
|
message?: string;
|
|
58
58
|
soft?: boolean;
|
|
59
59
|
}
|
|
60
|
+
type HarnessExpectation = HarnessAssertion | readonly [name: string, passed: boolean, details?: Omit<HarnessAssertion, 'name' | 'status'>];
|
|
60
61
|
interface HarnessTiming {
|
|
61
62
|
name: string;
|
|
62
63
|
durationMs: number;
|
|
@@ -65,6 +66,7 @@ interface HarnessTiming {
|
|
|
65
66
|
}
|
|
66
67
|
interface HarnessReport {
|
|
67
68
|
pass: boolean;
|
|
69
|
+
skipped?: boolean;
|
|
68
70
|
assertions: HarnessAssertion[];
|
|
69
71
|
timings: HarnessTiming[];
|
|
70
72
|
error?: string;
|
|
@@ -83,7 +85,8 @@ interface HarnessScope {
|
|
|
83
85
|
resource?: T;
|
|
84
86
|
timing: HarnessTiming;
|
|
85
87
|
}>;
|
|
86
|
-
check(...assertions:
|
|
88
|
+
check(...assertions: HarnessExpectation[]): void;
|
|
89
|
+
skip(name: string, message: string, metadata?: Record<string, unknown>): never;
|
|
87
90
|
measure(name: string, run: () => Promise<unknown>): Promise<HarnessTiming>;
|
|
88
91
|
/** Register retirement immediately after allocation; cleanup ignores workload cancellation. */
|
|
89
92
|
defer(retire: () => Promise<unknown>): void;
|
|
@@ -100,9 +103,9 @@ interface HarnessScenario<T> {
|
|
|
100
103
|
advisory?: boolean;
|
|
101
104
|
when?: (state: T) => boolean;
|
|
102
105
|
run(state: T, scope: HarnessScope): Promise<unknown>;
|
|
103
|
-
expect?: (state: T, timing: HarnessTiming) =>
|
|
106
|
+
expect?: (state: T, timing: HarnessTiming) => HarnessExpectation[];
|
|
104
107
|
}>;
|
|
105
|
-
expectations?: (state: T) =>
|
|
108
|
+
expectations?: (state: T) => HarnessExpectation[];
|
|
106
109
|
metadata?: (state: T) => Record<string, unknown>;
|
|
107
110
|
/** Includes setup and steps. Transports must honor scope.signal. */
|
|
108
111
|
budgetMs: number;
|
|
@@ -112,5 +115,5 @@ declare function runSandboxScenario<T>(spec: HarnessScenario<T>, options?: {
|
|
|
112
115
|
signal?: AbortSignal;
|
|
113
116
|
measure?: HarnessScope['measure'];
|
|
114
117
|
}): Promise<HarnessReport>;
|
|
115
|
-
export {
|
|
116
|
-
//# sourceMappingURL=sandbox-harness-
|
|
118
|
+
export { HarnessScenario as a, SandboxDriver as c, HarnessReport as i, SandboxHarnessResult as l, HarnessConfig as n, HarnessScope as o, HarnessExpectation as r, HarnessTiming as s, HarnessAssertion as t, runSandboxScenario as u };
|
|
119
|
+
//# sourceMappingURL=sandbox-harness-K3LuyORm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sandbox-harness-K3LuyORm.d.ts","names":[],"sources":["../src/sandbox-harness.ts"],"mappings":"A,UAMiB;;EAEf;;EAEA;;EAEA;EACA;;EAEA;;EAEA;EACA;EACA,MAAM;;EAEN,aAAa;;UAGE;EACf;EACA,MACE,gBACA,gBACA;IACG;IAAoB;;;UAGV;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA,KACE,OAAO,wBACP,iBACA,QAAQ,gBACP,QAAQ;;UAiDI;EACf;EACA,QAAQ;EACR,MAAM;EACN,OAAO;EACP;;EAEA;;;UA4De;EACf;EACA;EACA;EACA;EACA;EACA;;KAEU,qBACR,6BACU,cAAc,iBAAiB,UAAU,KAAK;UAC3C;EACf;EACA;EACA;EACA;;UAEe;EACf;EACA;EACA,YAAY;EACZ,SAAS;EACT;EACA,WAAW;;UAEI,gBAAgB;EAC/B;EACA,OAAO,QAAQ,cAAc,QAAQ;EACrC,OAAO,UAAU,IAAI;EACrB,SAAS,UAAU,GAAG,QAAQ,gBAAgB;EAC9C,YAAY,UAAU;;UAEP;EACf,QAAQ;EACR,SAAS,GAAG,OAAO,gBAAgB,KAAK;IAAU,WAAW;IAAG,QAAQ;;EACxE,SAAS,YAAY;EACrB,KAAK,cAAc,iBAAiB,WAAW;EAC/C,QAAQ,cAAc,WAAW,mBAAmB,QAAQ;;EAE5D,MAAM,cAAc;EACpB,KAAK,GACH,YAAY,QAAQ,IACpB,QAAQ,OAAO,eACf;IAAU;IAAmB;MAC5B,QAAQ;;UAEI,gBAAgB;EAC/B,MAAM,OAAO,eAAe,QAAQ;EACpC,QAAQ;IACN;;IAEA;IACA,QAAQ,OAAO;IACf,IAAI,OAAO,GAAG,OAAO,eAAe;IACpC,UAAU,OAAO,GAAG,QAAQ,kBAAkB;;EAEhD,gBAAgB,OAAO,MAAM;EAC7B,YAAY,OAAO,MAAM;;EAEzB;;;iBAIoB,mBAAmB,GACvC,MAAM,gBAAgB,IACtB;EACE,SAAS;EACT,UAAU;IAEX,QAAQ"}
|
package/dist/{skillopt-optimization-method-DpasRfXS.js → skillopt-optimization-method-O35aMdxU.js}
RENAMED
|
@@ -4,7 +4,7 @@ import { t as decidePairedPromotion } from "./paired-promotion-decision-5Pl_qJLX
|
|
|
4
4
|
import { a as heldoutSignificance, o as pairHoldout } from "./statistical-heldout-DYF5f10K.js";
|
|
5
5
|
import { d as mapConcurrent } from "./journal-C1uLJi0w.js";
|
|
6
6
|
import { n as contentHash, t as canonicalJson } from "./verdict-cache-BVH-fl18.js";
|
|
7
|
-
import { $ as resolveRunDir, C as surfaceDiff, G as assertIndependentEvaluationSplit, J as exposeFinalEvidence, K as captureFinalEvidencePolicy, R as captureJudge, T as assertSearchHistoryAdmissionOptions, Y as reserveFinalEvidence, Z as runCampaign, b as SearchRecorder, c as readGepaCandidatePopulationArtifact, d as optimizationTokenUsageFromSummary, l as combineComparisonCosts, o as executeOptimizationMethod, q as evaluationUnitMap, s as assertGepaCandidatePopulationSummary, u as costFromLedgerSummary, w as surfaceNode, x as developmentClaim } from "./llm-judge-
|
|
7
|
+
import { $ as resolveRunDir, C as surfaceDiff, G as assertIndependentEvaluationSplit, J as exposeFinalEvidence, K as captureFinalEvidencePolicy, R as captureJudge, T as assertSearchHistoryAdmissionOptions, Y as reserveFinalEvidence, Z as runCampaign, b as SearchRecorder, c as readGepaCandidatePopulationArtifact, d as optimizationTokenUsageFromSummary, l as combineComparisonCosts, o as executeOptimizationMethod, q as evaluationUnitMap, s as assertGepaCandidatePopulationSummary, u as costFromLedgerSummary, w as surfaceNode, x as developmentClaim } from "./llm-judge-CyDyqb6V.js";
|
|
8
8
|
import { D as assertCompleteCampaign, T as assertCampaignDesign, f as campaignBreakdown, t as createCampaignEvidenceReceipt } from "./campaign-evidence-BICB3a9Z.js";
|
|
9
9
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CK8-2W7p.js";
|
|
10
10
|
import { M as fsCampaignStorage, _ as resolveExternalOptimizerCallbackLimits, d as assertExternalOptimizerModelBudget, f as assertJsonValue, g as removeCredentialEnvironment, h as isExternalTextCandidate, i as startExternalOptimizerModelProxy, j as createRunCostLedger, m as isCandidateText, n as closeExternalOptimizerResources, p as assertNoCredentialValues, r as runWithCleanup, t as runExternalOptimizerProcess, v as resolveExternalOptimizerProcessLimits, y as safePathComponent } from "./external-optimizer-subprocess-CeHuAaMJ.js";
|
|
@@ -3223,4 +3223,4 @@ async function finishSkillOptRun(context, result, outputDir, modelProxy) {
|
|
|
3223
3223
|
}
|
|
3224
3224
|
export { importExternalEvaluations as a, composeGate as c, readExternalOptimizerObservationArtifact as d, externalSurface as i, externalTextOptimizationMethod as l, compareOptimizationMethods as n, importGepaPopulation as o, gepaOptimizationMethod as r, heldOutGate as s, skillOptOptimizationMethod as t, decodeExternalTextCandidate as u };
|
|
3225
3225
|
|
|
3226
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
3226
|
+
//# sourceMappingURL=skillopt-optimization-method-O35aMdxU.js.map
|