@miller-tech/uap 1.50.1 → 1.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/benchmarks/paired/ablation.d.ts +36 -0
- package/dist/benchmarks/paired/ablation.d.ts.map +1 -0
- package/dist/benchmarks/paired/ablation.js +96 -0
- package/dist/benchmarks/paired/ablation.js.map +1 -0
- package/dist/benchmarks/paired/adapter.d.ts +95 -0
- package/dist/benchmarks/paired/adapter.d.ts.map +1 -0
- package/dist/benchmarks/paired/adapter.js +456 -0
- package/dist/benchmarks/paired/adapter.js.map +1 -0
- package/dist/benchmarks/paired/index.d.ts +17 -0
- package/dist/benchmarks/paired/index.d.ts.map +1 -0
- package/dist/benchmarks/paired/index.js +17 -0
- package/dist/benchmarks/paired/index.js.map +1 -0
- package/dist/benchmarks/paired/report.d.ts +61 -0
- package/dist/benchmarks/paired/report.d.ts.map +1 -0
- package/dist/benchmarks/paired/report.js +204 -0
- package/dist/benchmarks/paired/report.js.map +1 -0
- package/dist/benchmarks/paired/runner.d.ts +25 -0
- package/dist/benchmarks/paired/runner.d.ts.map +1 -0
- package/dist/benchmarks/paired/runner.js +125 -0
- package/dist/benchmarks/paired/runner.js.map +1 -0
- package/dist/benchmarks/paired/scaffold.d.ts +28 -0
- package/dist/benchmarks/paired/scaffold.d.ts.map +1 -0
- package/dist/benchmarks/paired/scaffold.js +74 -0
- package/dist/benchmarks/paired/scaffold.js.map +1 -0
- package/dist/benchmarks/paired/stats.d.ts +68 -0
- package/dist/benchmarks/paired/stats.d.ts.map +1 -0
- package/dist/benchmarks/paired/stats.js +182 -0
- package/dist/benchmarks/paired/stats.js.map +1 -0
- package/dist/benchmarks/paired/suite.d.ts +43 -0
- package/dist/benchmarks/paired/suite.d.ts.map +1 -0
- package/dist/benchmarks/paired/suite.js +117 -0
- package/dist/benchmarks/paired/suite.js.map +1 -0
- package/dist/benchmarks/paired/types.d.ts +195 -0
- package/dist/benchmarks/paired/types.d.ts.map +1 -0
- package/dist/benchmarks/paired/types.js +111 -0
- package/dist/benchmarks/paired/types.js.map +1 -0
- package/dist/bin/cli.js +22 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/cli/bench.d.ts +21 -0
- package/dist/cli/bench.d.ts.map +1 -0
- package/dist/cli/bench.js +94 -0
- package/dist/cli/bench.js.map +1 -0
- package/docs/benchmarks/PAIRED_HARNESS.md +112 -0
- package/docs/benchmarks/README.md +1 -0
- package/package.json +1 -1
- package/src/policies/enforcers/__pycache__/_common.cpython-312.pyc +0 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-component ablation.
|
|
3
|
+
*
|
|
4
|
+
* Builds a leave-one-out condition matrix (baseline, full, and full-minus-each-
|
|
5
|
+
* component) and attributes the *marginal* contribution of each UAP component by
|
|
6
|
+
* pairing `uap-full` against `no-<component>` cell-for-cell. This is the
|
|
7
|
+
* scaffold-ablation method (HAL / SWE-Agent): turn off one mechanism at a time
|
|
8
|
+
* and measure the delta in resolve-rate and cost, ranking components so we can
|
|
9
|
+
* answer "is the whole protocol worth its overhead, or just two of six features?"
|
|
10
|
+
*/
|
|
11
|
+
import { PairedDeltaResult, PairedOptions } from './stats.js';
|
|
12
|
+
import { RunnerOutput } from './runner.js';
|
|
13
|
+
import { Condition, UapComponent } from './types.js';
|
|
14
|
+
export declare function ablationLabel(component: UapComponent): string;
|
|
15
|
+
/** baseline + uap-full + one leave-one-out condition per component. */
|
|
16
|
+
export declare function buildAblationConditions(): Condition[];
|
|
17
|
+
export interface ComponentContribution {
|
|
18
|
+
component: UapComponent;
|
|
19
|
+
/** success(full) - success(no-component), paired. Positive => component helps. */
|
|
20
|
+
correctnessDelta: PairedDeltaResult;
|
|
21
|
+
/** token(full) - token(no-component), paired. Positive => component costs tokens. */
|
|
22
|
+
tokenDelta: PairedDeltaResult | null;
|
|
23
|
+
fullSuccess: number;
|
|
24
|
+
ablatedSuccess: number;
|
|
25
|
+
}
|
|
26
|
+
export interface AblationReport {
|
|
27
|
+
fullLabel: string;
|
|
28
|
+
contributions: ComponentContribution[];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Rank components by marginal contribution. Requires the run to have used the
|
|
32
|
+
* ablation conditions (uap-full present plus the no-<component> arms).
|
|
33
|
+
*/
|
|
34
|
+
export declare function analyzeAblation(output: RunnerOutput, opts?: PairedOptions): AblationReport;
|
|
35
|
+
export declare function renderAblationMarkdown(r: AblationReport): string;
|
|
36
|
+
//# sourceMappingURL=ablation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ablation.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/ablation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAqB,iBAAiB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,SAAS,EAKT,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,wBAAgB,aAAa,CAAC,SAAS,EAAE,YAAY,GAAG,MAAM,CAE7D;AAED,uEAAuE;AACvE,wBAAgB,uBAAuB,IAAI,SAAS,EAAE,CAOrD;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,YAAY,CAAC;IACxB,kFAAkF;IAClF,gBAAgB,EAAE,iBAAiB,CAAC;IACpC,qFAAqF;IACrF,UAAU,EAAE,iBAAiB,GAAG,IAAI,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,qBAAqB,EAAE,CAAC;CACxC;AAYD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,GAAE,aAAkB,GAAG,cAAc,CAuC9F;AAED,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAsBhE"}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-component ablation.
|
|
3
|
+
*
|
|
4
|
+
* Builds a leave-one-out condition matrix (baseline, full, and full-minus-each-
|
|
5
|
+
* component) and attributes the *marginal* contribution of each UAP component by
|
|
6
|
+
* pairing `uap-full` against `no-<component>` cell-for-cell. This is the
|
|
7
|
+
* scaffold-ablation method (HAL / SWE-Agent): turn off one mechanism at a time
|
|
8
|
+
* and measure the delta in resolve-rate and cost, ranking components so we can
|
|
9
|
+
* answer "is the whole protocol worth its overhead, or just two of six features?"
|
|
10
|
+
*/
|
|
11
|
+
import { mean, pairedDelta } from './stats.js';
|
|
12
|
+
import { makeBaselineCondition, makeFullCondition, UAP_COMPONENTS, } from './types.js';
|
|
13
|
+
export function ablationLabel(component) {
|
|
14
|
+
return `no-${component}`;
|
|
15
|
+
}
|
|
16
|
+
/** baseline + uap-full + one leave-one-out condition per component. */
|
|
17
|
+
export function buildAblationConditions() {
|
|
18
|
+
const full = new Set(UAP_COMPONENTS);
|
|
19
|
+
const leaveOneOut = UAP_COMPONENTS.map((drop) => ({
|
|
20
|
+
label: ablationLabel(drop),
|
|
21
|
+
components: new Set([...full].filter((c) => c !== drop)),
|
|
22
|
+
}));
|
|
23
|
+
return [makeBaselineCondition(), makeFullCondition(), ...leaveOneOut];
|
|
24
|
+
}
|
|
25
|
+
function cellKey(r) {
|
|
26
|
+
return `${r.taskId}#${r.seed}`;
|
|
27
|
+
}
|
|
28
|
+
function cellMap(records, condition) {
|
|
29
|
+
const m = new Map();
|
|
30
|
+
for (const r of records)
|
|
31
|
+
if (r.condition === condition)
|
|
32
|
+
m.set(cellKey(r), r);
|
|
33
|
+
return m;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Rank components by marginal contribution. Requires the run to have used the
|
|
37
|
+
* ablation conditions (uap-full present plus the no-<component> arms).
|
|
38
|
+
*/
|
|
39
|
+
export function analyzeAblation(output, opts = {}) {
|
|
40
|
+
const fullLabel = 'uap-full';
|
|
41
|
+
const fullMap = cellMap(output.records, fullLabel);
|
|
42
|
+
if (fullMap.size === 0) {
|
|
43
|
+
throw new Error(`analyzeAblation: no '${fullLabel}' records present`);
|
|
44
|
+
}
|
|
45
|
+
const contributions = [];
|
|
46
|
+
for (const component of UAP_COMPONENTS) {
|
|
47
|
+
const ablMap = cellMap(output.records, ablationLabel(component));
|
|
48
|
+
if (ablMap.size === 0)
|
|
49
|
+
continue; // this component wasn't ablated in the run
|
|
50
|
+
const correctnessDeltas = [];
|
|
51
|
+
const tokenDeltas = [];
|
|
52
|
+
const fullCorrect = [];
|
|
53
|
+
const ablCorrect = [];
|
|
54
|
+
for (const [k, full] of fullMap) {
|
|
55
|
+
const abl = ablMap.get(k);
|
|
56
|
+
if (!abl)
|
|
57
|
+
continue;
|
|
58
|
+
correctnessDeltas.push((full.metrics.correct ? 1 : 0) - (abl.metrics.correct ? 1 : 0));
|
|
59
|
+
fullCorrect.push(full.metrics.correct ? 1 : 0);
|
|
60
|
+
ablCorrect.push(abl.metrics.correct ? 1 : 0);
|
|
61
|
+
if (full.metrics.tokens != null && abl.metrics.tokens != null) {
|
|
62
|
+
tokenDeltas.push(full.metrics.tokens - abl.metrics.tokens);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
contributions.push({
|
|
66
|
+
component,
|
|
67
|
+
correctnessDelta: pairedDelta(correctnessDeltas, opts),
|
|
68
|
+
tokenDelta: tokenDeltas.length ? pairedDelta(tokenDeltas, opts) : null,
|
|
69
|
+
fullSuccess: fullCorrect.length ? mean(fullCorrect) : NaN,
|
|
70
|
+
ablatedSuccess: ablCorrect.length ? mean(ablCorrect) : NaN,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
// Rank by correctness contribution descending (most valuable first).
|
|
74
|
+
contributions.sort((a, b) => b.correctnessDelta.meanDelta - a.correctnessDelta.meanDelta);
|
|
75
|
+
return { fullLabel, contributions };
|
|
76
|
+
}
|
|
77
|
+
export function renderAblationMarkdown(r) {
|
|
78
|
+
const L = [];
|
|
79
|
+
L.push(`## Component ablation (marginal contribution vs \`${r.fullLabel}\`)`);
|
|
80
|
+
L.push('');
|
|
81
|
+
L.push(`| Component | Δ success (full − ablated) | 95% CI | p | Δ tokens | sig |`);
|
|
82
|
+
L.push(`|---|--:|--:|--:|--:|:-:|`);
|
|
83
|
+
for (const c of r.contributions) {
|
|
84
|
+
const cd = c.correctnessDelta;
|
|
85
|
+
const tok = c.tokenDelta ? c.tokenDelta.meanDelta.toFixed(0) : 'n/a';
|
|
86
|
+
L.push(`| \`${c.component}\` | ${(cd.meanDelta * 100).toFixed(1)}pp | ` +
|
|
87
|
+
`[${(cd.ci.lower * 100).toFixed(1)}, ${(cd.ci.upper * 100).toFixed(1)}]pp | ` +
|
|
88
|
+
`${cd.pValue.toFixed(3)} | ${tok} | ${cd.significant ? '✅' : '–'} |`);
|
|
89
|
+
}
|
|
90
|
+
L.push('');
|
|
91
|
+
L.push(`> A component earns its keep when its Δ success CI is above 0. Components with ` +
|
|
92
|
+
`CI spanning 0 but positive Δ tokens are pure overhead — candidates to drop.`);
|
|
93
|
+
L.push('');
|
|
94
|
+
return L.join('\n');
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=ablation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ablation.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/ablation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,IAAI,EAAE,WAAW,EAAoC,MAAM,YAAY,CAAC;AAEjF,OAAO,EAEL,qBAAqB,EACrB,iBAAiB,EAEjB,cAAc,GAEf,MAAM,YAAY,CAAC;AAEpB,MAAM,UAAU,aAAa,CAAC,SAAuB;IACnD,OAAO,MAAM,SAAS,EAAE,CAAC;AAC3B,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,uBAAuB;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAChD,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC;QAC1B,UAAU,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAA8B;KACtF,CAAC,CAAC,CAAC;IACJ,OAAO,CAAC,qBAAqB,EAAE,EAAE,iBAAiB,EAAE,EAAE,GAAG,WAAW,CAAC,CAAC;AACxE,CAAC;AAiBD,SAAS,OAAO,CAAC,CAAY;IAC3B,OAAO,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,OAAO,CAAC,OAAoB,EAAE,SAAiB;IACtD,MAAM,CAAC,GAAG,IAAI,GAAG,EAAqB,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS;YAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7E,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,MAAoB,EAAE,OAAsB,EAAE;IAC5E,MAAM,SAAS,GAAG,UAAU,CAAC;IAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,wBAAwB,SAAS,mBAAmB,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,aAAa,GAA4B,EAAE,CAAC;IAClD,KAAK,MAAM,SAAS,IAAI,cAAc,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC;QACjE,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS,CAAC,2CAA2C;QAE5E,MAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvF,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC9D,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,aAAa,CAAC,IAAI,CAAC;YACjB,SAAS;YACT,gBAAgB,EAAE,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC;YACtD,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACtE,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG;YACzD,cAAc,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,qEAAqE;IACrE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,SAAS,GAAG,CAAC,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC1F,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,CAAiB;IACtD,MAAM,CAAC,GAAa,EAAE,CAAC;IACvB,CAAC,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC;IAC9E,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;IACnF,CAAC,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;QAChC,MAAM,EAAE,GAAG,CAAC,CAAC,gBAAgB,CAAC;QAC9B,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,CAAC,CAAC,IAAI,CACJ,OAAO,CAAC,CAAC,SAAS,QAAQ,CAAC,EAAE,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;YAC9D,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ;YAC7E,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CACvE,CAAC;IACJ,CAAC;IACD,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC,IAAI,CACJ,iFAAiF;QAC/E,6EAA6E,CAChF,CAAC;IACF,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Agent adapters — drive the "agent under test" for one run.
|
|
3
|
+
*
|
|
4
|
+
* Two real adapters (opencode, claude) spawn the agent as a subprocess against
|
|
5
|
+
* the scratch repo and parse usage metrics from its output. A MockAdapter makes
|
|
6
|
+
* the whole harness runnable and unit-testable offline (it does NOT make real
|
|
7
|
+
* claims — it is a deterministic stand-in for exercising the runner/stats/report
|
|
8
|
+
* paths without a live model).
|
|
9
|
+
*
|
|
10
|
+
* Adapters never throw on agent failure: they capture it in `error` so a single
|
|
11
|
+
* crashed run degrades to an incorrect result rather than aborting the suite.
|
|
12
|
+
*/
|
|
13
|
+
import { AgentAdapter, AgentRunContext, AgentRunResult } from './types.js';
|
|
14
|
+
export type UsageParser = (stdout: string, stderr: string) => Partial<AgentRunResult>;
|
|
15
|
+
export interface SubprocessAdapterConfig {
|
|
16
|
+
id: string;
|
|
17
|
+
/** Executable, e.g. 'opencode' or 'claude'. */
|
|
18
|
+
bin: string;
|
|
19
|
+
/**
|
|
20
|
+
* Argument template. The literal tokens '{instruction}' and '{workdir}' are
|
|
21
|
+
* replaced per run. Everything else is passed through verbatim.
|
|
22
|
+
*/
|
|
23
|
+
args: string[];
|
|
24
|
+
/** Parse tokens/turns/toolCalls from agent output. */
|
|
25
|
+
parseUsage: UsageParser;
|
|
26
|
+
/** Pass the instruction on stdin instead of substituting into args. */
|
|
27
|
+
instructionOnStdin?: boolean;
|
|
28
|
+
}
|
|
29
|
+
export declare class SubprocessAdapter implements AgentAdapter {
|
|
30
|
+
private readonly cfg;
|
|
31
|
+
readonly id: string;
|
|
32
|
+
constructor(cfg: SubprocessAdapterConfig);
|
|
33
|
+
run(ctx: AgentRunContext): Promise<AgentRunResult>;
|
|
34
|
+
}
|
|
35
|
+
/** Sum the last token-usage object emitted in Claude Code's `--output-format json`. */
|
|
36
|
+
export declare const parseClaudeUsage: UsageParser;
|
|
37
|
+
/**
|
|
38
|
+
* Parser for opencode `--format json` output, which is a JSONL event stream
|
|
39
|
+
* (step_start / tool_use / step_finish / text). We aggregate it:
|
|
40
|
+
* - turns = number of step_finish events
|
|
41
|
+
* - toolCalls = number of tool_use events
|
|
42
|
+
* - tokens = total tokens metered = Σ over step_finish of
|
|
43
|
+
* (input + output + cache.read) — matches provider billing and
|
|
44
|
+
* captures the context overhead an injected UAP surface adds.
|
|
45
|
+
* - costUsd = Σ cost (often 0 for a local model → reported as null).
|
|
46
|
+
*/
|
|
47
|
+
export declare const parseOpencodeUsage: UsageParser;
|
|
48
|
+
/** Claude Code headless adapter (`claude -p ... --output-format json`). */
|
|
49
|
+
export declare function claudeAdapter(model: string): SubprocessAdapter;
|
|
50
|
+
/** opencode headless adapter (`opencode run ...`). */
|
|
51
|
+
export declare function opencodeAdapter(model: string): SubprocessAdapter;
|
|
52
|
+
export interface MockBehavior {
|
|
53
|
+
/** Base probability the bare agent resolves a task of this difficulty. */
|
|
54
|
+
baseSuccess: Record<'easy' | 'medium' | 'hard', number>;
|
|
55
|
+
/** Additive correctness lift per enabled UAP component (capped at 1.0). */
|
|
56
|
+
liftPerComponent: number;
|
|
57
|
+
/** Token cost of the bare run; each component adds overhead tokens. */
|
|
58
|
+
baseTokens: number;
|
|
59
|
+
overheadTokensPerComponent: number;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Deterministic simulated agent. Given a seed, it reproducibly "resolves" tasks
|
|
63
|
+
* with a probability that the UAP components nudge upward — purely to exercise
|
|
64
|
+
* the runner/stats/report end-to-end without a live model. Clearly NOT a source
|
|
65
|
+
* of real benchmark claims; the real adapters above are.
|
|
66
|
+
*/
|
|
67
|
+
export declare class MockAdapter implements AgentAdapter {
|
|
68
|
+
private readonly behavior;
|
|
69
|
+
readonly id = "mock";
|
|
70
|
+
constructor(behavior?: MockBehavior);
|
|
71
|
+
run(ctx: AgentRunContext): Promise<AgentRunResult>;
|
|
72
|
+
}
|
|
73
|
+
/** Stable hash of a string to [0,1). */
|
|
74
|
+
export declare function hash01(s: string): number;
|
|
75
|
+
export interface RawAdapterConfig {
|
|
76
|
+
/** OpenAI-compatible chat endpoint. Default ik-llama :8080. */
|
|
77
|
+
endpoint?: string;
|
|
78
|
+
/** Max execute->verify->fix iterations when gating. Default 4. */
|
|
79
|
+
maxGateIters?: number;
|
|
80
|
+
temperature?: number;
|
|
81
|
+
}
|
|
82
|
+
/** Parse <<<FILE path>>> ... <<<END>>> blocks from a model response. Pure. */
|
|
83
|
+
export declare function parseFileBlocks(text: string): {
|
|
84
|
+
path: string;
|
|
85
|
+
content: string;
|
|
86
|
+
}[];
|
|
87
|
+
export declare class RawCompletionAdapter implements AgentAdapter {
|
|
88
|
+
readonly id = "raw";
|
|
89
|
+
private readonly endpoint;
|
|
90
|
+
private readonly maxGateIters;
|
|
91
|
+
private readonly temperature;
|
|
92
|
+
constructor(cfg?: RawAdapterConfig);
|
|
93
|
+
run(ctx: AgentRunContext): Promise<AgentRunResult>;
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAc,MAAM,YAAY,CAAC;AAMvF,MAAM,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAEtF,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,+CAA+C;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,sDAAsD;IACtD,UAAU,EAAE,WAAW,CAAC;IACxB,uEAAuE;IACvE,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAOD,qBAAa,iBAAkB,YAAW,YAAY;IAExC,OAAO,CAAC,QAAQ,CAAC,GAAG;IADhC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBACS,GAAG,EAAE,uBAAuB;IAInD,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA8CzD;AA6GD,uFAAuF;AACvF,eAAO,MAAM,gBAAgB,EAAE,WAiB9B,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,EAAE,WAsChC,CAAC;AAMF,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAO9D;AAED,sDAAsD;AACtD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CAShE;AAMD,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC,MAAM,GAAG,QAAQ,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;IACxD,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,CAAC;IACzB,uEAAuE;IACvE,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B,EAAE,MAAM,CAAC;CACpC;AASD;;;;;GAKG;AACH,qBAAa,WAAY,YAAW,YAAY;IAElC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IADrC,QAAQ,CAAC,EAAE,UAAU;gBACQ,QAAQ,GAAE,YAA2B;IAE5D,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CA6BzD;AAED,wCAAwC;AACxC,wBAAgB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAOxC;AAYD,MAAM,WAAW,gBAAgB;IAC/B,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AASD,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CASjF;AA8ED,qBAAa,oBAAqB,YAAW,YAAY;IACvD,QAAQ,CAAC,EAAE,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBACzB,GAAG,GAAE,gBAAqB;IAOhC,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;CAyEzD"}
|