@dsh-enhanced/assistant-evaluation 0.1.32 → 0.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/lib/benchmark/cli.d.ts.map +1 -1
- package/lib/benchmark/cli.js +57 -21
- package/lib/benchmark/cli.js.map +1 -1
- package/lib/benchmark/native-goal-services.d.ts +64 -0
- package/lib/benchmark/native-goal-services.d.ts.map +1 -0
- package/lib/benchmark/native-goal-services.js +52 -0
- package/lib/benchmark/native-goal-services.js.map +1 -0
- package/lib/benchmark/native-skills-runtime.d.ts +155 -0
- package/lib/benchmark/native-skills-runtime.d.ts.map +1 -0
- package/lib/benchmark/native-skills-runtime.js +336 -0
- package/lib/benchmark/native-skills-runtime.js.map +1 -0
- package/lib/benchmark/native-skills.d.ts +142 -0
- package/lib/benchmark/native-skills.d.ts.map +1 -0
- package/lib/benchmark/native-skills.js +259 -0
- package/lib/benchmark/native-skills.js.map +1 -0
- package/lib/benchmark/native.d.ts +7 -0
- package/lib/benchmark/native.d.ts.map +1 -1
- package/lib/benchmark/native.js.map +1 -1
- package/lib/benchmark/strategy-capabilities.d.ts.map +1 -1
- package/lib/benchmark/strategy-capabilities.js +9 -0
- package/lib/benchmark/strategy-capabilities.js.map +1 -1
- package/lib/benchmark/strategy-config.d.ts +3 -2
- package/lib/benchmark/strategy-config.d.ts.map +1 -1
- package/lib/benchmark/strategy-config.js +19 -6
- package/lib/benchmark/strategy-config.js.map +1 -1
- package/lib/benchmark/strategy-corpus-v2.d.ts +18 -0
- package/lib/benchmark/strategy-corpus-v2.d.ts.map +1 -0
- package/lib/benchmark/strategy-corpus-v2.js +143 -0
- package/lib/benchmark/strategy-corpus-v2.js.map +1 -0
- package/lib/benchmark/strategy-corpus.d.ts +9 -0
- package/lib/benchmark/strategy-corpus.d.ts.map +1 -1
- package/lib/benchmark/strategy-corpus.js +3 -3
- package/lib/benchmark/strategy-corpus.js.map +1 -1
- package/lib/benchmark/strategy-evidence.d.ts.map +1 -1
- package/lib/benchmark/strategy-evidence.js +50 -9
- package/lib/benchmark/strategy-evidence.js.map +1 -1
- package/lib/benchmark/strategy-executor.d.ts.map +1 -1
- package/lib/benchmark/strategy-executor.js +6 -4
- package/lib/benchmark/strategy-executor.js.map +1 -1
- package/lib/benchmark/strategy-goal-runtime.d.ts.map +1 -1
- package/lib/benchmark/strategy-goal-runtime.js +13 -55
- package/lib/benchmark/strategy-goal-runtime.js.map +1 -1
- package/lib/benchmark/strategy-meter.d.ts +2 -2
- package/lib/benchmark/strategy-meter.d.ts.map +1 -1
- package/lib/benchmark/strategy-meter.js +98 -4
- package/lib/benchmark/strategy-meter.js.map +1 -1
- package/lib/benchmark/strategy-plan.d.ts +7 -0
- package/lib/benchmark/strategy-plan.d.ts.map +1 -1
- package/lib/benchmark/strategy-plan.js +12 -2
- package/lib/benchmark/strategy-plan.js.map +1 -1
- package/lib/benchmark/strategy-projection.d.ts +157 -0
- package/lib/benchmark/strategy-projection.d.ts.map +1 -0
- package/lib/benchmark/strategy-projection.js +171 -0
- package/lib/benchmark/strategy-projection.js.map +1 -0
- package/lib/benchmark/super-relay.d.ts +26 -0
- package/lib/benchmark/super-relay.d.ts.map +1 -0
- package/lib/benchmark/super-relay.js +173 -0
- package/lib/benchmark/super-relay.js.map +1 -0
- package/lib/benchmark/traex.d.ts +17 -0
- package/lib/benchmark/traex.d.ts.map +1 -0
- package/lib/benchmark/traex.js +139 -0
- package/lib/benchmark/traex.js.map +1 -0
- package/lib/service.d.ts +25 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +20 -4
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +46 -1
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +13 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +90 -4
- package/lib/store.js.map +1 -1
- package/lib/types.d.ts +13 -0
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +49 -12
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isAbsolute, relative } from 'node:path';
|
|
2
|
-
import {
|
|
2
|
+
import { isStrategySuite, strategyCasesForSuite, strategyTaskForSuite } from './strategy-corpus-v2.js';
|
|
3
3
|
import { benchmarkAssert, benchmarkInteger, benchmarkObject, benchmarkSnapshot } from './schema.js';
|
|
4
4
|
const MAX_RATE = 1_000_000_000;
|
|
5
5
|
const routeId = (value, field) => benchmarkAssert(typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/u.test(value), `invalid ${field}`);
|
|
@@ -15,11 +15,11 @@ export function parseStrategyBenchmarkConfig(value) {
|
|
|
15
15
|
const copy = benchmarkSnapshot(value);
|
|
16
16
|
const raw = benchmarkObject(copy, ['suite', 'id', 'cases', 'persona', 'model', 'budget', 'execution', 'repeats', 'seed', 'image', 'dockerPath', 'stepMaxDurationMs', 'workspaceDirectory', 'stateDirectory',
|
|
17
17
|
...((copy !== null && typeof copy === 'object' && Object.hasOwn(copy, 'stopTimeoutMs')) ? ['stopTimeoutMs'] : [])]);
|
|
18
|
-
benchmarkAssert(raw.suite
|
|
18
|
+
benchmarkAssert(isStrategySuite(raw.suite), 'unsupported strategy suite');
|
|
19
19
|
identifier(raw.id, 'strategy benchmark id');
|
|
20
20
|
benchmarkAssert(typeof raw.persona === 'string' && raw.persona.trim().length > 0 && raw.persona.length <= 32_768, 'invalid strategy persona');
|
|
21
21
|
benchmarkAssert(Array.isArray(raw.cases) && raw.cases.length >= 1 && raw.cases.length <= 100, 'invalid strategy cases');
|
|
22
|
-
const known = new Set(
|
|
22
|
+
const known = new Set(strategyCasesForSuite(raw.suite).map(task => task.id));
|
|
23
23
|
const selected = new Set();
|
|
24
24
|
for (const caseId of raw.cases) {
|
|
25
25
|
identifier(caseId, 'strategy case id');
|
|
@@ -37,11 +37,18 @@ export function parseStrategyBenchmarkConfig(value) {
|
|
|
37
37
|
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'outputLimitMode')) ? ['outputLimitMode'] : []),
|
|
38
38
|
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'cacheReadUsdMicrosPerMillionTokens')) ? ['cacheReadUsdMicrosPerMillionTokens'] : []),
|
|
39
39
|
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'cacheWriteUsdMicrosPerMillionTokens')) ? ['cacheWriteUsdMicrosPerMillionTokens'] : []),
|
|
40
|
+
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'observationMode')) ? ['observationMode'] : []),
|
|
40
41
|
]);
|
|
41
42
|
routeId(model.provider, 'provider');
|
|
42
43
|
routeId(model.model, 'model');
|
|
43
44
|
benchmarkAssert(model.temperature === null || typeof model.temperature === 'number' && Number.isFinite(model.temperature) && model.temperature >= 0 && model.temperature <= 2, 'invalid temperature');
|
|
44
|
-
|
|
45
|
+
// The single source of truth for metering mode; the executor derives plan.execution.observationMode from it.
|
|
46
|
+
benchmarkAssert(model.observationMode === undefined
|
|
47
|
+
|| model.observationMode === 'enforced-upper-bound-provider-output'
|
|
48
|
+
|| model.observationMode === 'observed-call-count', 'invalid model observation mode');
|
|
49
|
+
const callCounting = model.observationMode === 'observed-call-count';
|
|
50
|
+
if (!callCounting)
|
|
51
|
+
benchmarkAssert((model.inputLimitMode ?? 'upper-bound') === 'upper-bound' && (model.outputLimitMode ?? 'provider') === 'provider', 'strategy requires upper-bound input and provider output limits');
|
|
45
52
|
benchmarkInteger(model.maxOutputTokens, 1, budget.outputTokens);
|
|
46
53
|
benchmarkAssert((model.inputUsdMicrosPerMillionTokens === null) === (model.outputUsdMicrosPerMillionTokens === null), 'both token rates must be present or absent');
|
|
47
54
|
if (model.inputUsdMicrosPerMillionTokens !== null) {
|
|
@@ -53,7 +60,13 @@ export function parseStrategyBenchmarkConfig(value) {
|
|
|
53
60
|
rate(model[field]);
|
|
54
61
|
digest(model.adapterDigest, 'adapter digest');
|
|
55
62
|
digest(model.tokenCounterDigest, 'token counter digest');
|
|
56
|
-
if (
|
|
63
|
+
if (callCounting) {
|
|
64
|
+
// Call-count mode measures model requests only: no tariffs, no monetary budget, zero input-token budget.
|
|
65
|
+
// budget.outputTokens is a positive structural placeholder (equal to the per-call cap), never measured.
|
|
66
|
+
benchmarkAssert(budget.costUsdMicros === null && budget.inputTokens === 0, 'call-count mode requires null cost and zero input-token budget');
|
|
67
|
+
benchmarkAssert([model.inputUsdMicrosPerMillionTokens, model.outputUsdMicrosPerMillionTokens, model.cacheReadUsdMicrosPerMillionTokens, model.cacheWriteUsdMicrosPerMillionTokens].every(value => value === null || value === undefined), 'call-count mode requires null token tariffs');
|
|
68
|
+
}
|
|
69
|
+
else if (budget.costUsdMicros !== null)
|
|
57
70
|
benchmarkAssert([model.inputUsdMicrosPerMillionTokens, model.outputUsdMicrosPerMillionTokens, model.cacheReadUsdMicrosPerMillionTokens, model.cacheWriteUsdMicrosPerMillionTokens].every(value => value !== null && value !== undefined), 'priced budget requires complete tariff');
|
|
58
71
|
const execution = benchmarkObject(raw.execution, ['modelCalls', 'maxOutputTokensPerCall', 'maxGoalRounds']);
|
|
59
72
|
benchmarkInteger(execution.modelCalls, 1, 10_000);
|
|
@@ -70,7 +83,7 @@ export function parseStrategyBenchmarkConfig(value) {
|
|
|
70
83
|
benchmarkInteger(raw.stepMaxDurationMs, 1_000, 300_000);
|
|
71
84
|
benchmarkAssert(raw.stepMaxDurationMs * 3 < budget.durationMs, 'strategy step duration does not fit benchmark duration');
|
|
72
85
|
for (const caseId of selected) {
|
|
73
|
-
const verification =
|
|
86
|
+
const verification = strategyTaskForSuite(raw.suite, caseId).verification;
|
|
74
87
|
benchmarkAssert(verification.maxDurationMs * verification.cases.length < raw.stepMaxDurationMs, 'strategy verification does not fit execution window');
|
|
75
88
|
}
|
|
76
89
|
if (raw.stopTimeoutMs !== undefined)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategy-config.js","sourceRoot":"","sources":["../../src/benchmark/strategy-config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAChD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"strategy-config.js","sourceRoot":"","sources":["../../src/benchmark/strategy-config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,oBAAoB,EAAsB,MAAM,yBAAyB,CAAA;AAC1H,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAqBnG,MAAM,QAAQ,GAAG,aAAa,CAAA;AAC9B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,uCAAuC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAC9K,MAAM,UAAU,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAC/K,MAAM,MAAM,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AACvJ,MAAM,IAAI,GAAG,CAAC,KAAc,EAAQ,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;AAC3E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAClM,MAAM,QAAQ,GAAG,CAAC,QAAgB,EAAE,UAAkB,EAAW,EAAE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC3C,OAAO,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;AAClI,CAAC,CAAA;AAED,MAAM,UAAU,4BAA4B,CAAC,KAAc;IACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,gBAAgB;QACzM,GAAG,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACrH,eAAe,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,4BAA4B,CAAC,CAAA;IACzE,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAA;IAC3C,eAAe,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,EAAE,0BAA0B,CAAC,CAAA;IAC7I,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,wBAAwB,CAAC,CAAA;IACvH,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAChH,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAAC,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,iDAAiD,CAAC,CAAC;QAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAAC,CAAC;IAEhN,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,CAAC,CAA+B,CAAA;IACrJ,gBAAgB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IAClD,KAAK,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,cAAc,EAAE,WAAW,CAAU;QAAE,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;IAC5H,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI;QAAE,gBAAgB,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;IAEtF,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gCAAgC,EAAE,iCAAiC,EAAE,eAAe,EAAE,oBAAoB;QACzM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,oCAAoC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1K,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5K,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACrI,CAAiC,CAAA;IAClC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAClE,eAAe,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,EAAE,qBAAqB,CAAC,CAAA;IACrM,6GAA6G;IAC7G,eAAe,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS;WAC9C,KAAK,CAAC,eAAe,KAAK,sCAAsC;WAChE,KAAK,CAAC,eAAe,KAAK,qBAAqB,EAAE,gCAAgC,CAAC,CAAA;IACvF,MAAM,YAAY,GAAG,KAAK,CAAC,eAAe,KAAK,qBAAqB,CAAA;IACpE,IAAI,CAAC,YAAY;QAAE,eAAe,CAAC,CAAC,KAAK,CAAC,cAAc,IAAI,aAAa,CAAC,KAAK,aAAa,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,UAAU,CAAC,KAAK,UAAU,EAAE,gEAAgE,CAAC,CAAA;IACvN,gBAAgB,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;IAC/D,eAAe,CAAC,CAAC,KAAK,CAAC,8BAA8B,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,+BAA+B,KAAK,IAAI,CAAC,EAAE,4CAA4C,CAAC,CAAA;IACnK,IAAI,KAAK,CAAC,8BAA8B,KAAK,IAAI,EAAE,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAAC,CAAC;IAC9I,KAAK,MAAM,KAAK,IAAI,CAAC,oCAAoC,EAAE,qCAAqC,CAAU;QAAE,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IACvL,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,sBAAsB,CAAC,CAAA;IACvG,IAAI,YAAY,EAAE,CAAC;QACjB,yGAAyG;QACzG,wGAAwG;QACxG,eAAe,CAAC,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,CAAC,EAAE,gEAAgE,CAAC,CAAA;QAC5I,eAAe,CAAC,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,EAAE,6CAA6C,CAAC,CAAA;IAC1R,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI;QAAE,eAAe,CAAC,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,EAAE,wCAAwC,CAAC,CAAA;IAE7T,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,YAAY,EAAE,wBAAwB,EAAE,eAAe,CAAC,CAAyC,CAAA;IACnJ,gBAAgB,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;IAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,EAAE,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IAAC,gBAAgB,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;IAC/K,eAAe,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,sBAAsB,EAAE,wDAAwD,CAAC,CAAA;IACrI,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IAC/E,eAAe,CAAC,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,yBAAyB,CAAC,CAAA;IACrH,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,kBAAkB,EAAE,qBAAqB,CAAC,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAA;IACjJ,eAAe,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,kBAA4B,EAAE,GAAG,CAAC,cAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAwB,EAAE,GAAG,CAAC,kBAA4B,CAAC,EAAE,gDAAgD,CAAC,CAAA;IACzN,gBAAgB,CAAC,GAAG,CAAC,iBAAiB,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IACvD,eAAe,CAAE,GAAG,CAAC,iBAA4B,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,wDAAwD,CAAC,CAAA;IACpI,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,oBAAoB,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,YAAY,CAAA;QACzE,eAAe,CAAC,YAAY,CAAC,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,GAAI,GAAG,CAAC,iBAA4B,EAAE,qDAAqD,CAAC,CAAA;IACpK,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS;QAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACvF,OAAO,IAA+B,CAAA;AACxC,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { BenchmarkCase, BenchmarkPlan } from './types.js';
|
|
2
|
+
import { type StrategyGoalTask } from './strategy-goal-runtime.js';
|
|
3
|
+
import { type StrategyDevelopmentCorpusTask } from './strategy-corpus.js';
|
|
4
|
+
export type StrategySuite = 'strategy-v1' | 'strategy-v2';
|
|
5
|
+
export declare const strategySuites: readonly StrategySuite[];
|
|
6
|
+
export declare const isStrategySuite: (value: unknown) => value is StrategySuite;
|
|
7
|
+
/** This public surface intentionally excludes independent verification vectors. */
|
|
8
|
+
export declare const strategyV2Corpus: readonly StrategyDevelopmentCorpusTask[];
|
|
9
|
+
/** Dataset identity commits to the complete private verification manifest. */
|
|
10
|
+
export declare const strategyV2Dataset: BenchmarkPlan['dataset'];
|
|
11
|
+
export declare function strategyV2Cases(): readonly BenchmarkCase[];
|
|
12
|
+
/** Host-only full task material, including private verifier vectors. */
|
|
13
|
+
export declare function strategyV2Task(caseId: string): Readonly<StrategyGoalTask>;
|
|
14
|
+
export declare const strategyCorpusForSuite: (suite: StrategySuite) => readonly StrategyDevelopmentCorpusTask[];
|
|
15
|
+
export declare const strategyDatasetForSuite: (suite: StrategySuite) => BenchmarkPlan["dataset"];
|
|
16
|
+
export declare const strategyCasesForSuite: (suite: StrategySuite) => readonly BenchmarkCase[];
|
|
17
|
+
export declare const strategyTaskForSuite: (suite: StrategySuite, caseId: string) => Readonly<StrategyGoalTask>;
|
|
18
|
+
//# sourceMappingURL=strategy-corpus-v2.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-corpus-v2.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus-v2.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC9D,OAAO,EAA2B,KAAK,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EASL,KAAK,6BAA6B,EACnC,MAAM,sBAAsB,CAAA;AAE7B,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG,aAAa,CAAA;AACzD,eAAO,MAAM,cAAc,EAAE,SAAS,aAAa,EAAmC,CAAA;AACtF,eAAO,MAAM,eAAe,GAAI,OAAO,OAAO,KAAG,KAAK,IAAI,aAAmE,CAAA;AAyG7H,mFAAmF;AACnF,eAAO,MAAM,gBAAgB,EAAE,SAAS,6BAA6B,EAAmF,CAAA;AAOxJ,8EAA8E;AAC9E,eAAO,MAAM,iBAAiB,EAAE,aAAa,CAAC,SAAS,CAGrD,CAAA;AAEF,wBAAgB,eAAe,IAAI,SAAS,aAAa,EAAE,CAE1D;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAIzE;AAGD,eAAO,MAAM,sBAAsB,GAAI,OAAO,aAAa,KAAG,SAAS,6BAA6B,EAC5B,CAAA;AACxE,eAAO,MAAM,uBAAuB,GAAI,OAAO,aAAa,KAAG,aAAa,CAAC,SAAS,CACZ,CAAA;AAC1E,eAAO,MAAM,qBAAqB,GAAI,OAAO,aAAa,KAAG,SAAS,aAAa,EACT,CAAA;AAC1E,eAAO,MAAM,oBAAoB,GAAI,OAAO,aAAa,EAAE,QAAQ,MAAM,KAAG,QAAQ,CAAC,gBAAgB,CACjB,CAAA"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harder public development tasks for the native Goal strategy benchmark (strategy-v2).
|
|
3
|
+
*
|
|
4
|
+
* This suite is deliberately more coupled and adversarial than strategy-v1: each task combines
|
|
5
|
+
* several exact constraints (half-open boundaries, midnight wrap, closed-intersection feasibility,
|
|
6
|
+
* paragraph-aware rewrapping, a quoted-CSV state machine) so a first-round artifact can plausibly
|
|
7
|
+
* fail independent verification. Like v1, the model sees only `objective`, `publicPrompt` and the
|
|
8
|
+
* single public example per task; the private verifier vectors never leave the Host.
|
|
9
|
+
*
|
|
10
|
+
* The v1 authored corpus is intentionally untouched so its dataset digest stays byte-identical.
|
|
11
|
+
* Every private expected byte below was produced by a POSIX-sh reference implementation and
|
|
12
|
+
* byte-compared inside the frozen verifier image (dash + busybox awk); it is not hand-computed.
|
|
13
|
+
*/
|
|
14
|
+
import { acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
|
|
15
|
+
import { strategyGoalTaskDigests } from './strategy-goal-runtime.js';
|
|
16
|
+
import { frozenJson, objective, prompt, strategyDevelopmentCases, strategyDevelopmentCorpus, strategyDevelopmentDataset, strategyDevelopmentTask, } from './strategy-corpus.js';
|
|
17
|
+
export const strategySuites = ['strategy-v1', 'strategy-v2'];
|
|
18
|
+
export const isStrategySuite = (value) => value === 'strategy-v1' || value === 'strategy-v2';
|
|
19
|
+
const sessionGapSplit = objective([
|
|
20
|
+
'reads one wall-clock timestamp in exact "HH:MM:SS" form per line (24-hour, zero-padded), and partitions the series into sessions.',
|
|
21
|
+
'The timestamps are non-decreasing except that a raw value smaller than the previous raw value means the series crosses exactly one midnight; add 86400 seconds to that value and every later value for that comparison.',
|
|
22
|
+
'Two consecutive events belong to the same session only when the difference between their absolute times is at most 300 seconds; a gap strictly greater than 300 seconds starts a new session (exactly 300 seconds stays in the same session).',
|
|
23
|
+
'For every session, in chronological session order, write one line "start-end count" where start and end are the session\'s first and last timestamps rendered as "HH:MM:SS" (a timestamp on the far side of midnight is shown as that wrapped wall-clock time by taking it modulo 86400) and count is the number of events in that session; a single event is rendered with start equal to end and count 1.',
|
|
24
|
+
'Every emitted line ends with one newline. Empty input produces no output.',
|
|
25
|
+
].join(' '));
|
|
26
|
+
const closedRangeIntersection = objective([
|
|
27
|
+
'reads lines of three whitespace-separated fields "name lo hi", giving one closed integer interval [lo, hi] constraint on name.',
|
|
28
|
+
'Accumulate every constraint with the same name: the resulting lower bound is the maximum of all lo values and the upper bound is the minimum of all hi values, i.e. the intersection of the closed intervals.',
|
|
29
|
+
'A result whose lower bound equals its upper bound is the single point [x, x] and is still feasible; it is empty only when the maximum lower bound is strictly greater than the minimum upper bound.',
|
|
30
|
+
'If ANY name has an empty intersection, write exactly one line containing "NONE". Otherwise write one "name lo hi" line per name, ordered by ascending name in ASCII byte order, each line ending with one newline.',
|
|
31
|
+
'Empty input produces no output.',
|
|
32
|
+
].join(' '));
|
|
33
|
+
const greedyParagraphWrap = objective([
|
|
34
|
+
'splits standard input into paragraphs separated by runs of blank lines, ignoring leading and trailing blank lines and paragraphs that contain only whitespace.',
|
|
35
|
+
'Within a paragraph, collapse every run of whitespace (spaces, tabs and newlines) into a single separator to obtain the ordered sequence of words.',
|
|
36
|
+
'Fill words greedily into lines exactly 40 columns wide: the first word starts a line; a later word is joined with a single space only when "current line length + 1 + word length <= 40", otherwise it starts a new line. A word longer than 40 columns is placed alone on its own line intact, never split or truncated.',
|
|
37
|
+
'Lines within a paragraph are separated by one newline; consecutive paragraphs are separated by exactly one blank line; any non-empty output ends with one newline.',
|
|
38
|
+
'Empty input or input containing only whitespace produces no output.',
|
|
39
|
+
].join(' '));
|
|
40
|
+
const quotedCsvAccountTotal = objective([
|
|
41
|
+
'skips the first line, which is the fixed header "account,jurisdiction,amount", and reads each remaining line as one record with exactly three comma-separated columns: account, jurisdiction, and an integer amount in cents that may be negative.',
|
|
42
|
+
'A column may be wrapped in double quotes; commas inside a quoted column are literal data, and two consecutive double quotes inside a quoted column denote one literal double quote.',
|
|
43
|
+
'Group records by the pair (jurisdiction, account) and sum the integer amounts in each group. The value -0 equals 0, and a group whose positive and negative amounts cancel to exactly 0 is still emitted.',
|
|
44
|
+
'For each distinct pair write one line with jurisdiction, account and the integer total joined by single tab characters as "jurisdiction<TAB>account<TAB>total", ending with one newline. Order the lines by ascending ASCII byte order of the whole line (equivalently jurisdiction first, then account).',
|
|
45
|
+
'Input containing only the header, or no records at all, produces no output.',
|
|
46
|
+
].join(' '));
|
|
47
|
+
const authoredV2 = [
|
|
48
|
+
{
|
|
49
|
+
id: 'session-gap-split', domain: 'code',
|
|
50
|
+
objective: sessionGapSplit,
|
|
51
|
+
examples: [{ stdin: '09:00:00\n09:01:00\n09:10:00\n', expectedStdout: '09:00:00-09:01:00 2\n09:10:00-09:10:00 1\n' }],
|
|
52
|
+
publicPrompt: prompt(sessionGapSplit, [{ stdin: '09:00:00\n09:01:00\n09:10:00\n', expectedStdout: '09:00:00-09:01:00 2\n09:10:00-09:10:00 1\n' }]),
|
|
53
|
+
artifactPath: 'answer.sh',
|
|
54
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
55
|
+
{ stdin: '', expectedStdout: '', expectedExitCode: 0 },
|
|
56
|
+
{ stdin: '12:34:56\n', expectedStdout: '12:34:56-12:34:56 1\n', expectedExitCode: 0 },
|
|
57
|
+
{ stdin: '09:00:00\n09:05:00\n', expectedStdout: '09:00:00-09:05:00 2\n', expectedExitCode: 0 },
|
|
58
|
+
{ stdin: '09:00:00\n09:05:01\n', expectedStdout: '09:00:00-09:00:00 1\n09:05:01-09:05:01 1\n', expectedExitCode: 0 },
|
|
59
|
+
{ stdin: '23:58:00\n00:02:00\n', expectedStdout: '23:58:00-00:02:00 2\n', expectedExitCode: 0 },
|
|
60
|
+
{ stdin: '23:59:00\n00:01:00\n00:10:00\n', expectedStdout: '23:59:00-00:01:00 2\n00:10:00-00:10:00 1\n', expectedExitCode: 0 },
|
|
61
|
+
{ stdin: '08:00:00\n08:00:00\n08:10:00\n', expectedStdout: '08:00:00-08:00:00 2\n08:10:00-08:10:00 1\n', expectedExitCode: 0 },
|
|
62
|
+
{ stdin: '09:00:00\n09:00:30\n09:10:00\n09:10:10\n09:30:00\n', expectedStdout: '09:00:00-09:00:30 2\n09:10:00-09:10:10 2\n09:30:00-09:30:00 1\n', expectedExitCode: 0 },
|
|
63
|
+
] },
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
id: 'closed-range-intersection', domain: 'code',
|
|
67
|
+
objective: closedRangeIntersection,
|
|
68
|
+
examples: [{ stdin: 'alpha 0 10\nalpha 5 15\n', expectedStdout: 'alpha 5 10\n' }],
|
|
69
|
+
publicPrompt: prompt(closedRangeIntersection, [{ stdin: 'alpha 0 10\nalpha 5 15\n', expectedStdout: 'alpha 5 10\n' }]),
|
|
70
|
+
artifactPath: 'answer.sh',
|
|
71
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
72
|
+
{ stdin: '', expectedStdout: '', expectedExitCode: 0 },
|
|
73
|
+
{ stdin: 'alpha 5 5\nalpha 5 10\n', expectedStdout: 'alpha 5 5\n', expectedExitCode: 0 },
|
|
74
|
+
{ stdin: 'beta 1 4\nbeta 5 10\n', expectedStdout: 'NONE\n', expectedExitCode: 0 },
|
|
75
|
+
{ stdin: 'gamma 0 100\ngamma 10 20\ngamma 21 30\n', expectedStdout: 'NONE\n', expectedExitCode: 0 },
|
|
76
|
+
{ stdin: 'zeta 1 2\ndelta 10 0\n', expectedStdout: 'NONE\n', expectedExitCode: 0 },
|
|
77
|
+
{ stdin: 'eta 3 8\neta 3 8\n', expectedStdout: 'eta 3 8\n', expectedExitCode: 0 },
|
|
78
|
+
{ stdin: 'theta 7 7\n', expectedStdout: 'theta 7 7\n', expectedExitCode: 0 },
|
|
79
|
+
{ stdin: 'b 0 10\na 1 5\na 4 9\n', expectedStdout: 'a 4 5\nb 0 10\n', expectedExitCode: 0 },
|
|
80
|
+
] },
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
id: 'greedy-paragraph-wrap', domain: 'code',
|
|
84
|
+
objective: greedyParagraphWrap,
|
|
85
|
+
examples: [{ stdin: 'the quick brown fox jumps\n', expectedStdout: 'the quick brown fox jumps\n' }],
|
|
86
|
+
publicPrompt: prompt(greedyParagraphWrap, [{ stdin: 'the quick brown fox jumps\n', expectedStdout: 'the quick brown fox jumps\n' }]),
|
|
87
|
+
artifactPath: 'answer.sh',
|
|
88
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
89
|
+
{ stdin: '', expectedStdout: '', expectedExitCode: 0 },
|
|
90
|
+
{ stdin: ' \n\t \n', expectedStdout: '', expectedExitCode: 0 },
|
|
91
|
+
{ stdin: ' one two \n', expectedStdout: 'one two\n', expectedExitCode: 0 },
|
|
92
|
+
{ stdin: 'abcdefghij abcdefghi abcdefghi abcdefghi end\n', expectedStdout: 'abcdefghij abcdefghi abcdefghi abcdefghi\nend\n', expectedExitCode: 0 },
|
|
93
|
+
{ stdin: 'before xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx after\n', expectedStdout: 'before\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nafter\n', expectedExitCode: 0 },
|
|
94
|
+
{ stdin: 'hello world\tfoo\n\n\nbar baz\n', expectedStdout: 'hello world foo\n\nbar baz\n', expectedExitCode: 0 },
|
|
95
|
+
{ stdin: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx y\n', expectedStdout: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\ny\n', expectedExitCode: 0 },
|
|
96
|
+
{ stdin: 'p1a p1b\np1c\n\n p2a p2b\n', expectedStdout: 'p1a p1b p1c\n\np2a p2b\n', expectedExitCode: 0 },
|
|
97
|
+
] },
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
id: 'quoted-csv-account-totals', domain: 'code',
|
|
101
|
+
objective: quotedCsvAccountTotal,
|
|
102
|
+
examples: [{ stdin: 'account,jurisdiction,amount\nfoo,US,100\nfoo,US,50\n', expectedStdout: 'US\tfoo\t150\n' }],
|
|
103
|
+
publicPrompt: prompt(quotedCsvAccountTotal, [{ stdin: 'account,jurisdiction,amount\nfoo,US,100\nfoo,US,50\n', expectedStdout: 'US\tfoo\t150\n' }]),
|
|
104
|
+
artifactPath: 'answer.sh',
|
|
105
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
106
|
+
{ stdin: '', expectedStdout: '', expectedExitCode: 0 },
|
|
107
|
+
{ stdin: 'account,jurisdiction,amount\n', expectedStdout: '', expectedExitCode: 0 },
|
|
108
|
+
{ stdin: 'account,jurisdiction,amount\nacme,US,500\nacme,US,-500\n', expectedStdout: 'US\tacme\t0\n', expectedExitCode: 0 },
|
|
109
|
+
{ stdin: 'account,jurisdiction,amount\nacme,US,-0\nacme,US,7\nneg,US,3\nneg,US,-10\n', expectedStdout: 'US\tacme\t7\nUS\tneg\t-7\n', expectedExitCode: 0 },
|
|
110
|
+
{ stdin: 'account,jurisdiction,amount\n"Smith, LLC",US,100\n"Smith, LLC",US,-40\n', expectedStdout: 'US\tSmith, LLC\t60\n', expectedExitCode: 0 },
|
|
111
|
+
{ stdin: 'account,jurisdiction,amount\n"a""b",US,10\n"a""b",US,2\n', expectedStdout: 'US\ta"b\t12\n', expectedExitCode: 0 },
|
|
112
|
+
{ stdin: 'account,jurisdiction,amount\nb,J2,1\na,J1,2\na,J2,3\nb,J2,10\na,J1,-2\n', expectedStdout: 'J1\ta\t0\nJ2\ta\t3\nJ2\tb\t11\n', expectedExitCode: 0 },
|
|
113
|
+
{ stdin: 'account,jurisdiction,amount\n"X, Y ""Z""",US,3\n"X, Y ""Z""",US,-1\n', expectedStdout: 'US\tX, Y "Z"\t2\n', expectedExitCode: 0 },
|
|
114
|
+
] },
|
|
115
|
+
},
|
|
116
|
+
];
|
|
117
|
+
/** This public surface intentionally excludes independent verification vectors. */
|
|
118
|
+
export const strategyV2Corpus = frozenJson(authoredV2.map(({ verification: _verification, ...task }) => task));
|
|
119
|
+
const fullTask = (task) => ({
|
|
120
|
+
objective: task.objective, publicPrompt: task.publicPrompt, artifactPath: task.artifactPath, verification: task.verification,
|
|
121
|
+
});
|
|
122
|
+
const taskDigests = (task) => strategyGoalTaskDigests(fullTask(task));
|
|
123
|
+
/** Dataset identity commits to the complete private verification manifest. */
|
|
124
|
+
export const strategyV2Dataset = frozenJson({
|
|
125
|
+
id: 'dsh-strategy-development-v2', version: '1', split: 'development',
|
|
126
|
+
digest: acceptanceDigest(authoredV2.map(task => ({ id: task.id, domain: task.domain, task: fullTask(task), digests: taskDigests(task) }))),
|
|
127
|
+
});
|
|
128
|
+
export function strategyV2Cases() {
|
|
129
|
+
return frozenJson(authoredV2.map(task => ({ id: task.id, domain: task.domain, ...taskDigests(task) })));
|
|
130
|
+
}
|
|
131
|
+
/** Host-only full task material, including private verifier vectors. */
|
|
132
|
+
export function strategyV2Task(caseId) {
|
|
133
|
+
const task = authoredV2.find(candidate => candidate.id === caseId);
|
|
134
|
+
if (task === undefined)
|
|
135
|
+
throw new Error(`unknown strategy-v2 case: ${caseId}`);
|
|
136
|
+
return frozenJson(fullTask(task));
|
|
137
|
+
}
|
|
138
|
+
// Suite-scoped resolvers so config/executor/cli share one place that maps a suite to its corpus.
|
|
139
|
+
export const strategyCorpusForSuite = (suite) => suite === 'strategy-v2' ? strategyV2Corpus : strategyDevelopmentCorpus;
|
|
140
|
+
export const strategyDatasetForSuite = (suite) => suite === 'strategy-v2' ? strategyV2Dataset : strategyDevelopmentDataset;
|
|
141
|
+
export const strategyCasesForSuite = (suite) => suite === 'strategy-v2' ? strategyV2Cases() : strategyDevelopmentCases();
|
|
142
|
+
export const strategyTaskForSuite = (suite, caseId) => suite === 'strategy-v2' ? strategyV2Task(caseId) : strategyDevelopmentTask(caseId);
|
|
143
|
+
//# sourceMappingURL=strategy-corpus-v2.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-corpus-v2.js","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus-v2.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAEzE,OAAO,EAAE,uBAAuB,EAAyB,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EACL,UAAU,EACV,SAAS,EACT,MAAM,EACN,wBAAwB,EACxB,yBAAyB,EACzB,0BAA0B,EAC1B,uBAAuB,GAGxB,MAAM,sBAAsB,CAAA;AAG7B,MAAM,CAAC,MAAM,cAAc,GAA6B,CAAC,aAAa,EAAE,aAAa,CAAC,CAAA;AACtF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,KAAc,EAA0B,EAAE,CAAC,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,aAAa,CAAA;AAE7H,MAAM,eAAe,GAAG,SAAS,CAAC;IAChC,mIAAmI;IACnI,yNAAyN;IACzN,+OAA+O;IAC/O,6YAA6Y;IAC7Y,2EAA2E;CAC5E,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAEZ,MAAM,uBAAuB,GAAG,SAAS,CAAC;IACxC,gIAAgI;IAChI,+MAA+M;IAC/M,qMAAqM;IACrM,oNAAoN;IACpN,iCAAiC;CAClC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAEZ,MAAM,mBAAmB,GAAG,SAAS,CAAC;IACpC,gKAAgK;IAChK,mJAAmJ;IACnJ,2TAA2T;IAC3T,oKAAoK;IACpK,qEAAqE;CACtE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAEZ,MAAM,qBAAqB,GAAG,SAAS,CAAC;IACtC,oPAAoP;IACpP,qLAAqL;IACrL,2MAA2M;IAC3M,2SAA2S;IAC3S,6EAA6E;CAC9E,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAEZ,MAAM,UAAU,GAAoC;IAClD;QACE,EAAE,EAAE,mBAAmB,EAAE,MAAM,EAAE,MAAM;QACvC,SAAS,EAAE,eAAe;QAC1B,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,cAAc,EAAE,4CAA4C,EAAE,CAAC;QACrH,YAAY,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,cAAc,EAAE,4CAA4C,EAAE,CAAC,CAAC;QAClJ,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACtD,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACrF,EAAE,KAAK,EAAE,sBAAsB,EAAE,cAAc,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC/F,EAAE,KAAK,EAAE,sBAAsB,EAAE,cAAc,EAAE,4CAA4C,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACpH,EAAE,KAAK,EAAE,sBAAsB,EAAE,cAAc,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC/F,EAAE,KAAK,EAAE,gCAAgC,EAAE,cAAc,EAAE,4CAA4C,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC9H,EAAE,KAAK,EAAE,gCAAgC,EAAE,cAAc,EAAE,4CAA4C,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC9H,EAAE,KAAK,EAAE,oDAAoD,EAAE,cAAc,EAAE,iEAAiE,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACxK,EAAE;KACJ;IACD;QACE,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,MAAM;QAC/C,SAAS,EAAE,uBAAuB;QAClC,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,cAAc,EAAE,cAAc,EAAE,CAAC;QACjF,YAAY,EAAE,MAAM,CAAC,uBAAuB,EAAE,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,cAAc,EAAE,cAAc,EAAE,CAAC,CAAC;QACtH,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACtD,EAAE,KAAK,EAAE,yBAAyB,EAAE,cAAc,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACxF,EAAE,KAAK,EAAE,uBAAuB,EAAE,cAAc,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACjF,EAAE,KAAK,EAAE,yCAAyC,EAAE,cAAc,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACnG,EAAE,KAAK,EAAE,wBAAwB,EAAE,cAAc,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAClF,EAAE,KAAK,EAAE,oBAAoB,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACjF,EAAE,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC5E,EAAE,KAAK,EAAE,wBAAwB,EAAE,cAAc,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC5F,EAAE;KACJ;IACD;QACE,EAAE,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM;QAC3C,SAAS,EAAE,mBAAmB;QAC9B,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,6BAA6B,EAAE,cAAc,EAAE,6BAA6B,EAAE,CAAC;QACnG,YAAY,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,EAAE,KAAK,EAAE,6BAA6B,EAAE,cAAc,EAAE,6BAA6B,EAAE,CAAC,CAAC;QACpI,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACtD,EAAE,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACjE,EAAE,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC9E,EAAE,KAAK,EAAE,gDAAgD,EAAE,cAAc,EAAE,iDAAiD,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACnJ,EAAE,KAAK,EAAE,+DAA+D,EAAE,cAAc,EAAE,iEAAiE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAClL,EAAE,KAAK,EAAE,mCAAmC,EAAE,cAAc,EAAE,8BAA8B,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACnH,EAAE,KAAK,EAAE,8CAA8C,EAAE,cAAc,EAAE,+CAA+C,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC/I,EAAE,KAAK,EAAE,iCAAiC,EAAE,cAAc,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC9G,EAAE;KACJ;IACD;QACE,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,MAAM;QAC/C,SAAS,EAAE,qBAAqB;QAChC,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,sDAAsD,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC;QAC/G,YAAY,EAAE,MAAM,CAAC,qBAAqB,EAAE,CAAC,EAAE,KAAK,EAAE,sDAAsD,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAClJ,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACtD,EAAE,KAAK,EAAE,+BAA+B,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACnF,EAAE,KAAK,EAAE,0DAA0D,EAAE,cAAc,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC3H,EAAE,KAAK,EAAE,4EAA4E,EAAE,cAAc,EAAE,4BAA4B,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC1J,EAAE,KAAK,EAAE,yEAAyE,EAAE,cAAc,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACjJ,EAAE,KAAK,EAAE,0DAA0D,EAAE,cAAc,EAAE,eAAe,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC3H,EAAE,KAAK,EAAE,yEAAyE,EAAE,cAAc,EAAE,iCAAiC,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC5J,EAAE,KAAK,EAAE,sEAAsE,EAAE,cAAc,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC5I,EAAE;KACJ;CACF,CAAA;AAED,mFAAmF;AACnF,MAAM,CAAC,MAAM,gBAAgB,GAA6C,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;AAExJ,MAAM,QAAQ,GAAG,CAAC,IAA0B,EAAoB,EAAE,CAAC,CAAC;IAClE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY;CAC7H,CAAC,CAAA;AACF,MAAM,WAAW,GAAG,CAAC,IAA0B,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AAE3F,8EAA8E;AAC9E,MAAM,CAAC,MAAM,iBAAiB,GAA6B,UAAU,CAAC;IACpE,EAAE,EAAE,6BAA6B,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa;IACrE,MAAM,EAAE,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;CAC3I,CAAC,CAAA;AAEF,MAAM,UAAU,eAAe;IAC7B,OAAO,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AACzG,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,CAAA;IAClE,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,EAAE,CAAC,CAAA;IAC9E,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACnC,CAAC;AAED,iGAAiG;AACjG,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,KAAoB,EAA4C,EAAE,CACvG,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,yBAAyB,CAAA;AACxE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,KAAoB,EAA4B,EAAE,CACxF,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,0BAA0B,CAAA;AAC1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,KAAoB,EAA4B,EAAE,CACtF,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,wBAAwB,EAAE,CAAA;AAC1E,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,KAAoB,EAAE,MAAc,EAA8B,EAAE,CACvG,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAA"}
|
|
@@ -11,6 +11,15 @@ export interface StrategyDevelopmentCorpusTask {
|
|
|
11
11
|
expectedStdout: string;
|
|
12
12
|
}[];
|
|
13
13
|
}
|
|
14
|
+
export interface AuthoredStrategyTask extends StrategyDevelopmentCorpusTask {
|
|
15
|
+
verification: StrategyGoalTask['verification'];
|
|
16
|
+
}
|
|
17
|
+
export declare const objective: (description: string) => string;
|
|
18
|
+
export declare const prompt: (value: string, examples: readonly {
|
|
19
|
+
stdin: string;
|
|
20
|
+
expectedStdout: string;
|
|
21
|
+
}[]) => string;
|
|
22
|
+
export declare function frozenJson<T>(value: T): T;
|
|
14
23
|
/** This public surface intentionally excludes independent verification vectors. */
|
|
15
24
|
export declare const strategyDevelopmentCorpus: readonly StrategyDevelopmentCorpusTask[];
|
|
16
25
|
/** Dataset identity commits to the complete private verification manifest. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategy-corpus.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,KAAK,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE9D,MAAM,WAAW,6BAA6B;IAC5C,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAC/D;
|
|
1
|
+
{"version":3,"file":"strategy-corpus.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,KAAK,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE9D,MAAM,WAAW,6BAA6B;IAC5C,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAC/D;AAED,MAAM,WAAW,oBAAqB,SAAQ,6BAA6B;IACzE,YAAY,EAAE,gBAAgB,CAAC,cAAc,CAAC,CAAA;CAC/C;AAED,eAAO,MAAM,SAAS,GAAI,aAAa,MAAM,KAAG,MAA4E,CAAA;AAC5H,eAAO,MAAM,MAAM,GAAI,OAAO,MAAM,EAAE,UAAU,SAAS;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,EAAE,KAAG,MAMzF,CAAA;AAqDd,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CASzC;AAED,mFAAmF;AACnF,eAAO,MAAM,yBAAyB,EAAE,SAAS,6BAA6B,EAAiF,CAAA;AAO/J,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,EAAE,aAAa,CAAC,SAAS,CAG9D,CAAA;AAEF,wBAAgB,wBAAwB,IAAI,SAAS,aAAa,EAAE,CAEnE;AAED,wEAAwE;AACxE,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAIlF"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/** Public synthetic development tasks for the native Goal strategy benchmark. */
|
|
2
2
|
import { acceptanceCanonicalJson, acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
|
|
3
3
|
import { strategyGoalTaskDigests } from './strategy-goal-runtime.js';
|
|
4
|
-
const objective = (description) => `Create a POSIX shell program named answer.sh that ${description}`;
|
|
5
|
-
const prompt = (value, examples) => [
|
|
4
|
+
export const objective = (description) => `Create a POSIX shell program named answer.sh that ${description}`;
|
|
5
|
+
export const prompt = (value, examples) => [
|
|
6
6
|
value,
|
|
7
7
|
'Public examples:\n' + examples.map(example => `stdin:\n${example.stdin}\nstdout:\n${example.expectedStdout}`).join('\n'),
|
|
8
8
|
'Create a Goal by calling goal_create with the exact objective stated above.',
|
|
@@ -59,7 +59,7 @@ const authored = [
|
|
|
59
59
|
] },
|
|
60
60
|
},
|
|
61
61
|
];
|
|
62
|
-
function frozenJson(value) {
|
|
62
|
+
export function frozenJson(value) {
|
|
63
63
|
const copy = JSON.parse(acceptanceCanonicalJson(value));
|
|
64
64
|
const freeze = (entry) => {
|
|
65
65
|
if (entry === null || typeof entry !== 'object')
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategy-corpus.js","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAClG,OAAO,EAAE,uBAAuB,EAAyB,MAAM,4BAA4B,CAAA;AAgB3F,MAAM,SAAS,GAAG,CAAC,WAAmB,EAAU,EAAE,CAAC,qDAAqD,WAAW,EAAE,CAAA;
|
|
1
|
+
{"version":3,"file":"strategy-corpus.js","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAClG,OAAO,EAAE,uBAAuB,EAAyB,MAAM,4BAA4B,CAAA;AAgB3F,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,WAAmB,EAAU,EAAE,CAAC,qDAAqD,WAAW,EAAE,CAAA;AAC5H,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,QAA8D,EAAU,EAAE,CAAC;IAC/G,KAAK;IACL,oBAAoB,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,OAAO,CAAC,KAAK,cAAc,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;IACzH,6EAA6E;IAC7E,0FAA0F;IAC1F,8EAA8E;CAC/E,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAEd,MAAM,QAAQ,GAAoC;IAChD;QACE,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;QACjC,SAAS,EAAE,SAAS,CAAC,yHAAyH,CAAC;QAC/I,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACtD,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,yHAAyH,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QACxM,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAChE,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACrE,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC1D,EAAE;KACJ;IACD;QACE,EAAE,EAAE,0BAA0B,EAAE,MAAM,EAAE,MAAM;QAC9C,SAAS,EAAE,SAAS,CAAC,4OAA4O,CAAC;QAClQ,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;QAC5D,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,4OAA4O,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QACjU,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,wBAAwB,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACxF,EAAE,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC/E,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACvD,EAAE;KACJ;IACD;QACE,EAAE,EAAE,6BAA6B,EAAE,MAAM,EAAE,MAAM;QACjD,SAAS,EAAE,SAAS,CAAC,gOAAgO,CAAC;QACtP,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC;QAC1E,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,gOAAgO,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnU,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,qCAAqC,EAAE,cAAc,EAAE,6BAA6B,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACpH,EAAE,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACzE,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACvD,EAAE;KACJ;IACD;QACE,EAAE,EAAE,8BAA8B,EAAE,MAAM,EAAE,MAAM;QAClD,SAAS,EAAE,SAAS,CAAC,6MAA6M,CAAC;QACnO,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC;QACxE,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,6MAA6M,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC9S,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,uBAAuB,EAAE,cAAc,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC5F,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACzE,EAAE,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC7E,EAAE;KACJ;CACF,CAAA;AAED,MAAM,UAAU,UAAU,CAAI,KAAQ;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAM,CAAA;IAC5D,MAAM,MAAM,GAAG,CAAC,KAAc,EAAQ,EAAE;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAM;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACvD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC,CAAA;IACD,MAAM,CAAC,IAAI,CAAC,CAAA;IACZ,OAAO,IAAI,CAAA;AACb,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,MAAM,yBAAyB,GAA6C,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;AAE/J,MAAM,QAAQ,GAAG,CAAC,IAA0B,EAAoB,EAAE,CAAC,CAAC;IAClE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY;CAC7H,CAAC,CAAA;AACF,MAAM,WAAW,GAAG,CAAC,IAA0B,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AAE3F,8EAA8E;AAC9E,MAAM,CAAC,MAAM,0BAA0B,GAA6B,UAAU,CAAC;IAC7E,EAAE,EAAE,0BAA0B,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa;IAClE,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;CACzI,CAAC,CAAA;AAEF,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AACvG,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,uBAAuB,CAAC,MAAc;IACpD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,CAAA;IAChE,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAA;IACvF,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACnC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"strategy-evidence.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-evidence.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAG7G,OAAO,KAAK,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAA;
|
|
1
|
+
{"version":3,"file":"strategy-evidence.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-evidence.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAG7G,OAAO,KAAK,EAAE,8BAA8B,EAAE,MAAM,qBAAqB,CAAA;AAEzE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAEnI,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAE/D,eAAO,MAAM,wBAAwB,yCAAyC,CAAA;AAC9E,MAAM,WAAW,uBAAuB;IAAG,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAAC,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE;AAC5L,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,iBAAiB,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAA;IACzM,IAAI,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,WAAW,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;IAChG,UAAU,EAAE,SAAS;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,aAAa,GAAG,QAAQ,GAAG,SAAS,CAAC;QAAC,OAAO,EAAE,QAAQ,GAAG,kBAAkB,GAAG,WAAW,GAAG,SAAS,CAAC;QAAC,QAAQ,EAAE,SAAS;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,OAAO,CAAA;SAAE,EAAE,CAAA;KAAE,EAAE,CAAA;IACpQ,kBAAkB,EAAE,SAAS;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE;YAAE,MAAM,EAAE,WAAW,GAAG,SAAS,CAAC;YAAC,SAAS,EAAE,OAAO,CAAC;YAAC,WAAW,EAAE,MAAM,CAAA;SAAE,GAAG,IAAI,CAAA;KAAE,EAAE,CAAA;IAC7N,yBAAyB,EAAE,MAAM,GAAG,IAAI,CAAA;IACxC,QAAQ,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,WAAW,GAAG,cAAc,CAAC;QAAC,QAAQ,EAAE,sBAAsB,CAAC;QAAC,OAAO,EAAE,uBAAuB,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,EAAE,CAAA;CACvK;AACD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,OAAO,wBAAwB,CAAC;IAAC,OAAO,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;IACpH,KAAK,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAAC,UAAU,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,CAAA;KAAE,CAAC;IAAC,QAAQ,EAAE,iBAAiB,CAAA;IACxI,KAAK,EAAE,8BAA8B,CAAC;IAAC,MAAM,EAAE,sBAAsB,CAAA;IACrE,OAAO,EAAE;QAAE,MAAM,EAAE,WAAW,GAAG,SAAS,CAAC;QAAC,OAAO,EAAE,UAAU,GAAG,cAAc,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,CAAA;CACnH;AACD,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,qBAAqB,EAAE,MAAM,CAAC;IAAG,UAAU,EAAE,MAAM,CAAA;CAAE;AAC1G,MAAM,WAAW,4BAA4B;IAAG,cAAc,EAAE,MAAM,CAAC;IAAC,kBAAkB,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAE;AAEjI,eAAO,MAAM,uBAAuB,wCAAwC,CAAA;AAC5E,MAAM,WAAW,uBAAuB;IACtC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,KAAK,EAAE,OAAO,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAA;IAChE,OAAO,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,CAAA;IAC5C,KAAK,EAAE,8BAA8B,GAAG,IAAI,CAAA;IAC5C,4FAA4F;IAC5F,YAAY,EAAE,OAAO,GAAG,IAAI,CAAA;IAC5B,mBAAmB,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;KAAE,GAAG,IAAI,CAAA;IAC5F,YAAY,EAAE,OAAO,GAAG,WAAW,GAAG,SAAS,GAAG,UAAU,GAAG,IAAI,CAAA;CACpE;AACD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,OAAO,uBAAuB,CAAC;IAAC,OAAO,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;IACnH,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,SAAS,GAAG,qBAAqB,GAAG,iBAAiB,GAAG,eAAe,CAAA;IACjH,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,uBAAuB,CAAA;CAClC;AACD,MAAM,WAAW,qBAAsB,SAAQ,IAAI,CAAC,oBAAoB,EAAE,MAAM,CAAC;IAAG,UAAU,EAAE,MAAM,CAAA;CAAE;AACxG,eAAO,MAAM,oBAAoB,qCAAqC,CAAA;AACtE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,OAAO,oBAAoB,CAAC;IAAC,OAAO,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;IAChH,oBAAoB,EAAE,MAAM,CAAA;IAC5B,YAAY,EAAE;QACZ,cAAc,EAAE,OAAO,CAAA;QACvB,iBAAiB,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,oBAAoB,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAC;YAAC,iBAAiB,EAAE,MAAM,CAAC;YAAC,aAAa,EAAE,MAAM,CAAA;SAAE,CAAA;QACxJ,QAAQ,EAAE,SAAS;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;YAAC,WAAW,EAAE,OAAO,GAAG,IAAI,CAAA;SAAE,EAAE,CAAA;KAClG,CAAA;CACF;AACD,MAAM,WAAW,kBAAmB,SAAQ,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC;IAAG,UAAU,EAAE,MAAM,CAAA;CAAE;AA+LlG,qBAAa,qBAAqB;;gBAEpB,KAAK,EAAE,4BAA4B;IAQ/C,KAAK,CAAC,KAAK,EAAE,qBAAqB,GAAG,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAmB/E,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,sBAAsB,CAAC;IAgB7G,8FAA8F;IAC9F,YAAY,CAAC,KAAK,EAAE,oBAAoB,GAAG,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAMrF,WAAW,CAAC,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAAC;IAGnH,SAAS,CAAC,KAAK,EAAE,iBAAiB,GAAG,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAM/E,QAAQ,CAAC,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,kBAAkB,CAAC;CA6C9G"}
|
|
@@ -138,16 +138,22 @@ function requestFor(plan, cell) {
|
|
|
138
138
|
fail('cell has no bound request');
|
|
139
139
|
return { planId: journal.id, dataset: journal.dataset, cell: exact, task, variant, budget: journal.budget };
|
|
140
140
|
}
|
|
141
|
-
function parseMeter(value) {
|
|
141
|
+
function parseMeter(value, expectedMode) {
|
|
142
142
|
const raw = keys(evidenceSnapshot(value), ['observationMode', 'budget', 'modelCalls', 'toolCalls', 'activeToolCalls', 'inputTokens', 'outputTokens', 'costUsdMicros', 'heldModelCalls', 'heldInputTokens', 'heldOutputTokens', 'heldCostUsdMicros', 'traces']);
|
|
143
|
-
if (raw.observationMode !== 'enforced-upper-bound-provider-output' || !Array.isArray(raw.traces) || raw.traces.length > 10_001)
|
|
143
|
+
if ((raw.observationMode !== 'enforced-upper-bound-provider-output' && raw.observationMode !== 'observed-call-count') || !Array.isArray(raw.traces) || raw.traces.length > 10_001)
|
|
144
144
|
fail('invalid meter snapshot');
|
|
145
|
+
// Cross-check the meter's self-reported mode against the plaintext mode frozen in the plan.
|
|
146
|
+
if (raw.observationMode !== expectedMode)
|
|
147
|
+
fail('meter observation mode differs from frozen plan');
|
|
148
|
+
const callsMode = raw.observationMode === 'observed-call-count';
|
|
145
149
|
const budget = keys(raw.budget, ['durationMs', 'inputTokens', 'outputTokens', 'costUsdMicros', 'toolCalls']);
|
|
146
150
|
for (const name of ['durationMs', 'inputTokens', 'outputTokens', 'toolCalls'])
|
|
147
151
|
if (!Number.isSafeInteger(budget[name]) || budget[name] < 0)
|
|
148
152
|
fail('invalid meter budget');
|
|
149
153
|
if (budget.costUsdMicros !== null && (!Number.isSafeInteger(budget.costUsdMicros) || budget.costUsdMicros < 0))
|
|
150
154
|
fail('invalid meter budget');
|
|
155
|
+
if (callsMode && (budget.costUsdMicros !== null || budget.inputTokens !== 0))
|
|
156
|
+
fail('invalid call-count meter budget');
|
|
151
157
|
for (const name of ['modelCalls', 'toolCalls', 'activeToolCalls', 'inputTokens', 'outputTokens', 'heldModelCalls', 'heldInputTokens', 'heldOutputTokens'])
|
|
152
158
|
if (!Number.isSafeInteger(raw[name]) || raw[name] < 0)
|
|
153
159
|
fail('invalid meter counters');
|
|
@@ -166,6 +172,29 @@ function parseMeter(value) {
|
|
|
166
172
|
if (!Number.isSafeInteger(item.id) || item.id < 1 || ids.has(item.id) || !Number.isSafeInteger(item.startedAt) || item.startedAt < 0 || item.completedAt !== null && (!Number.isSafeInteger(item.completedAt) || item.completedAt < item.startedAt) || !(item.sessionId === null || typeof item.sessionId === 'string') || !(item.agentId === null || typeof item.agentId === 'string') || typeof item.dispatched !== 'boolean' || !['preflight', 'reserved', 'streaming', 'settled', 'retained', 'rejected'].includes(item.phase) || !Number.isSafeInteger(item.reservedInputTokens) || !Number.isSafeInteger(item.reservedOutputTokens) || item.reservedInputTokens < 0 || item.reservedOutputTokens < 0 || !(item.reservedCostUsdMicros === null || Number.isSafeInteger(item.reservedCostUsdMicros) && item.reservedCostUsdMicros >= 0) || !(item.reason === null || ['cancelled', 'disposed', 'request-contract', 'input-bound', 'shared-budget', 'stream', 'usage', 'cost', 'tool-budget'].includes(item.reason)))
|
|
167
173
|
fail('invalid meter trace');
|
|
168
174
|
ids.add(item.id);
|
|
175
|
+
if (callsMode) {
|
|
176
|
+
// Call-count mode never measures tokens or money: every reservation is zero,
|
|
177
|
+
// cost is null, and a settled trace carries usage=null while still counting one call.
|
|
178
|
+
if (item.reservedInputTokens !== 0 || item.reservedOutputTokens !== 0 || item.reservedCostUsdMicros !== null || item.usage !== null)
|
|
179
|
+
fail('invalid call-count meter trace');
|
|
180
|
+
if (item.phase === 'settled') {
|
|
181
|
+
if (!item.dispatched || item.completedAt === null || item.reason !== null)
|
|
182
|
+
fail('invalid settled trace');
|
|
183
|
+
calls++;
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
if (item.phase === 'rejected' && item.dispatched)
|
|
187
|
+
fail('invalid rejected reservation');
|
|
188
|
+
if (item.phase === 'preflight' && (item.dispatched || item.completedAt !== null || item.reason !== null)
|
|
189
|
+
|| (item.phase === 'reserved' || item.phase === 'streaming') && (item.completedAt !== null || item.reason !== null)
|
|
190
|
+
|| item.phase === 'streaming' && !item.dispatched
|
|
191
|
+
|| (item.phase === 'retained' || item.phase === 'rejected') && (item.completedAt === null || item.reason === null))
|
|
192
|
+
fail('invalid meter phase');
|
|
193
|
+
if (item.phase === 'reserved' || item.phase === 'streaming' || item.phase === 'retained')
|
|
194
|
+
heldCalls++;
|
|
195
|
+
}
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
169
198
|
const reserved = item.reservedInputTokens > 0 || item.reservedOutputTokens > 0;
|
|
170
199
|
if (item.phase === 'settled') {
|
|
171
200
|
if (!item.dispatched || item.completedAt === null || item.reason !== null || !reserved)
|
|
@@ -199,13 +228,25 @@ function parseMeter(value) {
|
|
|
199
228
|
}
|
|
200
229
|
}
|
|
201
230
|
});
|
|
202
|
-
if (
|
|
231
|
+
if (callsMode) {
|
|
232
|
+
if (calls !== raw.modelCalls || heldCalls !== raw.heldModelCalls || raw.activeToolCalls > raw.toolCalls
|
|
233
|
+
|| input !== 0 || output !== 0 || heldInput !== 0 || heldOutput !== 0 || heldCost !== null
|
|
234
|
+
|| raw.inputTokens !== 0 || raw.outputTokens !== 0 || raw.costUsdMicros !== null
|
|
235
|
+
|| raw.heldInputTokens !== 0 || raw.heldOutputTokens !== 0 || raw.heldCostUsdMicros !== null)
|
|
236
|
+
fail('meter aggregate mismatch');
|
|
237
|
+
}
|
|
238
|
+
else if (calls !== raw.modelCalls || input !== raw.inputTokens || output !== raw.outputTokens || heldCalls !== raw.heldModelCalls || heldInput !== raw.heldInputTokens || heldOutput !== raw.heldOutputTokens || heldCost !== raw.heldCostUsdMicros || raw.activeToolCalls > raw.toolCalls)
|
|
203
239
|
fail('meter aggregate mismatch');
|
|
204
240
|
return raw;
|
|
205
241
|
}
|
|
206
242
|
function validateMeterBounds(meter, plan, budget) {
|
|
207
243
|
if (meter.modelCalls > plan.execution.modelCalls || meter.heldModelCalls > plan.execution.modelCalls || meter.modelCalls + meter.heldModelCalls > plan.execution.modelCalls
|
|
208
|
-
|| meter.toolCalls > budget.toolCalls || meter.activeToolCalls > meter.toolCalls
|
|
244
|
+
|| meter.toolCalls > budget.toolCalls || meter.activeToolCalls > meter.toolCalls)
|
|
245
|
+
fail('meter exceeds bound plan');
|
|
246
|
+
// Call-count mode has no token or monetary budget; only call/tool bounds are enforceable.
|
|
247
|
+
if (meter.observationMode === 'observed-call-count')
|
|
248
|
+
return;
|
|
249
|
+
if (meter.inputTokens > budget.inputTokens || meter.heldInputTokens > budget.inputTokens || meter.inputTokens + meter.heldInputTokens > budget.inputTokens
|
|
209
250
|
|| meter.outputTokens > budget.outputTokens || meter.heldOutputTokens > budget.outputTokens || meter.outputTokens + meter.heldOutputTokens > budget.outputTokens
|
|
210
251
|
|| meter.traces.some(trace => trace.reservedInputTokens > budget.inputTokens || trace.reservedOutputTokens > plan.execution.maxOutputTokensPerCall)
|
|
211
252
|
|| budget.costUsdMicros !== null && (meter.costUsdMicros === null || meter.heldCostUsdMicros === null || meter.costUsdMicros > budget.costUsdMicros || meter.heldCostUsdMicros > budget.costUsdMicros || meter.costUsdMicros + meter.heldCostUsdMicros > budget.costUsdMicros))
|
|
@@ -262,7 +303,7 @@ function parseNative(value) {
|
|
|
262
303
|
fail('receipt binding differs'); contractIds.add(contract.id); receiptIds.add(proof.id); });
|
|
263
304
|
return raw;
|
|
264
305
|
}
|
|
265
|
-
function parse(input) {
|
|
306
|
+
function parse(input, expectedMode) {
|
|
266
307
|
const raw = keys(evidenceSnapshot(input), ['protocol', 'version', 'planDigest', 'request', 'input', 'acceptance', 'versions', 'meter', 'native', 'outcome']);
|
|
267
308
|
if (raw.protocol !== strategyEvidenceProtocol || raw.version !== 1)
|
|
268
309
|
fail('unsupported protocol');
|
|
@@ -279,7 +320,7 @@ function parse(input) {
|
|
|
279
320
|
fail('invalid outcome');
|
|
280
321
|
const result = verdict(outcome.verdict);
|
|
281
322
|
const native = parseNative(raw.native);
|
|
282
|
-
const meter = parseMeter(raw.meter);
|
|
323
|
+
const meter = parseMeter(raw.meter, expectedMode);
|
|
283
324
|
const attributedSessions = new Set([native.parent.sessionId, ...native.strategies.flatMap(strategy => strategy.children.map(child => child.sessionId))]);
|
|
284
325
|
const knownNativeStop = native.runs.length > 0 && native.runs.every(run => run.executionStatus === 'succeeded' && run.quiescent)
|
|
285
326
|
&& native.strategies.every(strategy => strategy.outcome !== 'unknown' && strategy.children.every(child => child.quiescent))
|
|
@@ -319,7 +360,7 @@ export class StrategyEvidenceStore {
|
|
|
319
360
|
const plan = parseStrategyBenchmarkPlan(input.plan);
|
|
320
361
|
const expected = requestFor(plan, input.request.cell);
|
|
321
362
|
const { plan: _plan, ...unbound } = input;
|
|
322
|
-
const object = parse({ ...unbound, planDigest: strategyBenchmarkPlanDigest(plan) });
|
|
363
|
+
const object = parse({ ...unbound, planDigest: strategyBenchmarkPlanDigest(plan) }, plan.execution.observationMode);
|
|
323
364
|
if (!identical(object.request, expected) || !identical(object.meter.budget, expected.budget) || object.input.digest !== expected.task.inputDigest || object.acceptance.digest !== expected.task.acceptanceDigest || !identical(object.versions, expected.variant.versions))
|
|
324
365
|
fail('request identity drift');
|
|
325
366
|
validateMeterBounds(object.meter, plan, expected.budget);
|
|
@@ -373,7 +414,7 @@ export class StrategyEvidenceStore {
|
|
|
373
414
|
read(planInput, cell, digest) {
|
|
374
415
|
const plan = parseStrategyBenchmarkPlan(planInput);
|
|
375
416
|
const expected = requestFor(plan, cell);
|
|
376
|
-
const evidence = parse(this.#readEnvelope(strategyEvidenceProtocol, digest));
|
|
417
|
+
const evidence = parse(this.#readEnvelope(strategyEvidenceProtocol, digest), plan.execution.observationMode);
|
|
377
418
|
if (evidenceDigest(evidence) !== digest || evidence.planDigest !== strategyBenchmarkPlanDigest(plan) || !identical(evidence.request, expected) || !identical(evidence.meter.budget, expected.budget)
|
|
378
419
|
|| evidence.input.digest !== expected.task.inputDigest || evidence.acceptance.digest !== expected.task.acceptanceDigest || !identical(evidence.versions, expected.variant.versions))
|
|
379
420
|
fail('evidence binding differs');
|
|
@@ -510,7 +551,7 @@ function parseFailure(value, plan, cell) {
|
|
|
510
551
|
fail('invalid historical goal observation');
|
|
511
552
|
}
|
|
512
553
|
if (snapshot.meter !== null) {
|
|
513
|
-
const meter = parseMeter(snapshot.meter);
|
|
554
|
+
const meter = parseMeter(snapshot.meter, plan.execution.observationMode);
|
|
514
555
|
if (!identical(meter.budget, expected.budget))
|
|
515
556
|
fail('failure meter budget drift');
|
|
516
557
|
validateMeterBounds(meter, plan, expected.budget);
|