@dsh-enhanced/assistant-evaluation 0.1.24 → 0.1.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -3
- package/bin/dsh-benchmark.js +21 -0
- package/lib/benchmark/cli.d.ts +6 -0
- package/lib/benchmark/cli.d.ts.map +1 -0
- package/lib/benchmark/cli.js +332 -0
- package/lib/benchmark/cli.js.map +1 -0
- package/lib/benchmark/corpus.d.ts +28 -0
- package/lib/benchmark/corpus.d.ts.map +1 -0
- package/lib/benchmark/corpus.js +173 -0
- package/lib/benchmark/corpus.js.map +1 -0
- package/lib/benchmark/deepseek.d.ts +15 -0
- package/lib/benchmark/deepseek.d.ts.map +1 -0
- package/lib/benchmark/deepseek.js +143 -0
- package/lib/benchmark/deepseek.js.map +1 -0
- package/lib/benchmark/holdout-evidence.d.ts +126 -0
- package/lib/benchmark/holdout-evidence.d.ts.map +1 -0
- package/lib/benchmark/holdout-evidence.js +466 -0
- package/lib/benchmark/holdout-evidence.js.map +1 -0
- package/lib/benchmark/holdout-protocol.d.ts +68 -0
- package/lib/benchmark/holdout-protocol.d.ts.map +1 -0
- package/lib/benchmark/holdout-protocol.js +271 -0
- package/lib/benchmark/holdout-protocol.js.map +1 -0
- package/lib/benchmark/holdout-provider.d.ts +26 -0
- package/lib/benchmark/holdout-provider.d.ts.map +1 -0
- package/lib/benchmark/holdout-provider.js +382 -0
- package/lib/benchmark/holdout-provider.js.map +1 -0
- package/lib/benchmark/holdout.d.ts +83 -0
- package/lib/benchmark/holdout.d.ts.map +1 -0
- package/lib/benchmark/holdout.js +391 -0
- package/lib/benchmark/holdout.js.map +1 -0
- package/lib/benchmark/index.d.ts +9 -0
- package/lib/benchmark/index.d.ts.map +1 -0
- package/lib/benchmark/index.js +7 -0
- package/lib/benchmark/index.js.map +1 -0
- package/lib/benchmark/memory-corpus.d.ts +55 -0
- package/lib/benchmark/memory-corpus.d.ts.map +1 -0
- package/lib/benchmark/memory-corpus.js +153 -0
- package/lib/benchmark/memory-corpus.js.map +1 -0
- package/lib/benchmark/memory-runtime.d.ts +63 -0
- package/lib/benchmark/memory-runtime.d.ts.map +1 -0
- package/lib/benchmark/memory-runtime.js +75 -0
- package/lib/benchmark/memory-runtime.js.map +1 -0
- package/lib/benchmark/native.d.ts +46 -0
- package/lib/benchmark/native.d.ts.map +1 -0
- package/lib/benchmark/native.js +300 -0
- package/lib/benchmark/native.js.map +1 -0
- package/lib/benchmark/report.d.ts +4 -0
- package/lib/benchmark/report.d.ts.map +1 -0
- package/lib/benchmark/report.js +153 -0
- package/lib/benchmark/report.js.map +1 -0
- package/lib/benchmark/runner.d.ts +25 -0
- package/lib/benchmark/runner.d.ts.map +1 -0
- package/lib/benchmark/runner.js +119 -0
- package/lib/benchmark/runner.js.map +1 -0
- package/lib/benchmark/schema.d.ts +20 -0
- package/lib/benchmark/schema.d.ts.map +1 -0
- package/lib/benchmark/schema.js +183 -0
- package/lib/benchmark/schema.js.map +1 -0
- package/lib/benchmark/store.d.ts +18 -0
- package/lib/benchmark/store.d.ts.map +1 -0
- package/lib/benchmark/store.js +321 -0
- package/lib/benchmark/store.js.map +1 -0
- package/lib/benchmark/strategy-capabilities.d.ts +154 -0
- package/lib/benchmark/strategy-capabilities.d.ts.map +1 -0
- package/lib/benchmark/strategy-capabilities.js +300 -0
- package/lib/benchmark/strategy-capabilities.js.map +1 -0
- package/lib/benchmark/strategy-config.d.ts +26 -0
- package/lib/benchmark/strategy-config.d.ts.map +1 -0
- package/lib/benchmark/strategy-config.js +80 -0
- package/lib/benchmark/strategy-config.js.map +1 -0
- package/lib/benchmark/strategy-corpus.d.ts +21 -0
- package/lib/benchmark/strategy-corpus.d.ts.map +1 -0
- package/lib/benchmark/strategy-corpus.js +95 -0
- package/lib/benchmark/strategy-corpus.js.map +1 -0
- package/lib/benchmark/strategy-evidence.d.ts +161 -0
- package/lib/benchmark/strategy-evidence.d.ts.map +1 -0
- package/lib/benchmark/strategy-evidence.js +520 -0
- package/lib/benchmark/strategy-evidence.js.map +1 -0
- package/lib/benchmark/strategy-executor.d.ts +27 -0
- package/lib/benchmark/strategy-executor.d.ts.map +1 -0
- package/lib/benchmark/strategy-executor.js +178 -0
- package/lib/benchmark/strategy-executor.js.map +1 -0
- package/lib/benchmark/strategy-goal-runtime.d.ts +78 -0
- package/lib/benchmark/strategy-goal-runtime.d.ts.map +1 -0
- package/lib/benchmark/strategy-goal-runtime.js +324 -0
- package/lib/benchmark/strategy-goal-runtime.js.map +1 -0
- package/lib/benchmark/strategy-meter.d.ts +62 -0
- package/lib/benchmark/strategy-meter.d.ts.map +1 -0
- package/lib/benchmark/strategy-meter.js +236 -0
- package/lib/benchmark/strategy-meter.js.map +1 -0
- package/lib/benchmark/strategy-owner.d.ts +71 -0
- package/lib/benchmark/strategy-owner.d.ts.map +1 -0
- package/lib/benchmark/strategy-owner.js +282 -0
- package/lib/benchmark/strategy-owner.js.map +1 -0
- package/lib/benchmark/strategy-plan.d.ts +51 -0
- package/lib/benchmark/strategy-plan.d.ts.map +1 -0
- package/lib/benchmark/strategy-plan.js +71 -0
- package/lib/benchmark/strategy-plan.js.map +1 -0
- package/lib/benchmark/strategy-policy.d.ts +16 -0
- package/lib/benchmark/strategy-policy.d.ts.map +1 -0
- package/lib/benchmark/strategy-policy.js +77 -0
- package/lib/benchmark/strategy-policy.js.map +1 -0
- package/lib/benchmark/strategy.d.ts +17 -0
- package/lib/benchmark/strategy.d.ts.map +1 -0
- package/lib/benchmark/strategy.js +21 -0
- package/lib/benchmark/strategy.js.map +1 -0
- package/lib/benchmark/types.d.ts +133 -0
- package/lib/benchmark/types.d.ts.map +1 -0
- package/lib/benchmark/types.js +3 -0
- package/lib/benchmark/types.js.map +1 -0
- package/lib/benchmark/usage.d.ts +36 -0
- package/lib/benchmark/usage.d.ts.map +1 -0
- package/lib/benchmark/usage.js +84 -0
- package/lib/benchmark/usage.js.map +1 -0
- package/lib/service.d.ts +50 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +556 -9
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +94 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +27 -5
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +324 -25
- package/lib/store.js.map +1 -1
- package/lib/types.d.ts +114 -5
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +125 -18
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capability identity and live probes for the native Goal strategy benchmark.
|
|
3
|
+
*
|
|
4
|
+
* A plan may bind an expectation, but cannot attest a process it has not yet
|
|
5
|
+
* constructed. This module consequently keeps resolved source identity and
|
|
6
|
+
* observations separate: `capabilities` is safe to freeze in a plan, while a
|
|
7
|
+
* `StrategyCapabilityRuntimeObservation` is a per-cell fact collected after
|
|
8
|
+
* the owner has mounted its services.
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from 'node:crypto';
|
|
11
|
+
import { createRequire } from 'node:module';
|
|
12
|
+
import { readFileSync, readdirSync, realpathSync, statSync } from 'node:fs';
|
|
13
|
+
import { dirname, join, relative, resolve, sep } from 'node:path';
|
|
14
|
+
import { acceptanceCanonicalJson, acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
|
|
15
|
+
import { assertStrategyPolicyConfiguration, strategyPolicyRecipe } from './strategy-policy.js';
|
|
16
|
+
export const strategyBenchmarkMinimalRecipe = Object.freeze({
|
|
17
|
+
protocol: 'dsh-native-goal-strategy-recipe-v1',
|
|
18
|
+
parentTools: Object.freeze(['goal_create', 'goal_context', 'goal_checkpoint', 'isolation_run']),
|
|
19
|
+
strategyTool: 'goal_strategy',
|
|
20
|
+
strategyKinds: Object.freeze(['investigate', 'review', 'compare']),
|
|
21
|
+
/** Strategy is the only arm difference. Do not add task data here. */
|
|
22
|
+
strategyEnabled: true,
|
|
23
|
+
});
|
|
24
|
+
const sha256 = (value) => createHash('sha256').update(value).digest('hex');
|
|
25
|
+
const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype;
|
|
26
|
+
const canonical = (value) => acceptanceCanonicalJson(value);
|
|
27
|
+
const clone = (value) => JSON.parse(JSON.stringify(value));
|
|
28
|
+
function fail(message) { throw new Error(`strategy capabilities: ${message}`); }
|
|
29
|
+
function checkedText(value, label) {
|
|
30
|
+
if (typeof value !== 'string' || value.trim() === '' || Buffer.byteLength(value, 'utf8') > 256 * 1024)
|
|
31
|
+
fail(`invalid ${label}`);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
function checkedModule(value) {
|
|
35
|
+
if (!isPlainObject(value) || !/^(?:@[^/]+\/)?[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*$/u.test(value.packageName)
|
|
36
|
+
|| !Array.isArray(value.files) || value.files.length === 0)
|
|
37
|
+
fail('invalid source module');
|
|
38
|
+
const files = value.files.map(file => {
|
|
39
|
+
if (typeof file !== 'string' || file === '' || file.startsWith('/') || file.split(/[\\/]/u).some(part => part === '' || part === '.' || part === '..'))
|
|
40
|
+
fail('invalid source file');
|
|
41
|
+
return file;
|
|
42
|
+
}).sort();
|
|
43
|
+
if (new Set(files).size !== files.length)
|
|
44
|
+
fail('duplicate source file');
|
|
45
|
+
return { packageName: value.packageName, files };
|
|
46
|
+
}
|
|
47
|
+
function packageRoot(entry, packageName) {
|
|
48
|
+
let current = dirname(realpathSync(entry));
|
|
49
|
+
while (true) {
|
|
50
|
+
const manifest = join(current, 'package.json');
|
|
51
|
+
try {
|
|
52
|
+
const json = JSON.parse(readFileSync(manifest, 'utf8'));
|
|
53
|
+
if (json.name === packageName)
|
|
54
|
+
return current;
|
|
55
|
+
}
|
|
56
|
+
catch { }
|
|
57
|
+
const parent = dirname(current);
|
|
58
|
+
if (parent === current)
|
|
59
|
+
fail(`resolved package root missing for ${packageName}`);
|
|
60
|
+
current = parent;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export function resolveStrategyCapabilitySource(input) {
|
|
64
|
+
const module = checkedModule(input.module);
|
|
65
|
+
const directory = resolve(checkedText(input.resolverDirectory, 'resolver directory'));
|
|
66
|
+
const resolver = createRequire(join(directory, 'strategy-capabilities-resolver.cjs'));
|
|
67
|
+
let entry;
|
|
68
|
+
try {
|
|
69
|
+
entry = resolver.resolve(module.packageName);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
fail(`cannot resolve ${module.packageName}`);
|
|
73
|
+
}
|
|
74
|
+
const root = packageRoot(entry, module.packageName);
|
|
75
|
+
const manifest = readFileSync(join(root, 'package.json'), 'utf8');
|
|
76
|
+
let version;
|
|
77
|
+
try {
|
|
78
|
+
version = JSON.parse(manifest).version;
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
fail(`invalid package manifest for ${module.packageName}`);
|
|
82
|
+
}
|
|
83
|
+
if (typeof version !== 'string' || version === '')
|
|
84
|
+
fail(`invalid package version for ${module.packageName}`);
|
|
85
|
+
const files = module.files.map(path => {
|
|
86
|
+
const target = realpathSync(resolve(root, path));
|
|
87
|
+
if (relative(root, target).startsWith(`..${sep}`) || relative(root, target) === '..' || !statSync(target).isFile())
|
|
88
|
+
fail(`source escapes package root: ${module.packageName}/${path}`);
|
|
89
|
+
return Object.freeze({ path, digest: sha256(readFileSync(target)) });
|
|
90
|
+
});
|
|
91
|
+
const identity = { packageName: module.packageName, version, manifestDigest: sha256(manifest), files };
|
|
92
|
+
return Object.freeze({ ...identity, digest: acceptanceDigest(identity) });
|
|
93
|
+
}
|
|
94
|
+
function identities(directory, modules) {
|
|
95
|
+
if (!Array.isArray(modules) || modules.length === 0)
|
|
96
|
+
fail('empty source group');
|
|
97
|
+
return Object.freeze(modules.map(module => resolveStrategyCapabilitySource({ resolverDirectory: directory, module })));
|
|
98
|
+
}
|
|
99
|
+
function definition(value) {
|
|
100
|
+
if (!isPlainObject(value) || !/^[a-z][a-z0-9_]{0,127}$/u.test(value.name) || typeof value.description !== 'string')
|
|
101
|
+
fail('invalid tool definition');
|
|
102
|
+
// `execute` is deliberately absent: a source identity plus this model-visible
|
|
103
|
+
// definition is attestable without pretending closures have serializable code.
|
|
104
|
+
return Object.freeze({ name: value.name, description: value.description, parameters: clone(value.parameters), output: clone(value.output) });
|
|
105
|
+
}
|
|
106
|
+
function definitions(values) {
|
|
107
|
+
if (!Array.isArray(values) || values.length === 0)
|
|
108
|
+
fail('empty tool definition group');
|
|
109
|
+
const result = values.map(definition).sort((a, b) => a.name.localeCompare(b.name));
|
|
110
|
+
if (new Set(result.map(item => item.name)).size !== result.length)
|
|
111
|
+
fail('duplicate tool definition');
|
|
112
|
+
return Object.freeze(result);
|
|
113
|
+
}
|
|
114
|
+
function policy(value) {
|
|
115
|
+
if (!isPlainObject(value) || (value.effect !== 'allow' && value.effect !== 'deny'))
|
|
116
|
+
fail('invalid policy probe');
|
|
117
|
+
return Object.freeze({ request: clone(value.request), effect: value.effect, config: clone(value.config) });
|
|
118
|
+
}
|
|
119
|
+
function sourceDigest(label, source, extra) {
|
|
120
|
+
return acceptanceDigest({ label, source: source.map(item => item.digest), extra });
|
|
121
|
+
}
|
|
122
|
+
function createStrategyCapabilityExpectation(input) {
|
|
123
|
+
const resolverDirectory = resolve(checkedText(input.resolverDirectory, 'resolver directory'));
|
|
124
|
+
const persona = checkedText(input.persona, 'persona');
|
|
125
|
+
if (input.recipe !== undefined && canonical(input.recipe) !== canonical(strategyBenchmarkMinimalRecipe))
|
|
126
|
+
fail('strategy recipe differs from fixed benchmark recipe');
|
|
127
|
+
const commonTools = definitions(input.commonTools);
|
|
128
|
+
const strategyTools = definitions(input.strategyTools);
|
|
129
|
+
const commonPolicy = policy(input.commonPolicy);
|
|
130
|
+
const strategyPolicy = policy(input.strategyPolicy);
|
|
131
|
+
const sourceIdentity = Object.freeze({
|
|
132
|
+
'common.tools': identities(resolverDirectory, input.sources.common.tools),
|
|
133
|
+
'common.policy': identities(resolverDirectory, input.sources.common.policy),
|
|
134
|
+
'common.runtime': identities(resolverDirectory, input.sources.common.runtime),
|
|
135
|
+
'strategy.guide': identities(resolverDirectory, input.sources.strategy.guide),
|
|
136
|
+
'strategy.tool': identities(resolverDirectory, input.sources.strategy.tool),
|
|
137
|
+
'strategy.policy': identities(resolverDirectory, input.sources.strategy.policy),
|
|
138
|
+
'strategy.runtime': identities(resolverDirectory, input.sources.strategy.runtime),
|
|
139
|
+
});
|
|
140
|
+
const recipeDigest = acceptanceDigest(strategyBenchmarkMinimalRecipe);
|
|
141
|
+
const capabilities = Object.freeze({
|
|
142
|
+
common: Object.freeze({ persona: acceptanceDigest({ persona }), tools: sourceDigest('common-tools', sourceIdentity['common.tools'], { recipeDigest, definitions: commonTools }),
|
|
143
|
+
policy: sourceDigest('common-policy', sourceIdentity['common.policy'], { probe: commonPolicy }), runtime: sourceDigest('common-runtime', sourceIdentity['common.runtime'], { recipeDigest }) }),
|
|
144
|
+
strategy: Object.freeze({ guide: sourceDigest('strategy-guide', sourceIdentity['strategy.guide'], { recipeDigest }), tool: sourceDigest('strategy-tool', sourceIdentity['strategy.tool'], { recipeDigest, definitions: strategyTools }),
|
|
145
|
+
policy: sourceDigest('strategy-policy', sourceIdentity['strategy.policy'], { probe: strategyPolicy }), runtime: sourceDigest('strategy-runtime', sourceIdentity['strategy.runtime'], { recipeDigest }) }),
|
|
146
|
+
});
|
|
147
|
+
return Object.freeze({ capabilities, sourceIdentity, commonTools, strategyTools, commonPolicy, strategyPolicy, persona, recipeDigest, resolverDirectory,
|
|
148
|
+
resolvedSourceModules: Object.freeze({ 'common.tools': input.sources.common.tools, 'common.policy': input.sources.common.policy, 'common.runtime': input.sources.common.runtime,
|
|
149
|
+
'strategy.guide': input.sources.strategy.guide, 'strategy.tool': input.sources.strategy.tool, 'strategy.policy': input.sources.strategy.policy, 'strategy.runtime': input.sources.strategy.runtime }) });
|
|
150
|
+
}
|
|
151
|
+
// Expected tool names are a recipe, not invented schemas. Actual schemas are
|
|
152
|
+
// read from the mounted registry and compared with each model request below.
|
|
153
|
+
const fixedToolDefinitions = Object.freeze(strategyBenchmarkMinimalRecipe.parentTools.map(name => Object.freeze({ name, description: '', parameters: null, output: null })));
|
|
154
|
+
const fixedStrategyToolDefinitions = Object.freeze([
|
|
155
|
+
Object.freeze({ name: 'goal_strategy', description: '', parameters: null, output: null }),
|
|
156
|
+
]);
|
|
157
|
+
function allLibFiles(root) {
|
|
158
|
+
const lib = join(root, 'lib');
|
|
159
|
+
const files = [];
|
|
160
|
+
const visit = (directory) => {
|
|
161
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
162
|
+
const path = join(directory, entry.name);
|
|
163
|
+
if (entry.isDirectory())
|
|
164
|
+
visit(path);
|
|
165
|
+
else if (entry.isFile() && entry.name.endsWith('.js'))
|
|
166
|
+
files.push(relative(root, path));
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
visit(lib);
|
|
170
|
+
if (files.length === 0)
|
|
171
|
+
fail('resolved package has no deployed lib JavaScript');
|
|
172
|
+
return files.sort();
|
|
173
|
+
}
|
|
174
|
+
function fixedModule(resolverDirectory, packageName) {
|
|
175
|
+
const resolver = createRequire(join(resolverDirectory, 'strategy-capabilities-resolver.cjs'));
|
|
176
|
+
let entry;
|
|
177
|
+
try {
|
|
178
|
+
entry = resolver.resolve(packageName);
|
|
179
|
+
}
|
|
180
|
+
catch {
|
|
181
|
+
fail(`cannot resolve ${packageName}`);
|
|
182
|
+
}
|
|
183
|
+
return { packageName, files: allLibFiles(packageRoot(entry, packageName)) };
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Fixed benchmark capability construction. It hashes each selected installed
|
|
187
|
+
* package's manifest and every deployed JavaScript file under lib, so a lockfile range or
|
|
188
|
+
* a caller-provided digest cannot stand in for the code actually resolved.
|
|
189
|
+
*/
|
|
190
|
+
export function createFixedStrategyCapabilityExpectation(input) {
|
|
191
|
+
const resolverDirectory = resolve(checkedText(input.resolverDirectory, 'resolver directory'));
|
|
192
|
+
const goals = fixedModule(resolverDirectory, '@dsh-enhanced/assistant-goals');
|
|
193
|
+
const isolation = fixedModule(resolverDirectory, '@dsh-enhanced/assistant-isolation');
|
|
194
|
+
const policyModule = fixedModule(resolverDirectory, '@dsh-enhanced/assistant-policy');
|
|
195
|
+
const runtime = fixedModule(resolverDirectory, '@dsh-enhanced/assistant-evaluation');
|
|
196
|
+
const loaded = ['@deepseek-ai/cordis', '@deepseek-ai/dsh-agent', '@deepseek-ai/dsh-agent-loop', '@deepseek-ai/dsh-llm', '@deepseek-ai/dsh-session', '@deepseek-ai/dsh-session-projection', '@deepseek-ai/dsh-session-persistence', '@deepseek-ai/dsh-system-prompt', '@deepseek-ai/dsh-tools', '@deepseek-ai/dsh-goal', '@deepseek-ai/dsh-tool-goal', '@deepseek-ai/dsh-goal-round-driver', '@deepseek-ai/dsh-subagent', '@deepseek-ai/dsh-session-persistence-jsonl', '@dsh-enhanced/assistant-delivery', '@dsh-enhanced/assistant-verifier'].map(name => fixedModule(resolverDirectory, name));
|
|
197
|
+
if (input.modelProvider === 'deepseek-goal-metered') {
|
|
198
|
+
loaded.push(fixedModule(resolverDirectory, '@dsh-enhanced/assistant-deepseek-budget'));
|
|
199
|
+
loaded.push(fixedModule(resolverDirectory, '@deepseek-ai/dsh-credentials'));
|
|
200
|
+
}
|
|
201
|
+
const sources = { common: { tools: [goals, isolation], policy: [policyModule], runtime: [runtime, ...loaded] },
|
|
202
|
+
strategy: { guide: [goals], tool: [goals], policy: [policyModule], runtime: [goals, ...loaded] } };
|
|
203
|
+
// These names are the benchmark recipe's fixed contract. Runtime records
|
|
204
|
+
// the actual rich definitions emitted by DSH; it never accepts caller hashes.
|
|
205
|
+
const commonPolicy = { request: { action: 'execute', resource: { kind: 'tool', id: 'goal_create' } }, effect: 'allow', config: strategyPolicyRecipe(false) };
|
|
206
|
+
const strategyPolicy = { request: { action: 'execute', resource: { kind: 'tool', id: 'goal_strategy' } }, effect: 'allow', config: strategyPolicyRecipe(true) };
|
|
207
|
+
return createStrategyCapabilityExpectation({ resolverDirectory, persona: input.persona, ...(input.recipe === undefined ? {} : { recipe: input.recipe }), sources,
|
|
208
|
+
commonTools: fixedToolDefinitions, strategyTools: fixedStrategyToolDefinitions, commonPolicy, strategyPolicy });
|
|
209
|
+
}
|
|
210
|
+
const runtimeBrand = new WeakSet();
|
|
211
|
+
export class StrategyCapabilityRuntimeHost {
|
|
212
|
+
#ctx;
|
|
213
|
+
#parentRequest;
|
|
214
|
+
#policyRequests;
|
|
215
|
+
constructor(ctx) { this.#ctx = ctx; }
|
|
216
|
+
static fromMountedContext(ctx) {
|
|
217
|
+
if (!ctx || typeof ctx !== 'object' || typeof ctx.get !== 'function')
|
|
218
|
+
fail('trusted runtime requires a Context');
|
|
219
|
+
const host = new StrategyCapabilityRuntimeHost(ctx);
|
|
220
|
+
runtimeBrand.add(host);
|
|
221
|
+
return host;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Call this from the real `agent/request` hook with the definitions emitted
|
|
225
|
+
* to the parent model request. It is intentionally a one-shot capture: a
|
|
226
|
+
* later synthetic array cannot replace the observed request.
|
|
227
|
+
*/
|
|
228
|
+
captureParentRequest(request) {
|
|
229
|
+
if (!runtimeBrand.has(this))
|
|
230
|
+
fail('untrusted runtime host');
|
|
231
|
+
if (this.#parentRequest !== undefined)
|
|
232
|
+
fail('parent request already captured');
|
|
233
|
+
this.#parentRequest = Object.freeze({ system: checkedText(request.system, 'parent request system'), tools: definitions(request.tools) });
|
|
234
|
+
}
|
|
235
|
+
/** Capture the real owner-bound requests used for the public Policy API. */
|
|
236
|
+
capturePolicyRequests(requests) {
|
|
237
|
+
if (!runtimeBrand.has(this) || this.#policyRequests !== undefined || !isPlainObject(requests))
|
|
238
|
+
fail('invalid policy request capture');
|
|
239
|
+
this.#policyRequests = { common: clone(requests.common), strategy: clone(requests.strategy) };
|
|
240
|
+
}
|
|
241
|
+
observe(expectation, enabled, parentScope) {
|
|
242
|
+
if (!runtimeBrand.has(this))
|
|
243
|
+
fail('untrusted runtime host');
|
|
244
|
+
const goals = this.#ctx.get('assistantGoals');
|
|
245
|
+
const assistantPolicy = this.#ctx.get('assistantPolicy');
|
|
246
|
+
const tools = this.#ctx.tools ?? this.#ctx.get('tools');
|
|
247
|
+
if (!goals || !assistantPolicy || !tools || typeof tools.get !== 'function')
|
|
248
|
+
fail('required runtime services are not mounted');
|
|
249
|
+
if (this.#parentRequest === undefined)
|
|
250
|
+
fail('actual parent request was not captured');
|
|
251
|
+
if (this.#parentRequest.system !== expectation.persona)
|
|
252
|
+
fail('actual parent request persona drift');
|
|
253
|
+
const expectedNames = [...expectation.commonTools, ...(enabled ? expectation.strategyTools : [])].map(item => item.name).sort();
|
|
254
|
+
const observedNames = this.#parentRequest.tools.map(item => item.name).sort();
|
|
255
|
+
if (canonical(observedNames) !== canonical(expectedNames))
|
|
256
|
+
fail('actual parent request tool drift');
|
|
257
|
+
const actual = [...expectation.commonTools, ...(enabled ? expectation.strategyTools : [])].map(expected => {
|
|
258
|
+
const value = tools.get(expected.name, parentScope);
|
|
259
|
+
if (!value)
|
|
260
|
+
fail(`parent tool missing: ${expected.name}`);
|
|
261
|
+
const observed = definition({ name: String(value.name), description: String(value.description), parameters: value.parameters,
|
|
262
|
+
output: value.output?.schema ?? null });
|
|
263
|
+
if (observed.name !== expected.name)
|
|
264
|
+
fail(`parent tool definition drift: ${expected.name}`);
|
|
265
|
+
const captured = this.#parentRequest.tools.find(item => item.name === expected.name);
|
|
266
|
+
if (captured.description !== observed.description || canonical(captured.parameters) !== canonical(observed.parameters))
|
|
267
|
+
fail(`parent tool definition drift: ${expected.name}`);
|
|
268
|
+
return observed;
|
|
269
|
+
});
|
|
270
|
+
const evaluate = assistantPolicy.evaluate;
|
|
271
|
+
if (typeof evaluate !== 'function')
|
|
272
|
+
fail('assistantPolicy evaluate is unavailable');
|
|
273
|
+
const inspect = assistantPolicy.inspectHostConfiguration;
|
|
274
|
+
if (typeof inspect !== 'function')
|
|
275
|
+
fail('assistantPolicy configuration inspection is unavailable');
|
|
276
|
+
if (this.#policyRequests === undefined)
|
|
277
|
+
fail('actual policy requests were not captured');
|
|
278
|
+
const common = evaluate.call(assistantPolicy, this.#policyRequests.common)?.effect;
|
|
279
|
+
const strategy = evaluate.call(assistantPolicy, this.#policyRequests.strategy)?.effect;
|
|
280
|
+
if (common !== 'allow' || strategy !== (enabled ? 'allow' : 'deny'))
|
|
281
|
+
fail('policy decision drift');
|
|
282
|
+
const commonRequest = this.#policyRequests.common;
|
|
283
|
+
const workspace = commonRequest.subject?.workspace;
|
|
284
|
+
if (typeof workspace !== 'string' || workspace.length === 0)
|
|
285
|
+
fail('policy request workspace is unavailable');
|
|
286
|
+
const configuration = assertStrategyPolicyConfiguration(inspect.call(assistantPolicy), enabled, workspace);
|
|
287
|
+
const actualSources = Object.fromEntries(Object.entries(expectation.resolvedSourceModules).map(([key, modules]) => [key, identities(expectation.resolverDirectory, modules)]));
|
|
288
|
+
if (canonical(actualSources) !== canonical(expectation.sourceIdentity))
|
|
289
|
+
fail('resolved source identity drift');
|
|
290
|
+
const sourceDigest = acceptanceDigest(actualSources);
|
|
291
|
+
return Object.freeze({ sourceDigest, services: Object.freeze({ goals: true, assistantPolicy: true, tools: true }), parentTools: Object.freeze(actual),
|
|
292
|
+
policy: Object.freeze({ common: common, strategy: strategy, configuration, configDigest: acceptanceDigest(configuration) }), personaDigest: acceptanceDigest({ persona: this.#parentRequest.system }), dynamicGoalContext: 'not-attested' });
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
export function assertStrategyCapabilityRuntime(expectation, host, enabled, parentScope) {
|
|
296
|
+
if (!(host instanceof StrategyCapabilityRuntimeHost))
|
|
297
|
+
fail('runtime observation requires StrategyCapabilityRuntimeHost');
|
|
298
|
+
return host.observe(expectation, enabled, parentScope);
|
|
299
|
+
}
|
|
300
|
+
//# sourceMappingURL=strategy-capabilities.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-capabilities.js","sourceRoot":"","sources":["../../src/benchmark/strategy-capabilities.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AACjE,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAClG,OAAO,EAAE,iCAAiC,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAE9F,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC,MAAM,CAAC;IAC1D,QAAQ,EAAE,oCAAoC;IAC9C,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC;IAC/F,YAAY,EAAE,eAAe;IAC7B,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClE,uEAAuE;IACvE,eAAe,EAAE,IAAI;CACtB,CAAC,CAAA;AA2EF,MAAM,MAAM,GAAG,CAAC,KAAsB,EAAU,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACnG,MAAM,aAAa,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,SAAS,CAAA;AACrM,MAAM,SAAS,GAAG,CAAC,KAAc,EAAU,EAAE,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAA;AAC5E,MAAM,KAAK,GAAG,CAAI,KAAQ,EAAK,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAM,CAAA;AACxE,SAAS,IAAI,CAAC,OAAe,IAAW,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,EAAE,CAAC,CAAA,CAAC,CAAC;AAE9F,SAAS,WAAW,CAAC,KAAc,EAAE,KAAa;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,GAAG,GAAG,IAAI;QAAE,IAAI,CAAC,WAAW,KAAK,EAAE,CAAC,CAAA;IAC/H,OAAO,KAAK,CAAA;AACd,CAAC;AACD,SAAS,aAAa,CAAC,KAAqC;IAC1D,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;WACxG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,CAAC,uBAAuB,CAAC,CAAA;IAC3F,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACnC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC;YAAE,IAAI,CAAC,qBAAqB,CAAC,CAAA;QACnL,OAAO,IAAI,CAAA;IACb,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACT,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM;QAAE,IAAI,CAAC,uBAAuB,CAAC,CAAA;IACvE,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,CAAA;AAClD,CAAC;AACD,SAAS,WAAW,CAAC,KAAa,EAAE,WAAmB;IACrD,IAAI,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAC1C,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;QAC9C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAuB,CAAA;YAC7E,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;gBAAE,OAAO,OAAO,CAAA;QAC/C,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACV,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAC/B,IAAI,MAAM,KAAK,OAAO;YAAE,IAAI,CAAC,qCAAqC,WAAW,EAAE,CAAC,CAAA;QAChF,OAAO,GAAG,MAAM,CAAA;IAClB,CAAC;AACH,CAAC;AACD,MAAM,UAAU,+BAA+B,CAAC,KAA4E;IAC1H,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,CAAC,CAAA;IACrF,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,oCAAoC,CAAC,CAAC,CAAA;IACrF,IAAI,KAAa,CAAA;IACjB,IAAI,CAAC;QAAC,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAAC,CAAC;IAC3G,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAA;IACjE,IAAI,OAAgB,CAAA;IACpB,IAAI,CAAC;QAAC,OAAO,GAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA2B,CAAC,OAAO,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAAC,CAAC;IAC9I,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,EAAE;QAAE,IAAI,CAAC,+BAA+B,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;IAC5G,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACpC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;QAChD,IAAI,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,gCAAgC,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC,CAAA;QACtL,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAA;IACtE,CAAC,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAA;IACtG,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;AAC3E,CAAC;AACD,SAAS,UAAU,CAAC,SAAiB,EAAE,OAAkD;IACvF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,CAAC,oBAAoB,CAAC,CAAA;IAC/E,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,+BAA+B,CAAC,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;AACxH,CAAC;AACD,SAAS,UAAU,CAAC,KAAuC;IACzD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QAAE,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACnJ,8EAA8E;IAC9E,+EAA+E;IAC/E,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AAC9I,CAAC;AACD,SAAS,WAAW,CAAC,MAAmD;IACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,CAAC,6BAA6B,CAAC,CAAA;IACtF,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QAAE,IAAI,CAAC,2BAA2B,CAAC,CAAA;IACpG,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AAC9B,CAAC;AACD,SAAS,MAAM,CAAC,KAAoC;IAClD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;QAAE,IAAI,CAAC,sBAAsB,CAAC,CAAA;IAChH,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;AAC5G,CAAC;AACD,SAAS,YAAY,CAAC,KAAa,EAAE,MAAmD,EAAE,KAAc;IACtG,OAAO,gBAAgB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;AACpF,CAAC;AAED,SAAS,mCAAmC,CAAC,KAAyC;IACpF,MAAM,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC7F,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;IACrD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,8BAA8B,CAAC;QAAE,IAAI,CAAC,qDAAqD,CAAC,CAAA;IACpK,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAAC,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;IAC1G,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAAC,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IACpG,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC;QACnC,cAAc,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;QACzE,eAAe,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAC3E,gBAAgB,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7E,gBAAgB,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC7E,eAAe,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC3E,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC/E,kBAAkB,EAAE,UAAU,CAAC,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;KAClF,CAAC,CAAA;IACF,MAAM,YAAY,GAAG,gBAAgB,CAAC,8BAA8B,CAAC,CAAA;IACrE,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,cAAc,EAAE,cAAc,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;YAC7K,MAAM,EAAE,YAAY,CAAC,eAAe,EAAE,cAAc,CAAC,eAAe,CAAC,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,gBAAgB,EAAE,cAAc,CAAC,gBAAgB,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QACjM,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,gBAAgB,EAAE,cAAc,CAAC,gBAAgB,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,cAAc,CAAC,eAAe,CAAC,EAAE,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC;YACrO,MAAM,EAAE,YAAY,CAAC,iBAAiB,EAAE,cAAc,CAAC,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,kBAAkB,EAAE,cAAc,CAAC,kBAAkB,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;KAC5M,CAAC,CAAA;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,EAAE,YAAY,EAAE,iBAAiB;QACrJ,qBAAqB,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO;YAC7K,gBAAgB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;AAC9M,CAAC;AAED,6EAA6E;AAC7E,6EAA6E;AAC7E,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,8BAA8B,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC/F,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC5E,MAAM,4BAA4B,GAAG,MAAM,CAAC,MAAM,CAAC;IACjD,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC1F,CAAC,CAAA;AACF,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAAC,MAAM,KAAK,GAAa,EAAE,CAAA;IACzD,MAAM,KAAK,GAAG,CAAC,SAAiB,EAAQ,EAAE;QACxC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YACxC,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAA;iBAC/B,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;QACzF,CAAC;IACH,CAAC,CAAA;IACD,KAAK,CAAC,GAAG,CAAC,CAAA;IACV,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,CAAC,iDAAiD,CAAC,CAAA;IAC/E,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;AACrB,CAAC;AACD,SAAS,WAAW,CAAC,iBAAyB,EAAE,WAAmB;IACjE,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,iBAAiB,EAAE,oCAAoC,CAAC,CAAC,CAAA;IAC7F,IAAI,KAAa,CAAA;IACjB,IAAI,CAAC;QAAC,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,IAAI,CAAC,kBAAkB,WAAW,EAAE,CAAC,CAAA;IAAC,CAAC;IAC7F,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,EAAE,CAAA;AAC7E,CAAC;AACD;;;;GAIG;AACH,MAAM,UAAU,wCAAwC,CAAC,KAA8C;IACrG,MAAM,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,iBAAiB,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC7F,MAAM,KAAK,GAAG,WAAW,CAAC,iBAAiB,EAAE,+BAA+B,CAAC,CAAA;IAC7E,MAAM,SAAS,GAAG,WAAW,CAAC,iBAAiB,EAAE,mCAAmC,CAAC,CAAA;IACrF,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,EAAE,gCAAgC,CAAC,CAAA;IACrF,MAAM,OAAO,GAAG,WAAW,CAAC,iBAAiB,EAAE,oCAAoC,CAAC,CAAA;IACpF,MAAM,MAAM,GAAG,CAAC,qBAAqB,EAAE,wBAAwB,EAAE,6BAA6B,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,qCAAqC,EAAE,sCAAsC,EAAE,gCAAgC,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,4BAA4B,EAAE,oCAAoC,EAAE,2BAA2B,EAAE,4CAA4C,EAAE,kCAAkC,EAAE,kCAAkC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC,CAAA;IAChkB,IAAI,KAAK,CAAC,aAAa,KAAK,uBAAuB,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,yCAAyC,CAAC,CAAC,CAAA;QACtF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,8BAA8B,CAAC,CAAC,CAAA;IAC7E,CAAC;IACD,MAAM,OAAO,GAAmC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,EAAE;QAC5I,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,MAAM,CAAC,EAAE,EAAE,CAAA;IACpG,0EAA0E;IAC1E,8EAA8E;IAC9E,MAAM,YAAY,GAAG,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,MAAM,EAAE,OAAgB,EAAE,MAAM,EAAE,oBAAoB,CAAC,KAAK,CAAC,EAAE,CAAA;IACrK,MAAM,cAAc,GAAG,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,OAAgB,EAAE,MAAM,EAAE,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAA;IACxK,OAAO,mCAAmC,CAAC,EAAE,iBAAiB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO;QAC9J,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,4BAA4B,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAA;AACnH,CAAC;AAsBD,MAAM,YAAY,GAAG,IAAI,OAAO,EAAU,CAAA;AAC1C,MAAM,OAAO,6BAA6B;IAC/B,IAAI,CAA2B;IACxC,cAAc,CAAoF;IAClG,eAAe,CAAoD;IACnE,YAAoB,GAA8B,IAAI,IAAI,CAAC,IAAI,GAAG,GAAG,CAAA,CAAC,CAAC;IACvE,MAAM,CAAC,kBAAkB,CAAC,GAA8B;QACtD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,UAAU;YAAE,IAAI,CAAC,oCAAoC,CAAC,CAAA;QAChH,MAAM,IAAI,GAAG,IAAI,6BAA6B,CAAC,GAAG,CAAC,CAAC;QAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAAC,OAAO,IAAI,CAAA;IAC1F,CAAC;IACD;;;;OAIG;IACH,oBAAoB,CAAC,OAAwC;QAC3D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAC3D,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS;YAAE,IAAI,CAAC,iCAAiC,CAAC,CAAA;QAC9E,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,uBAAuB,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC1I,CAAC;IACD,4EAA4E;IAC5E,qBAAqB,CAAC,QAAgD;QACpE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC;YAAE,IAAI,CAAC,gCAAgC,CAAC,CAAA;QACrI,IAAI,CAAC,eAAe,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAA;IAC/F,CAAC;IACD,OAAO,CAAC,WAA0C,EAAE,OAAgB,EAAE,WAAqB;QACzF,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAAC,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAAC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAuC,CAAA;QACtM,IAAI,CAAC,KAAK,IAAI,CAAC,eAAe,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,UAAU;YAAE,IAAI,CAAC,2CAA2C,CAAC,CAAA;QAC9H,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS;YAAE,IAAI,CAAC,wCAAwC,CAAC,CAAA;QACrF,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,WAAW,CAAC,OAAO;YAAE,IAAI,CAAC,qCAAqC,CAAC,CAAA;QACnG,MAAM,aAAa,GAAG,CAAC,GAAG,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAA;QAC/H,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7E,IAAI,SAAS,CAAC,aAAa,CAAC,KAAK,SAAS,CAAC,aAAa,CAAC;YAAE,IAAI,CAAC,kCAAkC,CAAC,CAAA;QACnG,MAAM,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YACxG,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAA0D,CAAA;YAC5G,IAAI,CAAC,KAAK;gBAAE,IAAI,CAAC,wBAAwB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;YACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC1H,MAAM,EAAG,KAAK,CAAC,MAA2C,EAAE,MAAM,IAAI,IAAI,EAAE,CAAC,CAAA;YAC/E,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;gBAAE,IAAI,CAAC,iCAAiC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;YAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAe,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAE,CAAA;YACtF,IAAI,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,iCAAiC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9K,OAAO,QAAQ,CAAA;QACjB,CAAC,CAAC,CAAA;QACF,MAAM,QAAQ,GAAI,eAA6E,CAAC,QAAQ,CAAA;QACxG,IAAI,OAAO,QAAQ,KAAK,UAAU;YAAE,IAAI,CAAC,yCAAyC,CAAC,CAAA;QACnF,MAAM,OAAO,GAAI,eAAgE,CAAC,wBAAwB,CAAA;QAC1G,IAAI,OAAO,OAAO,KAAK,UAAU;YAAE,IAAI,CAAC,yDAAyD,CAAC,CAAA;QAClG,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS;YAAE,IAAI,CAAC,0CAA0C,CAAC,CAAA;QACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;QAClF,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;QACtF,IAAI,MAAM,KAAK,OAAO,IAAI,QAAQ,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,IAAI,CAAC,uBAAuB,CAAC,CAAA;QAClG,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,MAA+C,CAAA;QAC1F,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,EAAE,SAAS,CAAA;QAClD,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,IAAI,CAAC,yCAAyC,CAAC,CAAA;QAC5G,MAAM,aAAa,GAAG,iCAAiC,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,CAAA;QAC1G,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,WAAW,CAAC,iBAAiB,EAAE,OAAoD,CAAC,CAAC,CAAC,CAAC,CAAA;QAC3N,IAAI,SAAS,CAAC,aAAa,CAAC,KAAK,SAAS,CAAC,WAAW,CAAC,cAAc,CAAC;YAAE,IAAI,CAAC,gCAAgC,CAAC,CAAA;QAC9G,MAAM,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,CAAA;QACpD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;YACnJ,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAiB,EAAE,QAAQ,EAAE,QAA4B,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,aAAa,EAAE,gBAAgB,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,cAAc,EAAE,CAAC,CAAA;IAC/Q,CAAC;CACF;AACD,MAAM,UAAU,+BAA+B,CAAC,WAA0C,EAAE,IAAmC,EAAE,OAAgB,EAAE,WAAqB;IACtK,IAAI,CAAC,CAAC,IAAI,YAAY,6BAA6B,CAAC;QAAE,IAAI,CAAC,4DAA4D,CAAC,CAAA;IACxH,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,EAAE,WAAW,CAAC,CAAA;AACxD,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** Strict, side-effect-free operator configuration for strategy-v1. */
|
|
2
|
+
import type { NativeModelConfig } from './native.js';
|
|
3
|
+
import type { BenchmarkBudget } from './types.js';
|
|
4
|
+
export interface StrategyBenchmarkConfig {
|
|
5
|
+
suite: 'strategy-v1';
|
|
6
|
+
id: string;
|
|
7
|
+
cases: readonly string[];
|
|
8
|
+
persona: string;
|
|
9
|
+
model: NativeModelConfig;
|
|
10
|
+
budget: BenchmarkBudget;
|
|
11
|
+
execution: {
|
|
12
|
+
modelCalls: number;
|
|
13
|
+
maxOutputTokensPerCall: number;
|
|
14
|
+
maxGoalRounds: number;
|
|
15
|
+
};
|
|
16
|
+
repeats: number;
|
|
17
|
+
seed: number;
|
|
18
|
+
image: string;
|
|
19
|
+
dockerPath: string;
|
|
20
|
+
stepMaxDurationMs: number;
|
|
21
|
+
stopTimeoutMs?: number;
|
|
22
|
+
workspaceDirectory: string;
|
|
23
|
+
stateDirectory: string;
|
|
24
|
+
}
|
|
25
|
+
export declare function parseStrategyBenchmarkConfig(value: unknown): Readonly<StrategyBenchmarkConfig>;
|
|
26
|
+
//# sourceMappingURL=strategy-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-config.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-config.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAIpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjD,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,aAAa,CAAA;IACpB,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,iBAAiB,CAAA;IACxB,MAAM,EAAE,eAAe,CAAA;IACvB,SAAS,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,sBAAsB,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAA;IACxF,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB,EAAE,MAAM,CAAA;IACzB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;CACvB;AAaD,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,OAAO,GAAG,QAAQ,CAAC,uBAAuB,CAAC,CA+C9F"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { isAbsolute, relative } from 'node:path';
|
|
2
|
+
import { strategyDevelopmentCases, strategyDevelopmentTask } from './strategy-corpus.js';
|
|
3
|
+
import { benchmarkAssert, benchmarkInteger, benchmarkObject, benchmarkSnapshot } from './schema.js';
|
|
4
|
+
const MAX_RATE = 1_000_000_000;
|
|
5
|
+
const routeId = (value, field) => benchmarkAssert(typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/u.test(value), `invalid ${field}`);
|
|
6
|
+
const identifier = (value, field) => benchmarkAssert(typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(value), `invalid ${field}`);
|
|
7
|
+
const digest = (value, field) => benchmarkAssert(typeof value === 'string' && /^[a-f0-9]{64}$/u.test(value), `invalid ${field}`);
|
|
8
|
+
const rate = (value) => benchmarkInteger(value, 0, MAX_RATE);
|
|
9
|
+
const absolute = (value, field) => benchmarkAssert(typeof value === 'string' && isAbsolute(value) && value.length <= 4096 && !value.includes('\u0000'), `invalid ${field}`);
|
|
10
|
+
const contains = (ancestor, descendant) => {
|
|
11
|
+
const path = relative(ancestor, descendant);
|
|
12
|
+
return path === '' || (path !== '..' && !path.startsWith(`..${process.platform === 'win32' ? '\\' : '/'}`) && !isAbsolute(path));
|
|
13
|
+
};
|
|
14
|
+
export function parseStrategyBenchmarkConfig(value) {
|
|
15
|
+
const copy = benchmarkSnapshot(value);
|
|
16
|
+
const raw = benchmarkObject(copy, ['suite', 'id', 'cases', 'persona', 'model', 'budget', 'execution', 'repeats', 'seed', 'image', 'dockerPath', 'stepMaxDurationMs', 'workspaceDirectory', 'stateDirectory',
|
|
17
|
+
...((copy !== null && typeof copy === 'object' && Object.hasOwn(copy, 'stopTimeoutMs')) ? ['stopTimeoutMs'] : [])]);
|
|
18
|
+
benchmarkAssert(raw.suite === 'strategy-v1', 'unsupported strategy suite');
|
|
19
|
+
identifier(raw.id, 'strategy benchmark id');
|
|
20
|
+
benchmarkAssert(typeof raw.persona === 'string' && raw.persona.trim().length > 0 && raw.persona.length <= 32_768, 'invalid strategy persona');
|
|
21
|
+
benchmarkAssert(Array.isArray(raw.cases) && raw.cases.length >= 1 && raw.cases.length <= 100, 'invalid strategy cases');
|
|
22
|
+
const known = new Set(strategyDevelopmentCases().map(task => task.id));
|
|
23
|
+
const selected = new Set();
|
|
24
|
+
for (const caseId of raw.cases) {
|
|
25
|
+
identifier(caseId, 'strategy case id');
|
|
26
|
+
benchmarkAssert(known.has(caseId) && !selected.has(caseId), 'cases must be unique strategy development cases');
|
|
27
|
+
selected.add(caseId);
|
|
28
|
+
}
|
|
29
|
+
const budget = benchmarkObject(raw.budget, ['durationMs', 'inputTokens', 'outputTokens', 'costUsdMicros', 'toolCalls']);
|
|
30
|
+
benchmarkInteger(budget.durationMs, 1, 86_400_000);
|
|
31
|
+
for (const field of ['inputTokens', 'outputTokens', 'toolCalls'])
|
|
32
|
+
benchmarkInteger(budget[field], 0, 1_000_000_000);
|
|
33
|
+
if (budget.costUsdMicros !== null)
|
|
34
|
+
benchmarkInteger(budget.costUsdMicros, 0, MAX_RATE);
|
|
35
|
+
const model = benchmarkObject(raw.model, ['provider', 'model', 'temperature', 'maxOutputTokens', 'inputUsdMicrosPerMillionTokens', 'outputUsdMicrosPerMillionTokens', 'adapterDigest', 'tokenCounterDigest',
|
|
36
|
+
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'inputLimitMode')) ? ['inputLimitMode'] : []),
|
|
37
|
+
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'outputLimitMode')) ? ['outputLimitMode'] : []),
|
|
38
|
+
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'cacheReadUsdMicrosPerMillionTokens')) ? ['cacheReadUsdMicrosPerMillionTokens'] : []),
|
|
39
|
+
...((raw.model !== null && typeof raw.model === 'object' && Object.hasOwn(raw.model, 'cacheWriteUsdMicrosPerMillionTokens')) ? ['cacheWriteUsdMicrosPerMillionTokens'] : []),
|
|
40
|
+
]);
|
|
41
|
+
routeId(model.provider, 'provider');
|
|
42
|
+
routeId(model.model, 'model');
|
|
43
|
+
benchmarkAssert(model.temperature === null || typeof model.temperature === 'number' && Number.isFinite(model.temperature) && model.temperature >= 0 && model.temperature <= 2, 'invalid temperature');
|
|
44
|
+
benchmarkAssert((model.inputLimitMode ?? 'upper-bound') === 'upper-bound' && (model.outputLimitMode ?? 'provider') === 'provider', 'strategy requires upper-bound input and provider output limits');
|
|
45
|
+
benchmarkInteger(model.maxOutputTokens, 1, budget.outputTokens);
|
|
46
|
+
benchmarkAssert((model.inputUsdMicrosPerMillionTokens === null) === (model.outputUsdMicrosPerMillionTokens === null), 'both token rates must be present or absent');
|
|
47
|
+
if (model.inputUsdMicrosPerMillionTokens !== null) {
|
|
48
|
+
rate(model.inputUsdMicrosPerMillionTokens);
|
|
49
|
+
rate(model.outputUsdMicrosPerMillionTokens);
|
|
50
|
+
}
|
|
51
|
+
for (const field of ['cacheReadUsdMicrosPerMillionTokens', 'cacheWriteUsdMicrosPerMillionTokens'])
|
|
52
|
+
if (model[field] !== undefined && model[field] !== null)
|
|
53
|
+
rate(model[field]);
|
|
54
|
+
digest(model.adapterDigest, 'adapter digest');
|
|
55
|
+
digest(model.tokenCounterDigest, 'token counter digest');
|
|
56
|
+
if (budget.costUsdMicros !== null)
|
|
57
|
+
benchmarkAssert([model.inputUsdMicrosPerMillionTokens, model.outputUsdMicrosPerMillionTokens, model.cacheReadUsdMicrosPerMillionTokens, model.cacheWriteUsdMicrosPerMillionTokens].every(value => value !== null && value !== undefined), 'priced budget requires complete tariff');
|
|
58
|
+
const execution = benchmarkObject(raw.execution, ['modelCalls', 'maxOutputTokensPerCall', 'maxGoalRounds']);
|
|
59
|
+
benchmarkInteger(execution.modelCalls, 1, 10_000);
|
|
60
|
+
benchmarkInteger(execution.maxOutputTokensPerCall, 1, budget.outputTokens);
|
|
61
|
+
benchmarkInteger(execution.maxGoalRounds, 1, 32);
|
|
62
|
+
benchmarkAssert(model.maxOutputTokens === execution.maxOutputTokensPerCall, 'model output limit must equal strategy execution limit');
|
|
63
|
+
benchmarkInteger(raw.repeats, 2, 20);
|
|
64
|
+
benchmarkInteger(raw.seed, 0, 0xffffffff);
|
|
65
|
+
benchmarkAssert(typeof raw.image === 'string' && /^sha256:[a-f0-9]{64}$/u.test(raw.image), 'invalid immutable image');
|
|
66
|
+
absolute(raw.dockerPath, 'docker path');
|
|
67
|
+
absolute(raw.workspaceDirectory, 'workspace directory');
|
|
68
|
+
absolute(raw.stateDirectory, 'state directory');
|
|
69
|
+
benchmarkAssert(!contains(raw.workspaceDirectory, raw.stateDirectory) && !contains(raw.stateDirectory, raw.workspaceDirectory), 'strategy state and workspaces must be separate');
|
|
70
|
+
benchmarkInteger(raw.stepMaxDurationMs, 1_000, 300_000);
|
|
71
|
+
benchmarkAssert(raw.stepMaxDurationMs * 3 < budget.durationMs, 'strategy step duration does not fit benchmark duration');
|
|
72
|
+
for (const caseId of selected) {
|
|
73
|
+
const verification = strategyDevelopmentTask(caseId).verification;
|
|
74
|
+
benchmarkAssert(verification.maxDurationMs * verification.cases.length < raw.stepMaxDurationMs, 'strategy verification does not fit execution window');
|
|
75
|
+
}
|
|
76
|
+
if (raw.stopTimeoutMs !== undefined)
|
|
77
|
+
benchmarkInteger(raw.stopTimeoutMs, 1_000, 30_000);
|
|
78
|
+
return copy;
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=strategy-config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-config.js","sourceRoot":"","sources":["../../src/benchmark/strategy-config.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAChD,OAAO,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAA;AACxF,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAqBnG,MAAM,QAAQ,GAAG,aAAa,CAAA;AAC9B,MAAM,OAAO,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,uCAAuC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAC9K,MAAM,UAAU,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAC/K,MAAM,MAAM,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AACvJ,MAAM,IAAI,GAAG,CAAC,KAAc,EAAQ,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;AAC3E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE,CAAC,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAA;AAClM,MAAM,QAAQ,GAAG,CAAC,QAAgB,EAAE,UAAkB,EAAW,EAAE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC3C,OAAO,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;AAClI,CAAC,CAAA;AAED,MAAM,UAAU,4BAA4B,CAAC,KAAc;IACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,gBAAgB;QACzM,GAAG,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACrH,eAAe,CAAC,GAAG,CAAC,KAAK,KAAK,aAAa,EAAE,4BAA4B,CAAC,CAAA;IAC1E,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAA;IAC3C,eAAe,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,EAAE,0BAA0B,CAAC,CAAA;IAC7I,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,wBAAwB,CAAC,CAAA;IACvH,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,wBAAwB,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAC1G,KAAK,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAAC,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,iDAAiD,CAAC,CAAC;QAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAAC,CAAC;IAEhN,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,CAAC,CAA+B,CAAA;IACrJ,gBAAgB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IAClD,KAAK,MAAM,KAAK,IAAI,CAAC,aAAa,EAAE,cAAc,EAAE,WAAW,CAAU;QAAE,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;IAC5H,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI;QAAE,gBAAgB,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;IAEtF,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,gCAAgC,EAAE,iCAAiC,EAAE,eAAe,EAAE,oBAAoB;QACzM,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpI,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,oCAAoC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1K,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7K,CAAiC,CAAA;IAClC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAClE,eAAe,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,EAAE,qBAAqB,CAAC,CAAA;IACrM,eAAe,CAAC,CAAC,KAAK,CAAC,cAAc,IAAI,aAAa,CAAC,KAAK,aAAa,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,UAAU,CAAC,KAAK,UAAU,EAAE,gEAAgE,CAAC,CAAA;IACpM,gBAAgB,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;IAC/D,eAAe,CAAC,CAAC,KAAK,CAAC,8BAA8B,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,+BAA+B,KAAK,IAAI,CAAC,EAAE,4CAA4C,CAAC,CAAA;IACnK,IAAI,KAAK,CAAC,8BAA8B,KAAK,IAAI,EAAE,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAAC,CAAC;IAC9I,KAAK,MAAM,KAAK,IAAI,CAAC,oCAAoC,EAAE,qCAAqC,CAAU;QAAE,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IACvL,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,sBAAsB,CAAC,CAAA;IACvG,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI;QAAE,eAAe,CAAC,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,mCAAmC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,EAAE,wCAAwC,CAAC,CAAA;IAEtT,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,YAAY,EAAE,wBAAwB,EAAE,eAAe,CAAC,CAAyC,CAAA;IACnJ,gBAAgB,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;IAAC,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,EAAE,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;IAAC,gBAAgB,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;IAC/K,eAAe,CAAC,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,sBAAsB,EAAE,wDAAwD,CAAC,CAAA;IACrI,gBAAgB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IAC/E,eAAe,CAAC,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,yBAAyB,CAAC,CAAA;IACrH,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,kBAAkB,EAAE,qBAAqB,CAAC,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAA;IACjJ,eAAe,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,kBAA4B,EAAE,GAAG,CAAC,cAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAwB,EAAE,GAAG,CAAC,kBAA4B,CAAC,EAAE,gDAAgD,CAAC,CAAA;IACzN,gBAAgB,CAAC,GAAG,CAAC,iBAAiB,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IACvD,eAAe,CAAE,GAAG,CAAC,iBAA4B,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,wDAAwD,CAAC,CAAA;IACpI,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,YAAY,CAAA;QACjE,eAAe,CAAC,YAAY,CAAC,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,GAAI,GAAG,CAAC,iBAA4B,EAAE,qDAAqD,CAAC,CAAA;IACpK,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS;QAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACvF,OAAO,IAA+B,CAAA;AACxC,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type StrategyGoalTask } from './strategy-goal-runtime.js';
|
|
2
|
+
import type { BenchmarkCase, BenchmarkPlan } from './types.js';
|
|
3
|
+
export interface StrategyDevelopmentCorpusTask {
|
|
4
|
+
id: string;
|
|
5
|
+
domain: 'code';
|
|
6
|
+
objective: string;
|
|
7
|
+
publicPrompt: string;
|
|
8
|
+
artifactPath: string;
|
|
9
|
+
examples: readonly {
|
|
10
|
+
stdin: string;
|
|
11
|
+
expectedStdout: string;
|
|
12
|
+
}[];
|
|
13
|
+
}
|
|
14
|
+
/** This public surface intentionally excludes independent verification vectors. */
|
|
15
|
+
export declare const strategyDevelopmentCorpus: readonly StrategyDevelopmentCorpusTask[];
|
|
16
|
+
/** Dataset identity commits to the complete private verification manifest. */
|
|
17
|
+
export declare const strategyDevelopmentDataset: BenchmarkPlan['dataset'];
|
|
18
|
+
export declare function strategyDevelopmentCases(): readonly BenchmarkCase[];
|
|
19
|
+
/** Host-only full task material, including private verifier vectors. */
|
|
20
|
+
export declare function strategyDevelopmentTask(caseId: string): Readonly<StrategyGoalTask>;
|
|
21
|
+
//# sourceMappingURL=strategy-corpus.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-corpus.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,KAAK,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE9D,MAAM,WAAW,6BAA6B;IAC5C,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,SAAS;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAC/D;AA6ED,mFAAmF;AACnF,eAAO,MAAM,yBAAyB,EAAE,SAAS,6BAA6B,EAAiF,CAAA;AAO/J,8EAA8E;AAC9E,eAAO,MAAM,0BAA0B,EAAE,aAAa,CAAC,SAAS,CAG9D,CAAA;AAEF,wBAAgB,wBAAwB,IAAI,SAAS,aAAa,EAAE,CAEnE;AAED,wEAAwE;AACxE,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAIlF"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/** Public synthetic development tasks for the native Goal strategy benchmark. */
|
|
2
|
+
import { acceptanceCanonicalJson, acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
|
|
3
|
+
import { strategyGoalTaskDigests } from './strategy-goal-runtime.js';
|
|
4
|
+
const objective = (description) => `Create a POSIX shell program named answer.sh that ${description}`;
|
|
5
|
+
const prompt = (value, examples) => [
|
|
6
|
+
value,
|
|
7
|
+
'Public examples:\n' + examples.map(example => `stdin:\n${example.stdin}\nstdout:\n${example.expectedStdout}`).join('\n'),
|
|
8
|
+
'Create a Goal by calling goal_create with the exact objective stated above.',
|
|
9
|
+
'Use isolation_run with grant_id "benchmark-work" to create the fixed artifact answer.sh.',
|
|
10
|
+
'Do not return the solution in chat; the artifact is evaluated independently.',
|
|
11
|
+
].join('\n\n');
|
|
12
|
+
const authored = [
|
|
13
|
+
{
|
|
14
|
+
id: 'integer-sum', domain: 'code',
|
|
15
|
+
objective: objective('reads signed decimal integers separated by whitespace from standard input and writes their sum followed by one newline.'),
|
|
16
|
+
examples: [{ stdin: '4 -1\n', expectedStdout: '3\n' }],
|
|
17
|
+
publicPrompt: prompt(objective('reads signed decimal integers separated by whitespace from standard input and writes their sum followed by one newline.'), [{ stdin: '4 -1\n', expectedStdout: '3\n' }]),
|
|
18
|
+
artifactPath: 'answer.sh',
|
|
19
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 1_024, cases: [
|
|
20
|
+
{ stdin: '1 2 3\n', expectedStdout: '6\n', expectedExitCode: 0 },
|
|
21
|
+
{ stdin: '-5\n10\n-3\n', expectedStdout: '2\n', expectedExitCode: 0 },
|
|
22
|
+
{ stdin: '', expectedStdout: '0\n', expectedExitCode: 0 },
|
|
23
|
+
] },
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: 'merge-touching-intervals', domain: 'code',
|
|
27
|
+
objective: objective('reads one inclusive integer interval "start end" per line, merges intervals that overlap or are adjacent (next start <= current end + 1), and writes merged intervals in ascending order as "start end" lines, each ending in one newline.'),
|
|
28
|
+
examples: [{ stdin: '1 2\n2 3\n', expectedStdout: '1 3\n' }],
|
|
29
|
+
publicPrompt: prompt(objective('reads one inclusive integer interval "start end" per line, merges intervals that overlap or are adjacent (next start <= current end + 1), and writes merged intervals in ascending order as "start end" lines, each ending in one newline.'), [{ stdin: '1 2\n2 3\n', expectedStdout: '1 3\n' }]),
|
|
30
|
+
artifactPath: 'answer.sh',
|
|
31
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
32
|
+
{ stdin: '5 7\n1 3\n3 5\n10 12\n', expectedStdout: '1 7\n10 12\n', expectedExitCode: 0 },
|
|
33
|
+
{ stdin: '1 1\n2 2\n4 4\n', expectedStdout: '1 2\n4 4\n', expectedExitCode: 0 },
|
|
34
|
+
{ stdin: '', expectedStdout: '', expectedExitCode: 0 },
|
|
35
|
+
] },
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
id: 'word-frequency-lexical-ties', domain: 'code',
|
|
39
|
+
objective: objective('reads whitespace-separated words, counts exact byte-for-byte word frequency, then writes one "word count" line per distinct word ordered by descending count and ascending lexical order for ties, each ending in one newline.'),
|
|
40
|
+
examples: [{ stdin: 'red blue red\n', expectedStdout: 'red 2\nblue 1\n' }],
|
|
41
|
+
publicPrompt: prompt(objective('reads whitespace-separated words, counts exact byte-for-byte word frequency, then writes one "word count" line per distinct word ordered by descending count and ascending lexical order for ties, each ending in one newline.'), [{ stdin: 'red blue red\n', expectedStdout: 'red 2\nblue 1\n' }]),
|
|
42
|
+
artifactPath: 'answer.sh',
|
|
43
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
44
|
+
{ stdin: 'pear apple pear banana apple pear\n', expectedStdout: 'pear 3\napple 2\nbanana 1\n', expectedExitCode: 0 },
|
|
45
|
+
{ stdin: 'z a z a\n', expectedStdout: 'a 2\nz 2\n', expectedExitCode: 0 },
|
|
46
|
+
{ stdin: '', expectedStdout: '', expectedExitCode: 0 },
|
|
47
|
+
] },
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
id: 'dependency-topological-order', domain: 'code',
|
|
51
|
+
objective: objective('reads directed dependency edges "before after" per line and writes a lexically smallest topological ordering as one node per line; when no ordering exists, writes exactly "CYCLE" followed by one newline.'),
|
|
52
|
+
examples: [{ stdin: 'parse build\n', expectedStdout: 'parse\nbuild\n' }],
|
|
53
|
+
publicPrompt: prompt(objective('reads directed dependency edges "before after" per line and writes a lexically smallest topological ordering as one node per line; when no ordering exists, writes exactly "CYCLE" followed by one newline.'), [{ stdin: 'parse build\n', expectedStdout: 'parse\nbuild\n' }]),
|
|
54
|
+
artifactPath: 'answer.sh',
|
|
55
|
+
verification: { command: 'sh artifact < input', maxDurationMs: 5_000, maxOutputBytes: 2_048, cases: [
|
|
56
|
+
{ stdin: 'cook eat\nshop cook\n', expectedStdout: 'shop\ncook\neat\n', expectedExitCode: 0 },
|
|
57
|
+
{ stdin: 'b d\na d\n', expectedStdout: 'a\nb\nd\n', expectedExitCode: 0 },
|
|
58
|
+
{ stdin: 'a b\nb c\nc a\n', expectedStdout: 'CYCLE\n', expectedExitCode: 0 },
|
|
59
|
+
] },
|
|
60
|
+
},
|
|
61
|
+
];
|
|
62
|
+
function frozenJson(value) {
|
|
63
|
+
const copy = JSON.parse(acceptanceCanonicalJson(value));
|
|
64
|
+
const freeze = (entry) => {
|
|
65
|
+
if (entry === null || typeof entry !== 'object')
|
|
66
|
+
return;
|
|
67
|
+
for (const child of Object.values(entry))
|
|
68
|
+
freeze(child);
|
|
69
|
+
Object.freeze(entry);
|
|
70
|
+
};
|
|
71
|
+
freeze(copy);
|
|
72
|
+
return copy;
|
|
73
|
+
}
|
|
74
|
+
/** This public surface intentionally excludes independent verification vectors. */
|
|
75
|
+
export const strategyDevelopmentCorpus = frozenJson(authored.map(({ verification: _verification, ...task }) => task));
|
|
76
|
+
const fullTask = (task) => ({
|
|
77
|
+
objective: task.objective, publicPrompt: task.publicPrompt, artifactPath: task.artifactPath, verification: task.verification,
|
|
78
|
+
});
|
|
79
|
+
const taskDigests = (task) => strategyGoalTaskDigests(fullTask(task));
|
|
80
|
+
/** Dataset identity commits to the complete private verification manifest. */
|
|
81
|
+
export const strategyDevelopmentDataset = frozenJson({
|
|
82
|
+
id: 'dsh-strategy-development', version: '1', split: 'development',
|
|
83
|
+
digest: acceptanceDigest(authored.map(task => ({ id: task.id, domain: task.domain, task: fullTask(task), digests: taskDigests(task) }))),
|
|
84
|
+
});
|
|
85
|
+
export function strategyDevelopmentCases() {
|
|
86
|
+
return frozenJson(authored.map(task => ({ id: task.id, domain: task.domain, ...taskDigests(task) })));
|
|
87
|
+
}
|
|
88
|
+
/** Host-only full task material, including private verifier vectors. */
|
|
89
|
+
export function strategyDevelopmentTask(caseId) {
|
|
90
|
+
const task = authored.find(candidate => candidate.id === caseId);
|
|
91
|
+
if (task === undefined)
|
|
92
|
+
throw new Error(`unknown strategy development case: ${caseId}`);
|
|
93
|
+
return frozenJson(fullTask(task));
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=strategy-corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-corpus.js","sourceRoot":"","sources":["../../src/benchmark/strategy-corpus.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAClG,OAAO,EAAE,uBAAuB,EAAyB,MAAM,4BAA4B,CAAA;AAgB3F,MAAM,SAAS,GAAG,CAAC,WAAmB,EAAU,EAAE,CAAC,qDAAqD,WAAW,EAAE,CAAA;AACrH,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,QAA8D,EAAU,EAAE,CAAC;IACxG,KAAK;IACL,oBAAoB,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,WAAW,OAAO,CAAC,KAAK,cAAc,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;IACzH,6EAA6E;IAC7E,0FAA0F;IAC1F,8EAA8E;CAC/E,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;AAEd,MAAM,QAAQ,GAAoC;IAChD;QACE,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM;QACjC,SAAS,EAAE,SAAS,CAAC,yHAAyH,CAAC;QAC/I,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACtD,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,yHAAyH,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC;QACxM,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAChE,EAAE,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACrE,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC1D,EAAE;KACJ;IACD;QACE,EAAE,EAAE,0BAA0B,EAAE,MAAM,EAAE,MAAM;QAC9C,SAAS,EAAE,SAAS,CAAC,4OAA4O,CAAC;QAClQ,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;QAC5D,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,4OAA4O,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QACjU,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,wBAAwB,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACxF,EAAE,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC/E,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACvD,EAAE;KACJ;IACD;QACE,EAAE,EAAE,6BAA6B,EAAE,MAAM,EAAE,MAAM;QACjD,SAAS,EAAE,SAAS,CAAC,gOAAgO,CAAC;QACtP,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC;QAC1E,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,gOAAgO,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnU,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,qCAAqC,EAAE,cAAc,EAAE,6BAA6B,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACpH,EAAE,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACzE,EAAE,KAAK,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACvD,EAAE;KACJ;IACD;QACE,EAAE,EAAE,8BAA8B,EAAE,MAAM,EAAE,MAAM;QAClD,SAAS,EAAE,SAAS,CAAC,6MAA6M,CAAC;QACnO,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC;QACxE,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,6MAA6M,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC9S,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,EAAE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;gBAClG,EAAE,KAAK,EAAE,uBAAuB,EAAE,cAAc,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBAC5F,EAAE,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC,EAAE;gBACzE,EAAE,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,EAAE;aAC7E,EAAE;KACJ;CACF,CAAA;AAED,SAAS,UAAU,CAAI,KAAQ;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAM,CAAA;IAC5D,MAAM,MAAM,GAAG,CAAC,KAAc,EAAQ,EAAE;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAM;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACvD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC,CAAA;IACD,MAAM,CAAC,IAAI,CAAC,CAAA;IACZ,OAAO,IAAI,CAAA;AACb,CAAC;AAED,mFAAmF;AACnF,MAAM,CAAC,MAAM,yBAAyB,GAA6C,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;AAE/J,MAAM,QAAQ,GAAG,CAAC,IAA0B,EAAoB,EAAE,CAAC,CAAC;IAClE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY;CAC7H,CAAC,CAAA;AACF,MAAM,WAAW,GAAG,CAAC,IAA0B,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AAE3F,8EAA8E;AAC9E,MAAM,CAAC,MAAM,0BAA0B,GAA6B,UAAU,CAAC;IAC7E,EAAE,EAAE,0BAA0B,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa;IAClE,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;CACzI,CAAC,CAAA;AAEF,MAAM,UAAU,wBAAwB;IACtC,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;AACvG,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,uBAAuB,CAAC,MAAc;IACpD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,CAAA;IAChE,IAAI,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,EAAE,CAAC,CAAA;IACvF,OAAO,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACnC,CAAC"}
|