@dsh-enhanced/assistant-evaluation 0.1.24 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -3
- package/bin/dsh-benchmark.js +21 -0
- package/lib/benchmark/cli.d.ts +6 -0
- package/lib/benchmark/cli.d.ts.map +1 -0
- package/lib/benchmark/cli.js +332 -0
- package/lib/benchmark/cli.js.map +1 -0
- package/lib/benchmark/corpus.d.ts +28 -0
- package/lib/benchmark/corpus.d.ts.map +1 -0
- package/lib/benchmark/corpus.js +173 -0
- package/lib/benchmark/corpus.js.map +1 -0
- package/lib/benchmark/deepseek.d.ts +15 -0
- package/lib/benchmark/deepseek.d.ts.map +1 -0
- package/lib/benchmark/deepseek.js +143 -0
- package/lib/benchmark/deepseek.js.map +1 -0
- package/lib/benchmark/holdout-evidence.d.ts +126 -0
- package/lib/benchmark/holdout-evidence.d.ts.map +1 -0
- package/lib/benchmark/holdout-evidence.js +466 -0
- package/lib/benchmark/holdout-evidence.js.map +1 -0
- package/lib/benchmark/holdout-protocol.d.ts +68 -0
- package/lib/benchmark/holdout-protocol.d.ts.map +1 -0
- package/lib/benchmark/holdout-protocol.js +271 -0
- package/lib/benchmark/holdout-protocol.js.map +1 -0
- package/lib/benchmark/holdout-provider.d.ts +26 -0
- package/lib/benchmark/holdout-provider.d.ts.map +1 -0
- package/lib/benchmark/holdout-provider.js +382 -0
- package/lib/benchmark/holdout-provider.js.map +1 -0
- package/lib/benchmark/holdout.d.ts +83 -0
- package/lib/benchmark/holdout.d.ts.map +1 -0
- package/lib/benchmark/holdout.js +391 -0
- package/lib/benchmark/holdout.js.map +1 -0
- package/lib/benchmark/index.d.ts +9 -0
- package/lib/benchmark/index.d.ts.map +1 -0
- package/lib/benchmark/index.js +7 -0
- package/lib/benchmark/index.js.map +1 -0
- package/lib/benchmark/memory-corpus.d.ts +55 -0
- package/lib/benchmark/memory-corpus.d.ts.map +1 -0
- package/lib/benchmark/memory-corpus.js +153 -0
- package/lib/benchmark/memory-corpus.js.map +1 -0
- package/lib/benchmark/memory-runtime.d.ts +63 -0
- package/lib/benchmark/memory-runtime.d.ts.map +1 -0
- package/lib/benchmark/memory-runtime.js +75 -0
- package/lib/benchmark/memory-runtime.js.map +1 -0
- package/lib/benchmark/native.d.ts +46 -0
- package/lib/benchmark/native.d.ts.map +1 -0
- package/lib/benchmark/native.js +300 -0
- package/lib/benchmark/native.js.map +1 -0
- package/lib/benchmark/report.d.ts +4 -0
- package/lib/benchmark/report.d.ts.map +1 -0
- package/lib/benchmark/report.js +153 -0
- package/lib/benchmark/report.js.map +1 -0
- package/lib/benchmark/runner.d.ts +25 -0
- package/lib/benchmark/runner.d.ts.map +1 -0
- package/lib/benchmark/runner.js +119 -0
- package/lib/benchmark/runner.js.map +1 -0
- package/lib/benchmark/schema.d.ts +20 -0
- package/lib/benchmark/schema.d.ts.map +1 -0
- package/lib/benchmark/schema.js +183 -0
- package/lib/benchmark/schema.js.map +1 -0
- package/lib/benchmark/store.d.ts +18 -0
- package/lib/benchmark/store.d.ts.map +1 -0
- package/lib/benchmark/store.js +321 -0
- package/lib/benchmark/store.js.map +1 -0
- package/lib/benchmark/strategy-capabilities.d.ts +154 -0
- package/lib/benchmark/strategy-capabilities.d.ts.map +1 -0
- package/lib/benchmark/strategy-capabilities.js +300 -0
- package/lib/benchmark/strategy-capabilities.js.map +1 -0
- package/lib/benchmark/strategy-config.d.ts +26 -0
- package/lib/benchmark/strategy-config.d.ts.map +1 -0
- package/lib/benchmark/strategy-config.js +80 -0
- package/lib/benchmark/strategy-config.js.map +1 -0
- package/lib/benchmark/strategy-corpus.d.ts +21 -0
- package/lib/benchmark/strategy-corpus.d.ts.map +1 -0
- package/lib/benchmark/strategy-corpus.js +95 -0
- package/lib/benchmark/strategy-corpus.js.map +1 -0
- package/lib/benchmark/strategy-evidence.d.ts +161 -0
- package/lib/benchmark/strategy-evidence.d.ts.map +1 -0
- package/lib/benchmark/strategy-evidence.js +520 -0
- package/lib/benchmark/strategy-evidence.js.map +1 -0
- package/lib/benchmark/strategy-executor.d.ts +27 -0
- package/lib/benchmark/strategy-executor.d.ts.map +1 -0
- package/lib/benchmark/strategy-executor.js +178 -0
- package/lib/benchmark/strategy-executor.js.map +1 -0
- package/lib/benchmark/strategy-goal-runtime.d.ts +78 -0
- package/lib/benchmark/strategy-goal-runtime.d.ts.map +1 -0
- package/lib/benchmark/strategy-goal-runtime.js +324 -0
- package/lib/benchmark/strategy-goal-runtime.js.map +1 -0
- package/lib/benchmark/strategy-meter.d.ts +62 -0
- package/lib/benchmark/strategy-meter.d.ts.map +1 -0
- package/lib/benchmark/strategy-meter.js +236 -0
- package/lib/benchmark/strategy-meter.js.map +1 -0
- package/lib/benchmark/strategy-owner.d.ts +71 -0
- package/lib/benchmark/strategy-owner.d.ts.map +1 -0
- package/lib/benchmark/strategy-owner.js +282 -0
- package/lib/benchmark/strategy-owner.js.map +1 -0
- package/lib/benchmark/strategy-plan.d.ts +51 -0
- package/lib/benchmark/strategy-plan.d.ts.map +1 -0
- package/lib/benchmark/strategy-plan.js +71 -0
- package/lib/benchmark/strategy-plan.js.map +1 -0
- package/lib/benchmark/strategy-policy.d.ts +16 -0
- package/lib/benchmark/strategy-policy.d.ts.map +1 -0
- package/lib/benchmark/strategy-policy.js +77 -0
- package/lib/benchmark/strategy-policy.js.map +1 -0
- package/lib/benchmark/strategy.d.ts +17 -0
- package/lib/benchmark/strategy.d.ts.map +1 -0
- package/lib/benchmark/strategy.js +21 -0
- package/lib/benchmark/strategy.js.map +1 -0
- package/lib/benchmark/types.d.ts +133 -0
- package/lib/benchmark/types.d.ts.map +1 -0
- package/lib/benchmark/types.js +3 -0
- package/lib/benchmark/types.js.map +1 -0
- package/lib/benchmark/usage.d.ts +36 -0
- package/lib/benchmark/usage.d.ts.map +1 -0
- package/lib/benchmark/usage.js +84 -0
- package/lib/benchmark/usage.js.map +1 -0
- package/lib/service.d.ts +50 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +556 -9
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +94 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +27 -5
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +324 -25
- package/lib/store.js.map +1 -1
- package/lib/types.d.ts +114 -5
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +125 -18
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
import { chmodSync, closeSync, constants, existsSync, lstatSync, mkdirSync, openSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, dirname } from 'node:path';
|
|
3
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
4
|
+
import { acceptanceCanonicalJson } from '@dsh-enhanced/task-acceptance-contract';
|
|
5
|
+
import { BenchmarkError, benchmarkInteger, benchmarkPlanDigest, benchmarkResultParser, benchmarkSchedule, benchmarkSnapshot, parseBenchmarkPlan } from './schema.js';
|
|
6
|
+
const schemaVersion = 1;
|
|
7
|
+
const tableColumns = Object.freeze({
|
|
8
|
+
benchmark_journal_plans: ['id', 'digest', 'plan_json', 'cells_json'],
|
|
9
|
+
benchmark_journal_intents: ['plan_id', 'cell_id', 'cell_json', 'started_at'],
|
|
10
|
+
benchmark_journal_results: ['plan_id', 'cell_id', 'result_json'],
|
|
11
|
+
});
|
|
12
|
+
function databaseError(message) { throw new BenchmarkError(message); }
|
|
13
|
+
function assertPrivateFile(path) {
|
|
14
|
+
const stat = lstatSync(path);
|
|
15
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1)
|
|
16
|
+
databaseError('benchmark database must be one regular, unlinked file');
|
|
17
|
+
if ((stat.mode & 0o077) !== 0)
|
|
18
|
+
databaseError('benchmark database permissions must be 0600');
|
|
19
|
+
const uid = process.getuid?.();
|
|
20
|
+
if (uid !== undefined && stat.uid !== uid)
|
|
21
|
+
databaseError('benchmark database must be owned by the current OS user');
|
|
22
|
+
}
|
|
23
|
+
function prepareFile(path) {
|
|
24
|
+
if (!isAbsolute(path))
|
|
25
|
+
databaseError('benchmark database path must be absolute or :memory:');
|
|
26
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
27
|
+
if (!existsSync(path)) {
|
|
28
|
+
const descriptor = openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_RDWR | constants.O_NOFOLLOW, 0o600);
|
|
29
|
+
closeSync(descriptor);
|
|
30
|
+
chmodSync(path, 0o600);
|
|
31
|
+
}
|
|
32
|
+
assertPrivateFile(path);
|
|
33
|
+
}
|
|
34
|
+
function json(value, label) {
|
|
35
|
+
try {
|
|
36
|
+
return JSON.parse(value);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return databaseError(`invalid ${label} JSON in benchmark journal`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function schemaTables(database) {
|
|
43
|
+
return database.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all()
|
|
44
|
+
.map(row => row.name);
|
|
45
|
+
}
|
|
46
|
+
function schemaObjects(database) {
|
|
47
|
+
return database.prepare("SELECT type, name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'").all();
|
|
48
|
+
}
|
|
49
|
+
function validateSchema(database) {
|
|
50
|
+
const version = database.prepare('PRAGMA user_version').get().user_version;
|
|
51
|
+
if (version > schemaVersion)
|
|
52
|
+
databaseError(`benchmark schema ${version} is newer than supported schema ${schemaVersion}`);
|
|
53
|
+
const initialObjects = schemaObjects(database);
|
|
54
|
+
if (version === 0) {
|
|
55
|
+
if (initialObjects.length !== 0)
|
|
56
|
+
databaseError('non-empty benchmark schema version zero is not supported');
|
|
57
|
+
database.exec('BEGIN IMMEDIATE');
|
|
58
|
+
try {
|
|
59
|
+
database.exec(`
|
|
60
|
+
CREATE TABLE benchmark_journal_plans (
|
|
61
|
+
id TEXT PRIMARY KEY, digest TEXT NOT NULL, plan_json TEXT NOT NULL, cells_json TEXT NOT NULL
|
|
62
|
+
) STRICT;
|
|
63
|
+
CREATE TABLE benchmark_journal_intents (
|
|
64
|
+
plan_id TEXT PRIMARY KEY REFERENCES benchmark_journal_plans(id) ON DELETE RESTRICT,
|
|
65
|
+
cell_id TEXT NOT NULL, cell_json TEXT NOT NULL, started_at INTEGER NOT NULL
|
|
66
|
+
) STRICT;
|
|
67
|
+
CREATE TABLE benchmark_journal_results (
|
|
68
|
+
plan_id TEXT NOT NULL REFERENCES benchmark_journal_plans(id) ON DELETE RESTRICT,
|
|
69
|
+
cell_id TEXT NOT NULL, result_json TEXT NOT NULL,
|
|
70
|
+
PRIMARY KEY (plan_id, cell_id)
|
|
71
|
+
) STRICT, WITHOUT ROWID;
|
|
72
|
+
PRAGMA user_version = 1;
|
|
73
|
+
`);
|
|
74
|
+
database.exec('COMMIT');
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
database.exec('ROLLBACK');
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
if (version !== 0 && version !== schemaVersion)
|
|
82
|
+
databaseError(`unsupported benchmark schema ${version}`);
|
|
83
|
+
const actualTables = schemaTables(database);
|
|
84
|
+
const expectedTables = Object.keys(tableColumns).sort();
|
|
85
|
+
if (acceptanceCanonicalJson(actualTables) !== acceptanceCanonicalJson(expectedTables))
|
|
86
|
+
databaseError('incomplete or unrecognized benchmark schema version one');
|
|
87
|
+
for (const [table, expected] of Object.entries(tableColumns)) {
|
|
88
|
+
const actual = database.prepare(`PRAGMA table_info(${table})`).all().map(row => row.name);
|
|
89
|
+
if (acceptanceCanonicalJson(actual) !== acceptanceCanonicalJson(expected))
|
|
90
|
+
databaseError('incomplete benchmark schema version one');
|
|
91
|
+
}
|
|
92
|
+
const tableSql = new Map(database.prepare("SELECT name, sql FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").all().map(row => [row.name, row.sql]));
|
|
93
|
+
if (![...tableSql.values()].every(sql => /\bSTRICT\b/u.test(sql)) || !/WITHOUT\s+ROWID/iu.test(tableSql.get('benchmark_journal_results') ?? '')) {
|
|
94
|
+
databaseError('benchmark schema version one must use strict tables and a without-rowid result key');
|
|
95
|
+
}
|
|
96
|
+
const columns = (table) => database.prepare(`PRAGMA table_info(${table})`).all();
|
|
97
|
+
const primaryKey = (table) => columns(table).filter(column => column.pk > 0).sort((left, right) => left.pk - right.pk).map(column => column.name);
|
|
98
|
+
if (!same(primaryKey('benchmark_journal_plans'), ['id']) || !same(primaryKey('benchmark_journal_intents'), ['plan_id'])
|
|
99
|
+
|| !same(primaryKey('benchmark_journal_results'), ['plan_id', 'cell_id']))
|
|
100
|
+
databaseError('benchmark schema version one lacks required primary keys');
|
|
101
|
+
const foreignKeys = (table) => database.prepare(`PRAGMA foreign_key_list(${table})`).all();
|
|
102
|
+
if (!foreignKeys('benchmark_journal_intents').some(key => key.table === 'benchmark_journal_plans' && key.from === 'plan_id' && key.to === 'id')
|
|
103
|
+
|| !foreignKeys('benchmark_journal_results').some(key => key.table === 'benchmark_journal_plans' && key.from === 'plan_id' && key.to === 'id'))
|
|
104
|
+
databaseError('benchmark schema version one lacks required foreign keys');
|
|
105
|
+
const objects = database.prepare("SELECT type, name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' AND type <> 'table'").all();
|
|
106
|
+
if (objects.some(object => object.type !== 'index'))
|
|
107
|
+
databaseError('benchmark schema version one contains unsupported objects');
|
|
108
|
+
const quickCheck = database.prepare('PRAGMA quick_check').get();
|
|
109
|
+
if (quickCheck.quick_check !== 'ok' || database.prepare('PRAGMA foreign_key_check').all().length !== 0)
|
|
110
|
+
databaseError('benchmark journal integrity check failed');
|
|
111
|
+
}
|
|
112
|
+
function openDatabase(path) {
|
|
113
|
+
if (path !== ':memory:')
|
|
114
|
+
prepareFile(path);
|
|
115
|
+
const database = new DatabaseSync(path, { enableForeignKeyConstraints: true });
|
|
116
|
+
try {
|
|
117
|
+
database.exec('PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 250; PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL;');
|
|
118
|
+
validateSchema(database);
|
|
119
|
+
if (path !== ':memory:')
|
|
120
|
+
assertPrivateFile(path);
|
|
121
|
+
return database;
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
database.close();
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function same(left, right) { return acceptanceCanonicalJson(left) === acceptanceCanonicalJson(right); }
|
|
129
|
+
export class BenchmarkStore {
|
|
130
|
+
#database;
|
|
131
|
+
#compiled = new Map();
|
|
132
|
+
constructor(path) { this.#database = openDatabase(path); }
|
|
133
|
+
#transaction(operation) {
|
|
134
|
+
try {
|
|
135
|
+
this.#database.exec('BEGIN IMMEDIATE');
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
throw new BenchmarkError('benchmark journal transaction unavailable');
|
|
139
|
+
}
|
|
140
|
+
try {
|
|
141
|
+
const value = operation();
|
|
142
|
+
this.#database.exec('COMMIT');
|
|
143
|
+
return value;
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
this.#database.exec('ROLLBACK');
|
|
147
|
+
throw error;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
#readTransaction(operation) {
|
|
151
|
+
try {
|
|
152
|
+
this.#database.exec('BEGIN');
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
throw new BenchmarkError('benchmark journal transaction unavailable');
|
|
156
|
+
}
|
|
157
|
+
try {
|
|
158
|
+
const value = operation();
|
|
159
|
+
this.#database.exec('COMMIT');
|
|
160
|
+
return value;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
this.#database.exec('ROLLBACK');
|
|
164
|
+
throw error;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
#row(id) {
|
|
168
|
+
const row = this.#database.prepare('SELECT id, digest, plan_json, cells_json FROM benchmark_journal_plans WHERE id = ?').get(id);
|
|
169
|
+
if (!row)
|
|
170
|
+
databaseError(`benchmark plan ${id} was not found`);
|
|
171
|
+
return row;
|
|
172
|
+
}
|
|
173
|
+
#readPlan(id) {
|
|
174
|
+
const row = this.#row(id);
|
|
175
|
+
const plan = parseBenchmarkPlan(json(row.plan_json, 'plan'));
|
|
176
|
+
const digest = benchmarkPlanDigest(plan);
|
|
177
|
+
if (plan.id !== id || row.digest !== digest)
|
|
178
|
+
databaseError('benchmark journal plan digest does not match its manifest');
|
|
179
|
+
const compiled = this.#compile(plan, digest);
|
|
180
|
+
if (row.cells_json !== JSON.stringify(compiled.cells))
|
|
181
|
+
databaseError('benchmark journal contains an invalid schedule');
|
|
182
|
+
return plan;
|
|
183
|
+
}
|
|
184
|
+
#compile(plan, digest = benchmarkPlanDigest(plan)) {
|
|
185
|
+
const cached = this.#compiled.get(plan.id);
|
|
186
|
+
if (cached?.digest === digest)
|
|
187
|
+
return cached;
|
|
188
|
+
const cells = benchmarkSchedule(plan);
|
|
189
|
+
const compiled = Object.freeze({
|
|
190
|
+
digest, cells, positions: new Map(cells.map((cell, index) => [cell.id, index])), parseResult: benchmarkResultParser(plan),
|
|
191
|
+
});
|
|
192
|
+
this.#compiled.set(plan.id, compiled);
|
|
193
|
+
return compiled;
|
|
194
|
+
}
|
|
195
|
+
#readResults(plan) {
|
|
196
|
+
const compiled = this.#compile(plan);
|
|
197
|
+
const rows = this.#database.prepare('SELECT cell_id, result_json FROM benchmark_journal_results WHERE plan_id = ?').all(plan.id);
|
|
198
|
+
const seen = new Set();
|
|
199
|
+
const parsed = rows.map(row => {
|
|
200
|
+
const result = compiled.parseResult(json(row.result_json, 'result'));
|
|
201
|
+
if (result.cell.id !== row.cell_id || seen.has(row.cell_id))
|
|
202
|
+
databaseError('benchmark journal contains duplicate or mismatched results');
|
|
203
|
+
seen.add(row.cell_id);
|
|
204
|
+
return result;
|
|
205
|
+
});
|
|
206
|
+
return Object.freeze(parsed.sort((left, right) => compiled.positions.get(left.cell.id) - compiled.positions.get(right.cell.id)));
|
|
207
|
+
}
|
|
208
|
+
create(input) {
|
|
209
|
+
const plan = parseBenchmarkPlan(input);
|
|
210
|
+
const digest = benchmarkPlanDigest(plan);
|
|
211
|
+
const cells = this.#compile(plan, digest).cells;
|
|
212
|
+
this.#transaction(() => {
|
|
213
|
+
const row = this.#database.prepare('SELECT id, digest, plan_json, cells_json FROM benchmark_journal_plans WHERE id = ?').get(plan.id);
|
|
214
|
+
if (row) {
|
|
215
|
+
const existing = this.#readPlan(plan.id);
|
|
216
|
+
if (row.digest !== digest || !same(existing, plan))
|
|
217
|
+
databaseError('benchmark plan id already has different frozen content');
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
this.#database.prepare('INSERT INTO benchmark_journal_plans(id, digest, plan_json, cells_json) VALUES (?, ?, ?, ?)')
|
|
221
|
+
.run(plan.id, digest, acceptanceCanonicalJson(plan), JSON.stringify(cells));
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
plan(id) { return this.#readPlan(id); }
|
|
225
|
+
results(id) { return this.#readResults(this.#readPlan(id)); }
|
|
226
|
+
status(id) {
|
|
227
|
+
return this.#readTransaction(() => {
|
|
228
|
+
const plan = this.#readPlan(id);
|
|
229
|
+
const compiled = this.#compile(plan);
|
|
230
|
+
const results = this.#readResults(plan);
|
|
231
|
+
const intent = this.#database.prepare('SELECT cell_id, cell_json, started_at FROM benchmark_journal_intents WHERE plan_id = ?').get(id);
|
|
232
|
+
if (!intent)
|
|
233
|
+
return Object.freeze({ results, runningCell: null });
|
|
234
|
+
benchmarkInteger(intent.started_at);
|
|
235
|
+
const stored = benchmarkSnapshot(json(intent.cell_json, 'running intent'));
|
|
236
|
+
const scheduled = compiled.cells.find(cell => cell.id === intent.cell_id);
|
|
237
|
+
const completed = new Set(results.map(result => result.cell.id));
|
|
238
|
+
const firstUnfinished = compiled.cells.find(cell => !completed.has(cell.id));
|
|
239
|
+
if (intent.cell_id !== stored.id || !scheduled || !same(scheduled, stored) || !firstUnfinished || !same(firstUnfinished, stored)
|
|
240
|
+
|| results.some(result => result.status === 'unknown'))
|
|
241
|
+
databaseError('benchmark journal contains an invalid running intent');
|
|
242
|
+
return Object.freeze({ results, runningCell: scheduled });
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
start(id, input, now) {
|
|
246
|
+
benchmarkInteger(now);
|
|
247
|
+
return this.#transaction(() => {
|
|
248
|
+
const plan = this.#readPlan(id);
|
|
249
|
+
const cells = this.#compile(plan).cells;
|
|
250
|
+
const cell = benchmarkSnapshot(input);
|
|
251
|
+
const scheduled = cells.find(candidate => candidate.id === cell.id);
|
|
252
|
+
if (!scheduled || !same(scheduled, cell))
|
|
253
|
+
databaseError('benchmark cell is not an exact scheduled cell');
|
|
254
|
+
const results = this.#readResults(plan);
|
|
255
|
+
if (results.some(result => result.cell.id === cell.id))
|
|
256
|
+
return false;
|
|
257
|
+
if (results.some(result => result.status === 'unknown'))
|
|
258
|
+
databaseError('benchmark plan has an unknown terminal result and cannot continue');
|
|
259
|
+
const intent = this.#database.prepare('SELECT cell_id, cell_json, started_at FROM benchmark_journal_intents WHERE plan_id = ?').get(id);
|
|
260
|
+
if (intent) {
|
|
261
|
+
const stored = benchmarkSnapshot(json(intent.cell_json, 'running intent'));
|
|
262
|
+
if (intent.cell_id !== stored.id || !cells.some(candidate => same(candidate, stored)))
|
|
263
|
+
databaseError('benchmark journal contains an invalid running intent');
|
|
264
|
+
databaseError('benchmark plan has a running intent and cannot be started again');
|
|
265
|
+
}
|
|
266
|
+
const completed = new Set(results.map(result => result.cell.id));
|
|
267
|
+
const first = cells.find(candidate => !completed.has(candidate.id));
|
|
268
|
+
if (!first || first.id !== cell.id)
|
|
269
|
+
databaseError('benchmark cell is not the first unfinished scheduled cell');
|
|
270
|
+
this.#database.prepare('INSERT INTO benchmark_journal_intents(plan_id, cell_id, cell_json, started_at) VALUES (?, ?, ?, ?)')
|
|
271
|
+
.run(id, cell.id, acceptanceCanonicalJson(cell), now);
|
|
272
|
+
return true;
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
finish(id, input) {
|
|
276
|
+
this.#transaction(() => {
|
|
277
|
+
const plan = this.#readPlan(id);
|
|
278
|
+
const result = this.#compile(plan).parseResult(input);
|
|
279
|
+
const prior = this.#database.prepare('SELECT cell_id, result_json FROM benchmark_journal_results WHERE plan_id = ? AND cell_id = ?').get(id, result.cell.id);
|
|
280
|
+
if (prior) {
|
|
281
|
+
const stored = this.#compile(plan).parseResult(json(prior.result_json, 'result'));
|
|
282
|
+
if (prior.cell_id === result.cell.id && same(stored, result))
|
|
283
|
+
return;
|
|
284
|
+
databaseError('benchmark result replay has different content');
|
|
285
|
+
}
|
|
286
|
+
const intent = this.#database.prepare('SELECT cell_id, cell_json, started_at FROM benchmark_journal_intents WHERE plan_id = ?').get(id);
|
|
287
|
+
if (!intent)
|
|
288
|
+
databaseError('benchmark result has no running intent');
|
|
289
|
+
const cell = benchmarkSnapshot(json(intent.cell_json, 'running intent'));
|
|
290
|
+
if (intent.cell_id !== cell.id || !same(cell, result.cell) || result.startedAt !== intent.started_at)
|
|
291
|
+
databaseError('benchmark result does not match its running intent or started time');
|
|
292
|
+
this.#database.prepare('INSERT INTO benchmark_journal_results(plan_id, cell_id, result_json) VALUES (?, ?, ?)')
|
|
293
|
+
.run(id, result.cell.id, acceptanceCanonicalJson(result));
|
|
294
|
+
this.#database.prepare('DELETE FROM benchmark_journal_intents WHERE plan_id = ?').run(id);
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
interrupt(id, now) {
|
|
298
|
+
benchmarkInteger(now);
|
|
299
|
+
this.#transaction(() => {
|
|
300
|
+
const plan = this.#readPlan(id);
|
|
301
|
+
const intent = this.#database.prepare('SELECT cell_id, cell_json, started_at FROM benchmark_journal_intents WHERE plan_id = ?').get(id);
|
|
302
|
+
if (!intent)
|
|
303
|
+
return;
|
|
304
|
+
const cell = benchmarkSnapshot(json(intent.cell_json, 'running intent'));
|
|
305
|
+
if (intent.cell_id !== cell.id || !this.#compile(plan).cells.some(candidate => same(candidate, cell)))
|
|
306
|
+
databaseError('benchmark journal contains an invalid running intent');
|
|
307
|
+
if (now < intent.started_at)
|
|
308
|
+
databaseError('benchmark interruption precedes its started time');
|
|
309
|
+
const result = this.#compile(plan).parseResult({
|
|
310
|
+
cell, status: 'unknown', verdict: 'unknown', metrics: {
|
|
311
|
+
inputTokens: null, outputTokens: null, costUsdMicros: null, toolCalls: null, rework: null, interventions: null, latencyMs: null,
|
|
312
|
+
}, evidenceDigest: null, reason: 'interrupted', startedAt: intent.started_at, completedAt: now,
|
|
313
|
+
});
|
|
314
|
+
this.#database.prepare('INSERT INTO benchmark_journal_results(plan_id, cell_id, result_json) VALUES (?, ?, ?)')
|
|
315
|
+
.run(id, cell.id, acceptanceCanonicalJson(result));
|
|
316
|
+
this.#database.prepare('DELETE FROM benchmark_journal_intents WHERE plan_id = ?').run(id);
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
close() { this.#database.close(); }
|
|
320
|
+
}
|
|
321
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/benchmark/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACrG,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAChF,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAGpK,MAAM,aAAa,GAAG,CAAC,CAAA;AACvB,MAAM,YAAY,GAAgD,MAAM,CAAC,MAAM,CAAC;IAC9E,uBAAuB,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,CAAC;IACpE,yBAAyB,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC;IAC5E,yBAAyB,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC;CACjE,CAAC,CAAA;AAWF,SAAS,aAAa,CAAC,OAAe,IAAW,MAAM,IAAI,cAAc,CAAC,OAAO,CAAC,CAAA,CAAC,CAAC;AAEpF,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC;QAAE,aAAa,CAAC,uDAAuD,CAAC,CAAA;IACvI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;QAAE,aAAa,CAAC,6CAA6C,CAAC,CAAA;IAC3F,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAA;IAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,GAAG;QAAE,aAAa,CAAC,yDAAyD,CAAC,CAAA;AACrH,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,aAAa,CAAC,sDAAsD,CAAC,CAAA;IAC5F,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IAC1D,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;QACxH,SAAS,CAAC,UAAU,CAAC,CAAA;QACrB,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC;AAED,SAAS,IAAI,CAAC,KAAa,EAAE,KAAa;IACxC,IAAI,CAAC;QAAC,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAY,CAAA;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,aAAa,CAAC,WAAW,KAAK,4BAA4B,CAAC,CAAA;IAAC,CAAC;AAC1H,CAAC;AAED,SAAS,YAAY,CAAC,QAAsB;IAC1C,OAAQ,QAAQ,CAAC,OAAO,CAAC,gGAAgG,CAAC,CAAC,GAAG,EAA8B;SACzJ,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC;AAED,SAAS,aAAa,CAAC,QAAsB;IAC3C,OAAO,QAAQ,CAAC,OAAO,CAAC,qEAAqE,CAAC,CAAC,GAAG,EAA2C,CAAA;AAC/I,CAAC;AAED,SAAS,cAAc,CAAC,QAAsB;IAC5C,MAAM,OAAO,GAAI,QAAQ,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAA+B,CAAC,YAAY,CAAA;IACxG,IAAI,OAAO,GAAG,aAAa;QAAE,aAAa,CAAC,oBAAoB,OAAO,mCAAmC,aAAa,EAAE,CAAC,CAAA;IACzH,MAAM,cAAc,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC9C,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;YAAE,aAAa,CAAC,0DAA0D,CAAC,CAAA;QAC1G,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QAChC,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC;;;;;;;;;;;;;;KAcf,CAAC,CAAA;YACA,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACzB,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,IAAI,OAAO,KAAK,aAAa;QAAE,aAAa,CAAC,gCAAgC,OAAO,EAAE,CAAC,CAAA;IACxG,MAAM,YAAY,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAA;IAC3C,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAA;IACvD,IAAI,uBAAuB,CAAC,YAAY,CAAC,KAAK,uBAAuB,CAAC,cAAc,CAAC;QAAE,aAAa,CAAC,yDAAyD,CAAC,CAAA;IAC/J,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAI,QAAQ,CAAC,OAAO,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,GAAG,EAA8B,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACtH,IAAI,uBAAuB,CAAC,MAAM,CAAC,KAAK,uBAAuB,CAAC,QAAQ,CAAC;YAAE,aAAa,CAAC,yCAAyC,CAAC,CAAA;IACrI,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAE,QAAQ,CAAC,OAAO,CAAC,uFAAuF,CAAC,CAAC,GAAG,EAA2C,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IACnN,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,2BAA2B,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAChJ,aAAa,CAAC,oFAAoF,CAAC,CAAA;IACrG,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAC,GAAG,EAAyC,CAAA;IAC/H,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACzJ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,yBAAyB,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,2BAA2B,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;WAClH,CAAC,IAAI,CAAC,UAAU,CAAC,2BAA2B,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAAE,aAAa,CAAC,0DAA0D,CAAC,CAAA;IACtJ,MAAM,WAAW,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,2BAA2B,KAAK,GAAG,CAAC,CAAC,GAAG,EAAwD,CAAA;IACxJ,IAAI,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,yBAAyB,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;WAC1I,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,yBAAyB,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC;QAAE,aAAa,CAAC,0DAA0D,CAAC,CAAA;IAC3N,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,yFAAyF,CAAC,CAAC,GAAG,EAA2C,CAAA;IAC1K,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC;QAAE,aAAa,CAAC,2DAA2D,CAAC,CAAA;IAC/H,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC,GAAG,EAA6B,CAAA;IAC1F,IAAI,UAAU,CAAC,WAAW,KAAK,IAAI,IAAK,QAAQ,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,GAAG,EAAgB,CAAC,MAAM,KAAK,CAAC;QAAE,aAAa,CAAC,0CAA0C,CAAC,CAAA;AAClL,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,IAAI,KAAK,UAAU;QAAE,WAAW,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,2BAA2B,EAAE,IAAI,EAAE,CAAC,CAAA;IAC9E,IAAI,CAAC;QACH,QAAQ,CAAC,IAAI,CAAC,4GAA4G,CAAC,CAAA;QAC3H,cAAc,CAAC,QAAQ,CAAC,CAAA;QACxB,IAAI,IAAI,KAAK,UAAU;YAAE,iBAAiB,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,CAAC,KAAK,EAAE,CAAA;QAChB,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,IAAa,EAAE,KAAc,IAAa,OAAO,uBAAuB,CAAC,IAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,CAAC,CAAA,CAAC,CAAC;AAEjI,MAAM,OAAO,cAAc;IAChB,SAAS,CAAc;IACvB,SAAS,GAAG,IAAI,GAAG,EAAwB,CAAA;IAEpD,YAAY,IAAY,IAAI,IAAI,CAAC,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA,CAAC,CAAC;IAEjE,YAAY,CAAI,SAAkB;QAChC,IAAI,CAAC;YACH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,cAAc,CAAC,2CAA2C,CAAC,CAAA;QACvE,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,SAAS,EAAE,CAAA;YACzB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC7B,OAAO,KAAK,CAAA;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAC/B,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,gBAAgB,CAAI,SAAkB;QACpC,IAAI,CAAC;YACH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,cAAc,CAAC,2CAA2C,CAAC,CAAA;QACvE,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,SAAS,EAAE,CAAA;YACzB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC7B,OAAO,KAAK,CAAA;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAC/B,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED,IAAI,CAAC,EAAU;QACb,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,oFAAoF,CAAC,CAAC,GAAG,CAAC,EAAE,CAAwB,CAAA;QACvJ,IAAI,CAAC,GAAG;YAAE,aAAa,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;QAC7D,OAAO,GAAG,CAAA;IACZ,CAAC;IAED,SAAS,CAAC,EAAU;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;QAC5D,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM;YAAE,aAAa,CAAC,2DAA2D,CAAC,CAAA;QACvH,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC5C,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,aAAa,CAAC,gDAAgD,CAAC,CAAA;QACtH,OAAO,IAAI,CAAA;IACb,CAAC;IAED,QAAQ,CAAC,IAAmB,EAAE,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC1C,IAAI,MAAM,EAAE,MAAM,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;QAC5C,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;QACrC,MAAM,QAAQ,GAAiB,MAAM,CAAC,MAAM,CAAC;YAC3C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,qBAAqB,CAAC,IAAI,CAAC;SAC1H,CAAC,CAAA;QACF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QACrC,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED,YAAY,CAAC,IAAmB;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,8EAA8E,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAgB,CAAA;QAC/I,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAA;YACpE,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,aAAa,CAAC,4DAA4D,CAAC,CAAA;YACxI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YACrB,OAAO,MAAM,CAAA;QACf,CAAC,CAAC,CAAA;QACF,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAE,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAE,CAAC,CAAC,CAAA;IACpI,CAAC;IAED,MAAM,CAAC,KAAoB;QACzB,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;QACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAA;QAC/C,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,oFAAoF,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAA;YAC5J,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBACxC,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC;oBAAE,aAAa,CAAC,wDAAwD,CAAC,CAAA;gBAC3H,OAAM;YACR,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,4FAA4F,CAAC;iBACjH,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;QAC/E,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,CAAC,EAAU,IAA6B,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA,CAAC,CAAC;IAEvE,OAAO,CAAC,EAAU,IAAgC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAA,CAAC,CAAC;IAEhG,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;YACpC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,wFAAwF,CAAC,CAAC,GAAG,CAAC,EAAE,CAA0B,CAAA;YAChK,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;YAEjE,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;YACnC,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAkB,CAAA;YAC3F,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,OAAO,CAAC,CAAA;YACzE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;YAChE,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;YAC5E,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC;mBAC3H,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;gBAAE,aAAa,CAAC,sDAAsD,CAAC,CAAA;YAC/H,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,EAAU,EAAE,KAAoB,EAAE,GAAW;QACjD,gBAAgB,CAAC,GAAG,CAAC,CAAA;QACrB,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAA;YACvC,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC;gBAAE,aAAa,CAAC,+CAA+C,CAAC,CAAA;YACxG,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;YACvC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;gBAAE,OAAO,KAAK,CAAA;YACpE,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;gBAAE,aAAa,CAAC,mEAAmE,CAAC,CAAA;YAC3I,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,wFAAwF,CAAC,CAAC,GAAG,CAAC,EAAE,CAA0B,CAAA;YAChK,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAkB,CAAA;gBAC3F,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;oBAAE,aAAa,CAAC,sDAAsD,CAAC,CAAA;gBAC5J,aAAa,CAAC,iEAAiE,CAAC,CAAA;YAClF,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;YAChE,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAA;YACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE;gBAAE,aAAa,CAAC,2DAA2D,CAAC,CAAA;YAC9G,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,oGAAoG,CAAC;iBACzH,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,uBAAuB,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAA;YACvD,OAAO,IAAI,CAAA;QACb,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,CAAC,EAAU,EAAE,KAAsB;QACvC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE;YACrB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,8FAA8F,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAA0B,CAAA;YACrL,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAA;gBACjF,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;oBAAE,OAAM;gBACpE,aAAa,CAAC,+CAA+C,CAAC,CAAA;YAChE,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,wFAAwF,CAAC,CAAC,GAAG,CAAC,EAAE,CAA0B,CAAA;YAChK,IAAI,CAAC,MAAM;gBAAE,aAAa,CAAC,wCAAwC,CAAC,CAAA;YACpE,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAkB,CAAA;YACzF,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,UAAU;gBAAE,aAAa,CAAC,oEAAoE,CAAC,CAAA;YACzL,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,uFAAuF,CAAC;iBAC5G,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAA;YAC3D,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC3F,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,SAAS,CAAC,EAAU,EAAE,GAAW;QAC/B,gBAAgB,CAAC,GAAG,CAAC,CAAA;QACrB,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE;YACrB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,wFAAwF,CAAC,CAAC,GAAG,CAAC,EAAE,CAA0B,CAAA;YAChK,IAAI,CAAC,MAAM;gBAAE,OAAM;YACnB,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAkB,CAAA;YACzF,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBAAE,aAAa,CAAC,sDAAsD,CAAC,CAAA;YAC5K,IAAI,GAAG,GAAG,MAAM,CAAC,UAAU;gBAAE,aAAa,CAAC,kDAAkD,CAAC,CAAA;YAC9F,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC;gBAC7C,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE;oBACpD,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI;iBAChI,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG;aAC/F,CAAC,CAAA;YACF,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,uFAAuF,CAAC;iBAC5G,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAA;YACpD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC3F,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,KAAW,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA,CAAC,CAAC;CACzC"}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
export declare const strategyBenchmarkMinimalRecipe: Readonly<{
|
|
2
|
+
protocol: "dsh-native-goal-strategy-recipe-v1";
|
|
3
|
+
parentTools: readonly string[];
|
|
4
|
+
strategyTool: "goal_strategy";
|
|
5
|
+
strategyKinds: readonly string[];
|
|
6
|
+
/** Strategy is the only arm difference. Do not add task data here. */
|
|
7
|
+
strategyEnabled: true;
|
|
8
|
+
}>;
|
|
9
|
+
export interface StrategyCapabilitySourceModule {
|
|
10
|
+
/** Package specifier resolved from the evaluation package actually running. */
|
|
11
|
+
packageName: string;
|
|
12
|
+
/** Relative files below the resolved package root. These are deployed lib files. */
|
|
13
|
+
files: readonly string[];
|
|
14
|
+
}
|
|
15
|
+
export interface StrategyCapabilitySourceGroups {
|
|
16
|
+
common: {
|
|
17
|
+
tools: readonly StrategyCapabilitySourceModule[];
|
|
18
|
+
policy: readonly StrategyCapabilitySourceModule[];
|
|
19
|
+
runtime: readonly StrategyCapabilitySourceModule[];
|
|
20
|
+
};
|
|
21
|
+
strategy: {
|
|
22
|
+
guide: readonly StrategyCapabilitySourceModule[];
|
|
23
|
+
tool: readonly StrategyCapabilitySourceModule[];
|
|
24
|
+
policy: readonly StrategyCapabilitySourceModule[];
|
|
25
|
+
runtime: readonly StrategyCapabilitySourceModule[];
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export interface StrategyCapabilityToolDefinition {
|
|
29
|
+
name: string;
|
|
30
|
+
description: string;
|
|
31
|
+
parameters: unknown;
|
|
32
|
+
output: unknown;
|
|
33
|
+
}
|
|
34
|
+
export interface StrategyCapabilityPolicyProbe {
|
|
35
|
+
request: unknown;
|
|
36
|
+
/** The public `AssistantPolicyService.evaluate()` result expected before dispatch. */
|
|
37
|
+
effect: 'allow' | 'deny';
|
|
38
|
+
/** Exact validated policy configuration passed when the service was installed. */
|
|
39
|
+
config: unknown;
|
|
40
|
+
}
|
|
41
|
+
export interface StrategyCapabilityExpectationInput {
|
|
42
|
+
resolverDirectory: string;
|
|
43
|
+
persona: string;
|
|
44
|
+
recipe?: typeof strategyBenchmarkMinimalRecipe;
|
|
45
|
+
sources: StrategyCapabilitySourceGroups;
|
|
46
|
+
commonTools: readonly StrategyCapabilityToolDefinition[];
|
|
47
|
+
strategyTools: readonly StrategyCapabilityToolDefinition[];
|
|
48
|
+
commonPolicy: StrategyCapabilityPolicyProbe;
|
|
49
|
+
strategyPolicy: StrategyCapabilityPolicyProbe;
|
|
50
|
+
}
|
|
51
|
+
/** The only plan-construction input. Package and tool selection are fixed. */
|
|
52
|
+
export interface FixedStrategyCapabilityExpectationInput {
|
|
53
|
+
resolverDirectory: string;
|
|
54
|
+
/** Adds the supported production route implementation to both arms' source commitment. */
|
|
55
|
+
modelProvider?: string;
|
|
56
|
+
persona: string;
|
|
57
|
+
recipe?: typeof strategyBenchmarkMinimalRecipe;
|
|
58
|
+
}
|
|
59
|
+
export interface StrategyCapabilitySourceIdentity {
|
|
60
|
+
packageName: string;
|
|
61
|
+
version: string;
|
|
62
|
+
manifestDigest: string;
|
|
63
|
+
files: readonly {
|
|
64
|
+
path: string;
|
|
65
|
+
digest: string;
|
|
66
|
+
}[];
|
|
67
|
+
digest: string;
|
|
68
|
+
}
|
|
69
|
+
/** The eight stable slots already carried by StrategyBenchmarkCapabilities. */
|
|
70
|
+
export interface StrategyCapabilityExpectation {
|
|
71
|
+
readonly capabilities: {
|
|
72
|
+
readonly common: {
|
|
73
|
+
readonly persona: string;
|
|
74
|
+
readonly tools: string;
|
|
75
|
+
readonly policy: string;
|
|
76
|
+
readonly runtime: string;
|
|
77
|
+
};
|
|
78
|
+
readonly strategy: {
|
|
79
|
+
readonly guide: string;
|
|
80
|
+
readonly tool: string;
|
|
81
|
+
readonly policy: string;
|
|
82
|
+
readonly runtime: string;
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
readonly sourceIdentity: Readonly<Record<'common.tools' | 'common.policy' | 'common.runtime' | 'strategy.guide' | 'strategy.tool' | 'strategy.policy' | 'strategy.runtime', readonly StrategyCapabilitySourceIdentity[]>>;
|
|
86
|
+
readonly commonTools: readonly StrategyCapabilityToolDefinition[];
|
|
87
|
+
readonly strategyTools: readonly StrategyCapabilityToolDefinition[];
|
|
88
|
+
readonly commonPolicy: StrategyCapabilityPolicyProbe;
|
|
89
|
+
readonly strategyPolicy: StrategyCapabilityPolicyProbe;
|
|
90
|
+
readonly persona: string;
|
|
91
|
+
readonly recipeDigest: string;
|
|
92
|
+
/** Internal re-probe inputs; they are resolved again before every assertion. */
|
|
93
|
+
readonly resolverDirectory: string;
|
|
94
|
+
readonly resolvedSourceModules: Readonly<Record<'common.tools' | 'common.policy' | 'common.runtime' | 'strategy.guide' | 'strategy.tool' | 'strategy.policy' | 'strategy.runtime', readonly StrategyCapabilitySourceModule[]>>;
|
|
95
|
+
}
|
|
96
|
+
export declare function resolveStrategyCapabilitySource(input: {
|
|
97
|
+
resolverDirectory: string;
|
|
98
|
+
module: StrategyCapabilitySourceModule;
|
|
99
|
+
}): StrategyCapabilitySourceIdentity;
|
|
100
|
+
/**
|
|
101
|
+
* Fixed benchmark capability construction. It hashes each selected installed
|
|
102
|
+
* package's manifest and every deployed JavaScript file under lib, so a lockfile range or
|
|
103
|
+
* a caller-provided digest cannot stand in for the code actually resolved.
|
|
104
|
+
*/
|
|
105
|
+
export declare function createFixedStrategyCapabilityExpectation(input: FixedStrategyCapabilityExpectationInput): Readonly<StrategyCapabilityExpectation>;
|
|
106
|
+
/** Minimal public Context surface. Callers may not supply a look-alike host. */
|
|
107
|
+
export interface StrategyCapabilityContext {
|
|
108
|
+
get(name: string): unknown;
|
|
109
|
+
tools?: {
|
|
110
|
+
get(name: string, scope?: unknown): unknown;
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
export interface StrategyCapabilityParentRequest {
|
|
114
|
+
/** Captured from the actual LLM request options, never reconstructed later. */
|
|
115
|
+
system: string;
|
|
116
|
+
tools: readonly StrategyCapabilityToolDefinition[];
|
|
117
|
+
}
|
|
118
|
+
export interface StrategyCapabilityRuntimeObservation {
|
|
119
|
+
readonly sourceDigest: string;
|
|
120
|
+
readonly services: Readonly<{
|
|
121
|
+
goals: true;
|
|
122
|
+
assistantPolicy: true;
|
|
123
|
+
tools: true;
|
|
124
|
+
}>;
|
|
125
|
+
readonly parentTools: readonly StrategyCapabilityToolDefinition[];
|
|
126
|
+
readonly policy: Readonly<{
|
|
127
|
+
common: 'allow' | 'deny';
|
|
128
|
+
strategy: 'allow' | 'deny';
|
|
129
|
+
configuration: Readonly<Record<string, unknown>>;
|
|
130
|
+
configDigest: string;
|
|
131
|
+
}>;
|
|
132
|
+
/** This proves the registered persona section value, not a whole rendered prompt. */
|
|
133
|
+
readonly personaDigest: string;
|
|
134
|
+
readonly dynamicGoalContext: 'not-attested';
|
|
135
|
+
}
|
|
136
|
+
export declare class StrategyCapabilityRuntimeHost {
|
|
137
|
+
#private;
|
|
138
|
+
private constructor();
|
|
139
|
+
static fromMountedContext(ctx: StrategyCapabilityContext): StrategyCapabilityRuntimeHost;
|
|
140
|
+
/**
|
|
141
|
+
* Call this from the real `agent/request` hook with the definitions emitted
|
|
142
|
+
* to the parent model request. It is intentionally a one-shot capture: a
|
|
143
|
+
* later synthetic array cannot replace the observed request.
|
|
144
|
+
*/
|
|
145
|
+
captureParentRequest(request: StrategyCapabilityParentRequest): void;
|
|
146
|
+
/** Capture the real owner-bound requests used for the public Policy API. */
|
|
147
|
+
capturePolicyRequests(requests: {
|
|
148
|
+
common: unknown;
|
|
149
|
+
strategy: unknown;
|
|
150
|
+
}): void;
|
|
151
|
+
observe(expectation: StrategyCapabilityExpectation, enabled: boolean, parentScope?: unknown): Readonly<StrategyCapabilityRuntimeObservation>;
|
|
152
|
+
}
|
|
153
|
+
export declare function assertStrategyCapabilityRuntime(expectation: StrategyCapabilityExpectation, host: StrategyCapabilityRuntimeHost, enabled: boolean, parentScope?: unknown): Readonly<StrategyCapabilityRuntimeObservation>;
|
|
154
|
+
//# sourceMappingURL=strategy-capabilities.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"strategy-capabilities.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-capabilities.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,8BAA8B;;;;;IAKzC,uEAAuE;;EAEvE,CAAA;AAEF,MAAM,WAAW,8BAA8B;IAC7C,+EAA+E;IAC/E,WAAW,EAAE,MAAM,CAAA;IACnB,qFAAqF;IACrF,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CACzB;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE;QAAE,KAAK,EAAE,SAAS,8BAA8B,EAAE,CAAC;QAAC,MAAM,EAAE,SAAS,8BAA8B,EAAE,CAAC;QAAC,OAAO,EAAE,SAAS,8BAA8B,EAAE,CAAA;KAAE,CAAA;IACnK,QAAQ,EAAE;QAAE,KAAK,EAAE,SAAS,8BAA8B,EAAE,CAAC;QAAC,IAAI,EAAE,SAAS,8BAA8B,EAAE,CAAC;QAAC,MAAM,EAAE,SAAS,8BAA8B,EAAE,CAAC;QAAC,OAAO,EAAE,SAAS,8BAA8B,EAAE,CAAA;KAAE,CAAA;CACvN;AAED,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,OAAO,CAAA;IACnB,MAAM,EAAE,OAAO,CAAA;CAChB;AAED,MAAM,WAAW,6BAA6B;IAC5C,OAAO,EAAE,OAAO,CAAA;IAChB,sFAAsF;IACtF,MAAM,EAAE,OAAO,GAAG,MAAM,CAAA;IACxB,kFAAkF;IAClF,MAAM,EAAE,OAAO,CAAA;CAChB;AAED,MAAM,WAAW,kCAAkC;IACjD,iBAAiB,EAAE,MAAM,CAAA;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,8BAA8B,CAAA;IAC9C,OAAO,EAAE,8BAA8B,CAAA;IACvC,WAAW,EAAE,SAAS,gCAAgC,EAAE,CAAA;IACxD,aAAa,EAAE,SAAS,gCAAgC,EAAE,CAAA;IAC1D,YAAY,EAAE,6BAA6B,CAAA;IAC3C,cAAc,EAAE,6BAA6B,CAAA;CAC9C;AAED,+EAA+E;AAC/E,MAAM,WAAW,uCAAuC;IACtD,iBAAiB,EAAE,MAAM,CAAA;IACzB,0FAA0F;IAC1F,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,OAAO,8BAA8B,CAAA;CAC/C;AAED,MAAM,WAAW,gCAAgC;IAC/C,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,KAAK,EAAE,SAAS;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;IAClD,MAAM,EAAE,MAAM,CAAA;CACf;AAED,+EAA+E;AAC/E,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,YAAY,EAAE;QACrB,QAAQ,CAAC,MAAM,EAAE;YAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAA;QACxH,QAAQ,CAAC,QAAQ,EAAE;YAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAA;KACxH,CAAA;IACD,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,GAAG,eAAe,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,eAAe,GAAG,iBAAiB,GAAG,kBAAkB,EAAE,SAAS,gCAAgC,EAAE,CAAC,CAAC,CAAA;IACzN,QAAQ,CAAC,WAAW,EAAE,SAAS,gCAAgC,EAAE,CAAA;IACjE,QAAQ,CAAC,aAAa,EAAE,SAAS,gCAAgC,EAAE,CAAA;IACnE,QAAQ,CAAC,YAAY,EAAE,6BAA6B,CAAA;IACpD,QAAQ,CAAC,cAAc,EAAE,6BAA6B,CAAA;IACtD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAA;IAClC,QAAQ,CAAC,qBAAqB,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,GAAG,eAAe,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,eAAe,GAAG,iBAAiB,GAAG,kBAAkB,EAAE,SAAS,8BAA8B,EAAE,CAAC,CAAC,CAAA;CAC/N;AAmCD,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IAAE,iBAAiB,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,8BAA8B,CAAA;CAAE,GAAG,gCAAgC,CAkB9J;AA8ED;;;;GAIG;AACH,wBAAgB,wCAAwC,CAAC,KAAK,EAAE,uCAAuC,GAAG,QAAQ,CAAC,6BAA6B,CAAC,CAmBhJ;AAED,iFAAiF;AACjF,MAAM,WAAW,yBAAyB;IACxC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;IAC1B,KAAK,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,OAAO,CAAA;KAAE,CAAA;CACxD;AACD,MAAM,WAAW,+BAA+B;IAC9C,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,SAAS,gCAAgC,EAAE,CAAA;CACnD;AACD,MAAM,WAAW,oCAAoC;IACnD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAAE,KAAK,EAAE,IAAI,CAAC;QAAC,eAAe,EAAE,IAAI,CAAC;QAAC,KAAK,EAAE,IAAI,CAAA;KAAE,CAAC,CAAA;IAChF,QAAQ,CAAC,WAAW,EAAE,SAAS,gCAAgC,EAAE,CAAA;IACjE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;QAAE,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAC;QAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC3J,qFAAqF;IACrF,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,cAAc,CAAA;CAC5C;AAGD,qBAAa,6BAA6B;;IAIxC,OAAO;IACP,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,yBAAyB,GAAG,6BAA6B;IAIxF;;;;OAIG;IACH,oBAAoB,CAAC,OAAO,EAAE,+BAA+B,GAAG,IAAI;IAKpE,4EAA4E;IAC5E,qBAAqB,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI;IAI7E,OAAO,CAAC,WAAW,EAAE,6BAA6B,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC,oCAAoC,CAAC;CAqC7I;AACD,wBAAgB,+BAA+B,CAAC,WAAW,EAAE,6BAA6B,EAAE,IAAI,EAAE,6BAA6B,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC,oCAAoC,CAAC,CAGxN"}
|