bridgebench 3.1.0-alpha.0 → 3.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-JTVNKSMO.js → chunk-3CX4FIFZ.js} +1 -2
- package/dist/{chunk-NJTYVNP4.cjs → chunk-5OZGHLDH.cjs} +49 -50
- package/dist/{chunk-CIXITJW6.cjs → chunk-5PCDG7VA.cjs} +0 -1
- package/dist/{chunk-EQHRUV2I.js → chunk-KGZEQ6R5.js} +2 -3
- package/dist/{chunk-LFKEV2YL.js → chunk-MBC4C3VA.js} +3 -4
- package/dist/{chunk-4TWPCPRP.cjs → chunk-SMG3PORS.cjs} +10 -11
- package/dist/{chunk-UECBSKTD.js → chunk-UXFKTSQS.js} +0 -1
- package/dist/{chunk-7YCJSOK7.cjs → chunk-XPFZBHV2.cjs} +18 -19
- package/dist/cli.cjs +47 -48
- package/dist/cli.js +4 -5
- package/dist/client.cjs +5 -6
- package/dist/client.js +4 -5
- package/dist/contracts/index.cjs +2 -3
- package/dist/contracts/index.js +1 -2
- package/dist/index.cjs +4 -5
- package/dist/index.js +3 -4
- package/dist/tasks.cjs +3 -4
- package/dist/tasks.js +2 -3
- package/package.json +1 -1
- package/dist/chunk-4TWPCPRP.cjs.map +0 -1
- package/dist/chunk-7YCJSOK7.cjs.map +0 -1
- package/dist/chunk-CIXITJW6.cjs.map +0 -1
- package/dist/chunk-EQHRUV2I.js.map +0 -1
- package/dist/chunk-JTVNKSMO.js.map +0 -1
- package/dist/chunk-LFKEV2YL.js.map +0 -1
- package/dist/chunk-NJTYVNP4.cjs.map +0 -1
- package/dist/chunk-UECBSKTD.js.map +0 -1
- package/dist/cli.cjs.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/client.cjs.map +0 -1
- package/dist/client.js.map +0 -1
- package/dist/contracts/index.cjs.map +0 -1
- package/dist/contracts/index.js.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/tasks.cjs.map +0 -1
- package/dist/tasks.js.map +0 -1
package/dist/cli.cjs
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
var
|
|
8
|
+
var _chunkXPFZBHV2cjs = require('./chunk-XPFZBHV2.cjs');
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
|
|
@@ -16,7 +16,7 @@ var _chunk7YCJSOK7cjs = require('./chunk-7YCJSOK7.cjs');
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
var
|
|
19
|
+
var _chunk5OZGHLDHcjs = require('./chunk-5OZGHLDH.cjs');
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
|
|
@@ -27,12 +27,12 @@ var _chunkNJTYVNP4cjs = require('./chunk-NJTYVNP4.cjs');
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
var
|
|
30
|
+
var _chunkSMG3PORScjs = require('./chunk-SMG3PORS.cjs');
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
var
|
|
35
|
+
var _chunk5PCDG7VAcjs = require('./chunk-5PCDG7VA.cjs');
|
|
36
36
|
|
|
37
37
|
// src/commands.ts
|
|
38
38
|
var _path = require('path'); var _path2 = _interopRequireDefault(_path);
|
|
@@ -41,7 +41,7 @@ var _commander = require('commander');
|
|
|
41
41
|
// src/env.ts
|
|
42
42
|
var _fs = require('fs');
|
|
43
43
|
|
|
44
|
-
var ENV_PATH = _path2.default.join(
|
|
44
|
+
var ENV_PATH = _path2.default.join(_chunkSMG3PORScjs.findProjectRoot.call(void 0, _chunk5PCDG7VAcjs.importMetaUrl), ".env");
|
|
45
45
|
function loadProjectEnv() {
|
|
46
46
|
if (!_fs.existsSync.call(void 0, ENV_PATH)) return { status: "missing" };
|
|
47
47
|
try {
|
|
@@ -63,17 +63,17 @@ var defaultDependencies = {
|
|
|
63
63
|
setExitCode: (code) => {
|
|
64
64
|
process.exitCode = code;
|
|
65
65
|
},
|
|
66
|
-
createStore: (category) => new (0,
|
|
67
|
-
createTaskLoader: (category) => new (0,
|
|
68
|
-
createOpenRouter: (logger) => new (0,
|
|
69
|
-
publishTasks:
|
|
70
|
-
publishJournal:
|
|
71
|
-
resolveApiConfig:
|
|
66
|
+
createStore: (category) => new (0, _chunk5OZGHLDHcjs.ArenaStore)(_chunk5OZGHLDHcjs.categoryStoreConfig.call(void 0, category)),
|
|
67
|
+
createTaskLoader: (category) => new (0, _chunkSMG3PORScjs.TaskLoader)(category),
|
|
68
|
+
createOpenRouter: (logger) => new (0, _chunkSMG3PORScjs.OpenRouterClient)(_nullishCoalesce(process.env.OPENROUTER_API_KEY, () => ( "")), logger),
|
|
69
|
+
publishTasks: _chunkXPFZBHV2cjs.publishTasks,
|
|
70
|
+
publishJournal: _chunkXPFZBHV2cjs.publishJournal,
|
|
71
|
+
resolveApiConfig: _chunkXPFZBHV2cjs.resolveApiConfig
|
|
72
72
|
};
|
|
73
73
|
function parseCategory(value) {
|
|
74
|
-
const parsed =
|
|
74
|
+
const parsed = _chunk5PCDG7VAcjs.BenchmarkCategorySchema.safeParse(value);
|
|
75
75
|
if (!parsed.success) {
|
|
76
|
-
throw new (0, _commander.InvalidArgumentError)(`expected one of ${
|
|
76
|
+
throw new (0, _commander.InvalidArgumentError)(`expected one of ${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")}, received ${value}`);
|
|
77
77
|
}
|
|
78
78
|
return parsed.data;
|
|
79
79
|
}
|
|
@@ -95,7 +95,7 @@ function collectOption(value, previous = []) {
|
|
|
95
95
|
return [...previous, value];
|
|
96
96
|
}
|
|
97
97
|
function arenaRunConfigFromOptions(options) {
|
|
98
|
-
const competitorIds = _optionalChain([options, 'access', _ => _.competitor, 'optionalAccess', _2 => _2.length]) ?
|
|
98
|
+
const competitorIds = _optionalChain([options, 'access', _ => _.competitor, 'optionalAccess', _2 => _2.length]) ? _chunkSMG3PORScjs.resolveCompetitorRoster.call(void 0, options.competitor).map((model) => model.id) : void 0;
|
|
99
99
|
return {
|
|
100
100
|
category: options.category,
|
|
101
101
|
seed: options.seed,
|
|
@@ -111,7 +111,7 @@ function selectedCategories(options, command) {
|
|
|
111
111
|
throw new Error(`${command} accepts either --category or --all, not both`);
|
|
112
112
|
}
|
|
113
113
|
if (options.category) return [options.category];
|
|
114
|
-
if (options.all) return [...
|
|
114
|
+
if (options.all) return [..._chunk5PCDG7VAcjs.CATEGORIES];
|
|
115
115
|
throw new Error(`${command} requires --category <category> or --all`);
|
|
116
116
|
}
|
|
117
117
|
function logRoot(category) {
|
|
@@ -123,19 +123,19 @@ function displayPath(target) {
|
|
|
123
123
|
}
|
|
124
124
|
function buildProgram(overrides = {}) {
|
|
125
125
|
const dependencies = { ...defaultDependencies, ...overrides };
|
|
126
|
-
const program = new (0, _commander.Command)().name(
|
|
126
|
+
const program = new (0, _commander.Command)().name(_chunk5OZGHLDHcjs.PACKAGE_NAME).description("Autonomous BridgeBench arenas for reasoning and hallucination").version(_chunk5OZGHLDHcjs.ENGINE_VERSION).showHelpAfterError().configureHelp({ sortOptions: true, sortSubcommands: true }).configureOutput({
|
|
127
127
|
writeOut: (value) => dependencies.stdout(value.replace(/\n$/, "")),
|
|
128
128
|
writeErr: (value) => dependencies.stderr(value.replace(/\n$/, ""))
|
|
129
129
|
});
|
|
130
130
|
const models = program.command("models").description("Inspect or validate the pinned model registry");
|
|
131
131
|
models.command("list").description("List enabled competitors and judges").action(() => {
|
|
132
|
-
for (const model of
|
|
132
|
+
for (const model of _chunkSMG3PORScjs.listModels.call(void 0, )) {
|
|
133
133
|
dependencies.stdout(`${model.role.padEnd(10)} ${model.id.padEnd(40)} ${model.displayName}`);
|
|
134
134
|
}
|
|
135
135
|
});
|
|
136
136
|
models.command("validate").description("Validate pinned model IDs and judge capabilities against OpenRouter").action(async () => {
|
|
137
137
|
const client = dependencies.createOpenRouter();
|
|
138
|
-
for (const model of
|
|
138
|
+
for (const model of _chunkSMG3PORScjs.listModels.call(void 0, )) {
|
|
139
139
|
await client.validateModel(model);
|
|
140
140
|
dependencies.stdout(`\u2713 ${model.id} -> ${model.canonicalSlug}`);
|
|
141
141
|
}
|
|
@@ -143,20 +143,20 @@ function buildProgram(overrides = {}) {
|
|
|
143
143
|
const tasks = program.command("tasks").description("Validate or publish task packs");
|
|
144
144
|
tasks.command("validate").description("Validate public task schemas and pack balance").option(
|
|
145
145
|
"-c, --category <category>",
|
|
146
|
-
`limit to one category (${
|
|
146
|
+
`limit to one category (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
147
147
|
parseCategory
|
|
148
148
|
).option("--file <path>", "validate one public task without pack-balance checks").action(async (options) => {
|
|
149
149
|
if (options.file) {
|
|
150
150
|
if (options.category) {
|
|
151
151
|
throw new Error("tasks validate accepts --file or --category, not both");
|
|
152
152
|
}
|
|
153
|
-
const task = await
|
|
153
|
+
const task = await _chunkSMG3PORScjs.validatePublicTaskFile.call(void 0, options.file);
|
|
154
154
|
dependencies.stdout(
|
|
155
155
|
`\u2713 ${task.public.id}: public task validated (${task.public.category}/${task.public.cluster})`
|
|
156
156
|
);
|
|
157
157
|
return;
|
|
158
158
|
}
|
|
159
|
-
const categories = options.category ? [options.category] :
|
|
159
|
+
const categories = options.category ? [options.category] : _chunk5PCDG7VAcjs.CATEGORIES;
|
|
160
160
|
for (const category of categories) {
|
|
161
161
|
const loader = dependencies.createTaskLoader(category);
|
|
162
162
|
const loaded = await loader.loadAll();
|
|
@@ -169,12 +169,12 @@ function buildProgram(overrides = {}) {
|
|
|
169
169
|
});
|
|
170
170
|
tasks.command("publish").description("Publish public and private task halves to the configured API").option(
|
|
171
171
|
"-c, --category <category>",
|
|
172
|
-
`category to publish (${
|
|
172
|
+
`category to publish (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
173
173
|
parseCategory
|
|
174
174
|
).option("--all", "publish every category explicitly", false).action(async (options) => {
|
|
175
175
|
const categories = selectedCategories(options, "tasks publish");
|
|
176
176
|
const config = dependencies.resolveApiConfig();
|
|
177
|
-
dependencies.stdout(`Publishing ${categories.join(", ")} tasks to ${
|
|
177
|
+
dependencies.stdout(`Publishing ${categories.join(", ")} tasks to ${_chunkXPFZBHV2cjs.publishTarget.call(void 0, config)}`);
|
|
178
178
|
for (const category of categories) {
|
|
179
179
|
const result = await dependencies.publishTasks(category, config);
|
|
180
180
|
dependencies.stdout(`\u2713 ${category}: published ${result.imported} tasks`);
|
|
@@ -183,7 +183,7 @@ function buildProgram(overrides = {}) {
|
|
|
183
183
|
const arena = program.command("arena").description("Run and inspect arena matches");
|
|
184
184
|
arena.command("run").description("Run a paid deterministic match schedule").requiredOption(
|
|
185
185
|
"-c, --category <category>",
|
|
186
|
-
`arena to run (${
|
|
186
|
+
`arena to run (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
187
187
|
parseCategory
|
|
188
188
|
).option("-n, --matches <count>", "number of task-level matches", positiveInteger, 12).option("-s, --seed <seed>", "reproducible scheduling seed", "bridgebench-v3-mvp").option(
|
|
189
189
|
"--max-cost-usd <amount>",
|
|
@@ -196,7 +196,7 @@ function buildProgram(overrides = {}) {
|
|
|
196
196
|
collectOption
|
|
197
197
|
).option("--resume", "skip match IDs already present in the journal", false).option("--debug", "mirror structured log entries to the console", false).option("--no-health-stop", "do not halt when most matches have failed responses").action(async (options) => {
|
|
198
198
|
const config = arenaRunConfigFromOptions(options);
|
|
199
|
-
const logger = new (0,
|
|
199
|
+
const logger = new (0, _chunkSMG3PORScjs.FileArenaLogger)({
|
|
200
200
|
dir: logRoot(config.category),
|
|
201
201
|
verbose: options.debug
|
|
202
202
|
});
|
|
@@ -217,7 +217,7 @@ function buildProgram(overrides = {}) {
|
|
|
217
217
|
`[${event.data.completed}/${event.data.total}] ${event.data.taskId}: ${_nullishCoalesce(event.data.winnerModelId, () => ( "no contest"))} (${event.data.outcome}, $${event.data.costUsd.toFixed(4)})`
|
|
218
218
|
);
|
|
219
219
|
};
|
|
220
|
-
const runner = new (0,
|
|
220
|
+
const runner = new (0, _chunk5OZGHLDHcjs.ArenaRunner)(
|
|
221
221
|
dependencies.createOpenRouter(logger),
|
|
222
222
|
store,
|
|
223
223
|
progressOutput,
|
|
@@ -241,9 +241,9 @@ function buildProgram(overrides = {}) {
|
|
|
241
241
|
}
|
|
242
242
|
const runMatches = store.readAll().filter((match) => match.runId === result.runId);
|
|
243
243
|
if (runMatches.length > 0) {
|
|
244
|
-
const triage =
|
|
244
|
+
const triage = _chunk5OZGHLDHcjs.triageJournal.call(void 0, runMatches);
|
|
245
245
|
dependencies.stdout(`=== Run health ===
|
|
246
|
-
${
|
|
246
|
+
${_chunk5OZGHLDHcjs.formatTriage.call(void 0, triage)}`);
|
|
247
247
|
}
|
|
248
248
|
} finally {
|
|
249
249
|
process.removeListener("SIGINT", requestCancellation);
|
|
@@ -252,7 +252,7 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
252
252
|
});
|
|
253
253
|
arena.command("remote-run").description("Run a match schedule against the configured API and stream live arena events").requiredOption(
|
|
254
254
|
"-c, --category <category>",
|
|
255
|
-
`arena to run (${
|
|
255
|
+
`arena to run (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
256
256
|
parseCategory
|
|
257
257
|
).option("-n, --matches <count>", "number of task-level matches", positiveInteger, 12).option("-s, --seed <seed>", "reproducible scheduling seed", "bridgebench-v3-mvp").option(
|
|
258
258
|
"--max-cost-usd <amount>",
|
|
@@ -266,16 +266,16 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
266
266
|
).option("--resume", "skip match IDs already present in the journal", false).option("--debug", "mirror structured log entries to the console", false).option("--no-health-stop", "do not halt when most matches have failed responses").option("--mock", "use deterministic mock model completions instead of OpenRouter", false).option("--no-publish-matches", "skip publishing match results to the API as they complete").action(async (options) => {
|
|
267
267
|
const config = arenaRunConfigFromOptions(options);
|
|
268
268
|
const apiConfig = dependencies.resolveApiConfig();
|
|
269
|
-
const logger = new (0,
|
|
269
|
+
const logger = new (0, _chunkSMG3PORScjs.FileArenaLogger)({
|
|
270
270
|
dir: logRoot(config.category),
|
|
271
271
|
verbose: options.debug
|
|
272
272
|
});
|
|
273
273
|
dependencies.stdout(
|
|
274
|
-
`Remote ${config.category} run against ${
|
|
274
|
+
`Remote ${config.category} run against ${_chunkXPFZBHV2cjs.publishTarget.call(void 0, apiConfig)} (${options.mock ? "mock" : "openrouter"})`
|
|
275
275
|
);
|
|
276
276
|
dependencies.stdout(`Run log: ${displayPath(logger.filePath)}`);
|
|
277
277
|
try {
|
|
278
|
-
const result = await
|
|
278
|
+
const result = await _chunkXPFZBHV2cjs.runRemoteArena.call(void 0, apiConfig, {
|
|
279
279
|
config,
|
|
280
280
|
mock: options.mock,
|
|
281
281
|
publishMatches: options.publishMatches,
|
|
@@ -291,21 +291,21 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
291
291
|
});
|
|
292
292
|
arena.command("verify").description("Validate and replay outcomes, points, manifests, and Elo").option(
|
|
293
293
|
"-c, --category <category>",
|
|
294
|
-
`journal category (${
|
|
294
|
+
`journal category (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
295
295
|
parseCategory,
|
|
296
296
|
"reasoning"
|
|
297
297
|
).option("--journal <path>", "journal file to verify (defaults to the local category journal)").option("--manifests-dir <path>", "run-manifest directory (defaults to a runs/ sibling)").option("--json", "emit the machine-readable verification result", false).action(
|
|
298
298
|
(options) => {
|
|
299
|
-
const defaults =
|
|
299
|
+
const defaults = _chunk5OZGHLDHcjs.categoryStoreConfig.call(void 0, options.category);
|
|
300
300
|
const journalPath = options.journal ? _path2.default.resolve(options.journal) : defaults.journalPath;
|
|
301
301
|
const runsDir = options.manifestsDir ? _path2.default.resolve(options.manifestsDir) : options.journal ? _path2.default.join(_path2.default.dirname(journalPath), "runs") : defaults.runsDir;
|
|
302
|
-
const store = new (0,
|
|
302
|
+
const store = new (0, _chunk5OZGHLDHcjs.ArenaStore)({
|
|
303
303
|
...defaults,
|
|
304
304
|
journalPath,
|
|
305
305
|
runsDir,
|
|
306
306
|
readOnly: true
|
|
307
307
|
});
|
|
308
|
-
const verified =
|
|
308
|
+
const verified = _chunk5OZGHLDHcjs.verifyJournal.call(void 0, store.readAll(), options.category, {
|
|
309
309
|
manifestForRun: (runId) => store.readRunManifest(runId),
|
|
310
310
|
requireManifests: true
|
|
311
311
|
});
|
|
@@ -323,17 +323,17 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
323
323
|
);
|
|
324
324
|
arena.command("publish").description("Publish a verified match journal to the configured API").option(
|
|
325
325
|
"-c, --category <category>",
|
|
326
|
-
`journal to publish (${
|
|
326
|
+
`journal to publish (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
327
327
|
parseCategory
|
|
328
328
|
).option("--all", "publish every category explicitly", false).option("--allow-empty", "treat an empty journal as a successful no-op", false).action(async (options) => {
|
|
329
329
|
const categories = selectedCategories(options, "arena publish");
|
|
330
330
|
const config = dependencies.resolveApiConfig();
|
|
331
331
|
dependencies.stdout(
|
|
332
|
-
`Publishing ${categories.join(", ")} journals to ${
|
|
332
|
+
`Publishing ${categories.join(", ")} journals to ${_chunkXPFZBHV2cjs.publishTarget.call(void 0, config)}`
|
|
333
333
|
);
|
|
334
334
|
for (const category of categories) {
|
|
335
335
|
const store = dependencies.createStore(category);
|
|
336
|
-
|
|
336
|
+
_chunk5OZGHLDHcjs.verifyJournal.call(void 0, store.readAll(), category, {
|
|
337
337
|
manifestForRun: (runId) => store.readRunManifest(runId),
|
|
338
338
|
requireManifests: true
|
|
339
339
|
});
|
|
@@ -350,7 +350,7 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
350
350
|
});
|
|
351
351
|
arena.command("triage").description("Analyze failures, suspicious responses, and judging anomalies").option(
|
|
352
352
|
"-c, --category <category>",
|
|
353
|
-
`journal to analyze (${
|
|
353
|
+
`journal to analyze (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
354
354
|
parseCategory,
|
|
355
355
|
"reasoning"
|
|
356
356
|
).option("--run <runId>", "limit the report to one run ID").option("--json", "emit the machine-readable report", false).option("--strict", "exit nonzero when any anomaly is found", false).option("--allow-empty", "treat no matching journal lines as success", false).action(
|
|
@@ -362,9 +362,9 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
362
362
|
dependencies.stdout(message);
|
|
363
363
|
return;
|
|
364
364
|
}
|
|
365
|
-
const reports =
|
|
365
|
+
const reports = _chunk5OZGHLDHcjs.triageJournal.call(void 0, results);
|
|
366
366
|
dependencies.stdout(
|
|
367
|
-
options.json ? JSON.stringify(reports, null, 2) :
|
|
367
|
+
options.json ? JSON.stringify(reports, null, 2) : _chunk5OZGHLDHcjs.formatTriage.call(void 0, reports)
|
|
368
368
|
);
|
|
369
369
|
const anomalies = reports.reduce((sum, report) => sum + report.anomalies.length, 0);
|
|
370
370
|
if (options.strict && anomalies > 0) {
|
|
@@ -379,12 +379,12 @@ ${_chunkNJTYVNP4cjs.formatTriage.call(void 0, triage)}`);
|
|
|
379
379
|
});
|
|
380
380
|
program.command("report").description("Verify journals and rebuild JSON and Markdown reports").option(
|
|
381
381
|
"-c, --category <category>",
|
|
382
|
-
`limit to one category (${
|
|
382
|
+
`limit to one category (${_chunk5PCDG7VAcjs.CATEGORIES.join(", ")})`,
|
|
383
383
|
parseCategory
|
|
384
384
|
).action((options) => {
|
|
385
|
-
const categories = options.category ? [options.category] :
|
|
385
|
+
const categories = options.category ? [options.category] : _chunk5PCDG7VAcjs.CATEGORIES;
|
|
386
386
|
for (const category of categories) {
|
|
387
|
-
const snapshot =
|
|
387
|
+
const snapshot = _chunk5OZGHLDHcjs.writeReports.call(void 0, dependencies.createStore(category));
|
|
388
388
|
dependencies.stdout(`\u2713 ${category}: wrote reports for ${snapshot.matches.length} matches`);
|
|
389
389
|
}
|
|
390
390
|
});
|
|
@@ -395,7 +395,7 @@ async function runCli(argv = process.argv, overrides = {}) {
|
|
|
395
395
|
const envResult = loadProjectEnv();
|
|
396
396
|
if (envResult.status === "error") {
|
|
397
397
|
dependencies.stderr(
|
|
398
|
-
`Warning: project environment file could not be loaded: ${
|
|
398
|
+
`Warning: project environment file could not be loaded: ${_chunkSMG3PORScjs.redactSecrets.call(void 0, envResult.reason)}`
|
|
399
399
|
);
|
|
400
400
|
}
|
|
401
401
|
await buildProgram(dependencies).parseAsync(argv);
|
|
@@ -403,7 +403,6 @@ async function runCli(argv = process.argv, overrides = {}) {
|
|
|
403
403
|
|
|
404
404
|
// src/cli.ts
|
|
405
405
|
runCli().catch((error) => {
|
|
406
|
-
console.error(`BridgeBench: ${
|
|
406
|
+
console.error(`BridgeBench: ${_chunkSMG3PORScjs.sanitizeError.call(void 0, error)}`);
|
|
407
407
|
process.exitCode = 1;
|
|
408
408
|
});
|
|
409
|
-
//# sourceMappingURL=cli.cjs.map
|
package/dist/cli.js
CHANGED
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
publishTasks,
|
|
6
6
|
resolveApiConfig,
|
|
7
7
|
runRemoteArena
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-MBC4C3VA.js";
|
|
9
9
|
import {
|
|
10
10
|
ArenaRunner,
|
|
11
11
|
ArenaStore,
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
triageJournal,
|
|
17
17
|
verifyJournal,
|
|
18
18
|
writeReports
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-KGZEQ6R5.js";
|
|
20
20
|
import {
|
|
21
21
|
FileArenaLogger,
|
|
22
22
|
OpenRouterClient,
|
|
@@ -27,11 +27,11 @@ import {
|
|
|
27
27
|
resolveCompetitorRoster,
|
|
28
28
|
sanitizeError,
|
|
29
29
|
validatePublicTaskFile
|
|
30
|
-
} from "./chunk-
|
|
30
|
+
} from "./chunk-3CX4FIFZ.js";
|
|
31
31
|
import {
|
|
32
32
|
BenchmarkCategorySchema,
|
|
33
33
|
CATEGORIES
|
|
34
|
-
} from "./chunk-
|
|
34
|
+
} from "./chunk-UXFKTSQS.js";
|
|
35
35
|
|
|
36
36
|
// src/commands.ts
|
|
37
37
|
import path2 from "path";
|
|
@@ -405,4 +405,3 @@ runCli().catch((error) => {
|
|
|
405
405
|
console.error(`BridgeBench: ${sanitizeError(error)}`);
|
|
406
406
|
process.exitCode = 1;
|
|
407
407
|
});
|
|
408
|
-
//# sourceMappingURL=cli.js.map
|
package/dist/client.cjs
CHANGED
|
@@ -16,10 +16,10 @@
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
var
|
|
20
|
-
require('./chunk-
|
|
21
|
-
require('./chunk-
|
|
22
|
-
require('./chunk-
|
|
19
|
+
var _chunkXPFZBHV2cjs = require('./chunk-XPFZBHV2.cjs');
|
|
20
|
+
require('./chunk-5OZGHLDH.cjs');
|
|
21
|
+
require('./chunk-SMG3PORS.cjs');
|
|
22
|
+
require('./chunk-5PCDG7VA.cjs');
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
|
|
@@ -38,5 +38,4 @@ require('./chunk-CIXITJW6.cjs');
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
exports.ADMIN_KEY_HEADER =
|
|
42
|
-
//# sourceMappingURL=client.cjs.map
|
|
41
|
+
exports.ADMIN_KEY_HEADER = _chunkXPFZBHV2cjs.ADMIN_KEY_HEADER; exports.DEFAULT_API_TIMEOUT_MS = _chunkXPFZBHV2cjs.DEFAULT_API_TIMEOUT_MS; exports.REQUEST_SPACING_MS = _chunkXPFZBHV2cjs.REQUEST_SPACING_MS; exports.RemoteArenaEventSink = _chunkXPFZBHV2cjs.RemoteArenaEventSink; exports.chunk = _chunkXPFZBHV2cjs.chunk; exports.createRemoteRun = _chunkXPFZBHV2cjs.createRemoteRun; exports.delay = _chunkXPFZBHV2cjs.delay; exports.fetchExecutionPack = _chunkXPFZBHV2cjs.fetchExecutionPack; exports.getJson = _chunkXPFZBHV2cjs.getJson; exports.postChunks = _chunkXPFZBHV2cjs.postChunks; exports.postJson = _chunkXPFZBHV2cjs.postJson; exports.publishJournal = _chunkXPFZBHV2cjs.publishJournal; exports.publishJournalFromStore = _chunkXPFZBHV2cjs.publishJournalFromStore; exports.publishTarget = _chunkXPFZBHV2cjs.publishTarget; exports.publishTasks = _chunkXPFZBHV2cjs.publishTasks; exports.resolveApiConfig = _chunkXPFZBHV2cjs.resolveApiConfig; exports.runRemoteArena = _chunkXPFZBHV2cjs.runRemoteArena;
|
package/dist/client.js
CHANGED
|
@@ -16,10 +16,10 @@ import {
|
|
|
16
16
|
publishTasks,
|
|
17
17
|
resolveApiConfig,
|
|
18
18
|
runRemoteArena
|
|
19
|
-
} from "./chunk-
|
|
20
|
-
import "./chunk-
|
|
21
|
-
import "./chunk-
|
|
22
|
-
import "./chunk-
|
|
19
|
+
} from "./chunk-MBC4C3VA.js";
|
|
20
|
+
import "./chunk-KGZEQ6R5.js";
|
|
21
|
+
import "./chunk-3CX4FIFZ.js";
|
|
22
|
+
import "./chunk-UXFKTSQS.js";
|
|
23
23
|
export {
|
|
24
24
|
ADMIN_KEY_HEADER,
|
|
25
25
|
DEFAULT_API_TIMEOUT_MS,
|
|
@@ -39,4 +39,3 @@ export {
|
|
|
39
39
|
resolveApiConfig,
|
|
40
40
|
runRemoteArena
|
|
41
41
|
};
|
|
42
|
-
//# sourceMappingURL=client.js.map
|
package/dist/contracts/index.cjs
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
var
|
|
23
|
+
var _chunk5PCDG7VAcjs = require('../chunk-5PCDG7VA.cjs');
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
|
|
@@ -43,5 +43,4 @@ var _chunkCIXITJW6cjs = require('../chunk-CIXITJW6.cjs');
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
exports.BenchmarkCategorySchema =
|
|
47
|
-
//# sourceMappingURL=index.cjs.map
|
|
46
|
+
exports.BenchmarkCategorySchema = _chunk5PCDG7VAcjs.BenchmarkCategorySchema; exports.CATEGORIES = _chunk5PCDG7VAcjs.CATEGORIES; exports.CATEGORY_CLUSTERS = _chunk5PCDG7VAcjs.CATEGORY_CLUSTERS; exports.CATEGORY_META = _chunk5PCDG7VAcjs.CATEGORY_META; exports.CONTRACTS_VERSION = _chunk5PCDG7VAcjs.CONTRACTS_VERSION; exports.CompetitorResponseSchema = _chunk5PCDG7VAcjs.CompetitorResponseSchema; exports.JUDGE_VERDICT_TRANSPORT_SCHEMA = _chunk5PCDG7VAcjs.JUDGE_VERDICT_TRANSPORT_SCHEMA; exports.JudgeVerdictSchema = _chunk5PCDG7VAcjs.JudgeVerdictSchema; exports.JudgeVoteSchema = _chunk5PCDG7VAcjs.JudgeVoteSchema; exports.METHODOLOGY_VERSION = _chunk5PCDG7VAcjs.METHODOLOGY_VERSION; exports.MatchResultSchema = _chunk5PCDG7VAcjs.MatchResultSchema; exports.ModelCompletionSchema = _chunk5PCDG7VAcjs.ModelCompletionSchema; exports.ModelRoleSchema = _chunk5PCDG7VAcjs.ModelRoleSchema; exports.PanelDecisionSchema = _chunk5PCDG7VAcjs.PanelDecisionSchema; exports.TaskArtifactSchema = _chunk5PCDG7VAcjs.TaskArtifactSchema; exports.TaskPrivateSchema = _chunk5PCDG7VAcjs.TaskPrivateSchema; exports.TaskPublicSchema = _chunk5PCDG7VAcjs.TaskPublicSchema; exports.competitorContent = _chunk5PCDG7VAcjs.competitorContent; exports.competitorCost = _chunk5PCDG7VAcjs.competitorCost; exports.competitorOutputTokens = _chunk5PCDG7VAcjs.competitorOutputTokens; exports.competitorReasoningTokens = _chunk5PCDG7VAcjs.competitorReasoningTokens;
|
package/dist/contracts/index.js
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
competitorCost,
|
|
21
21
|
competitorOutputTokens,
|
|
22
22
|
competitorReasoningTokens
|
|
23
|
-
} from "../chunk-
|
|
23
|
+
} from "../chunk-UXFKTSQS.js";
|
|
24
24
|
export {
|
|
25
25
|
BenchmarkCategorySchema,
|
|
26
26
|
CATEGORIES,
|
|
@@ -44,4 +44,3 @@ export {
|
|
|
44
44
|
competitorOutputTokens,
|
|
45
45
|
competitorReasoningTokens
|
|
46
46
|
};
|
|
47
|
-
//# sourceMappingURL=index.js.map
|
package/dist/index.cjs
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
var
|
|
30
|
+
var _chunk5OZGHLDHcjs = require('./chunk-5OZGHLDH.cjs');
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
|
|
@@ -61,7 +61,7 @@ var _chunkNJTYVNP4cjs = require('./chunk-NJTYVNP4.cjs');
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
var
|
|
64
|
+
var _chunkSMG3PORScjs = require('./chunk-SMG3PORS.cjs');
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
|
|
@@ -84,7 +84,7 @@ var _chunk4TWPCPRPcjs = require('./chunk-4TWPCPRP.cjs');
|
|
|
84
84
|
|
|
85
85
|
|
|
86
86
|
|
|
87
|
-
var
|
|
87
|
+
var _chunk5PCDG7VAcjs = require('./chunk-5PCDG7VA.cjs');
|
|
88
88
|
|
|
89
89
|
|
|
90
90
|
|
|
@@ -167,5 +167,4 @@ var _chunkCIXITJW6cjs = require('./chunk-CIXITJW6.cjs');
|
|
|
167
167
|
|
|
168
168
|
|
|
169
169
|
|
|
170
|
-
exports.ArenaCancellationError =
|
|
171
|
-
//# sourceMappingURL=index.cjs.map
|
|
170
|
+
exports.ArenaCancellationError = _chunkSMG3PORScjs.ArenaCancellationError; exports.ArenaRunner = _chunk5OZGHLDHcjs.ArenaRunner; exports.ArenaStore = _chunk5OZGHLDHcjs.ArenaStore; exports.BenchmarkCategorySchema = _chunk5PCDG7VAcjs.BenchmarkCategorySchema; exports.CATEGORIES = _chunk5PCDG7VAcjs.CATEGORIES; exports.CATEGORY_CLUSTERS = _chunk5PCDG7VAcjs.CATEGORY_CLUSTERS; exports.CATEGORY_META = _chunk5PCDG7VAcjs.CATEGORY_META; exports.CONTRACTS_VERSION = _chunk5PCDG7VAcjs.CONTRACTS_VERSION; exports.CompetitorResponseSchema = _chunk5PCDG7VAcjs.CompetitorResponseSchema; exports.ELO_INITIAL = _chunk5OZGHLDHcjs.ELO_INITIAL; exports.ELO_K = _chunk5OZGHLDHcjs.ELO_K; exports.ENGINE_VERSION = _chunk5OZGHLDHcjs.ENGINE_VERSION; exports.FAST_RESPONSE_MS = _chunk5OZGHLDHcjs.FAST_RESPONSE_MS; exports.FileArenaLogger = _chunkSMG3PORScjs.FileArenaLogger; exports.JUDGE_VERDICT_TRANSPORT_SCHEMA = _chunk5PCDG7VAcjs.JUDGE_VERDICT_TRANSPORT_SCHEMA; exports.JudgePanel = _chunkSMG3PORScjs.JudgePanel; exports.JudgeVerdictSchema = _chunk5PCDG7VAcjs.JudgeVerdictSchema; exports.JudgeVoteSchema = _chunk5PCDG7VAcjs.JudgeVoteSchema; exports.LOW_OUTPUT_TOKENS = _chunk5OZGHLDHcjs.LOW_OUTPUT_TOKENS; exports.MAX_PROMPT_CHARS = _chunkSMG3PORScjs.MAX_PROMPT_CHARS; exports.METHODOLOGY_VERSION = _chunk5PCDG7VAcjs.METHODOLOGY_VERSION; exports.MODEL_REGISTRY = _chunkSMG3PORScjs.MODEL_REGISTRY; exports.MatchResultSchema = _chunk5PCDG7VAcjs.MatchResultSchema; exports.MockOpenRouterGateway = _chunk5OZGHLDHcjs.MockOpenRouterGateway; exports.ModelCompletionSchema = _chunk5PCDG7VAcjs.ModelCompletionSchema; exports.ModelRoleSchema = _chunk5PCDG7VAcjs.ModelRoleSchema; exports.OpenRouterClient = _chunkSMG3PORScjs.OpenRouterClient; exports.PACKAGE_NAME = _chunk5OZGHLDHcjs.PACKAGE_NAME; exports.PanelDecisionSchema = _chunk5PCDG7VAcjs.PanelDecisionSchema; exports.RUN_MANIFEST_VERSION = _chunk5OZGHLDHcjs.RUN_MANIFEST_VERSION; exports.RunManifestSchema = _chunk5OZGHLDHcjs.RunManifestSchema; exports.SOL_FABLE_PILOT_COMPETITOR_IDS = _chunkSMG3PORScjs.SOL_FABLE_PILOT_COMPETITOR_IDS; exports.TASKS_PER_CATEGORY = _chunkSMG3PORScjs.TASKS_PER_CATEGORY; exports.TASKS_PER_CLUSTER = _chunkSMG3PORScjs.TASKS_PER_CLUSTER; exports.TaskArtifactSchema = _chunk5PCDG7VAcjs.TaskArtifactSchema; exports.TaskLoader = _chunkSMG3PORScjs.TaskLoader; exports.TaskPrivateSchema = _chunk5PCDG7VAcjs.TaskPrivateSchema; exports.TaskPublicSchema = _chunk5PCDG7VAcjs.TaskPublicSchema; exports.anonymizeCompetitorOutput = _chunkSMG3PORScjs.anonymizeCompetitorOutput; exports.applyEloWin = _chunk5OZGHLDHcjs.applyEloWin; exports.assertPromptSize = _chunkSMG3PORScjs.assertPromptSize; exports.buildCompetitorPrompt = _chunkSMG3PORScjs.buildCompetitorPrompt; exports.buildJudgePayload = _chunkSMG3PORScjs.buildJudgePayload; exports.buildSnapshot = _chunk5OZGHLDHcjs.buildSnapshot; exports.canonicalJson = _chunk5OZGHLDHcjs.canonicalJson; exports.categoryStoreConfig = _chunk5OZGHLDHcjs.categoryStoreConfig; exports.classifyError = _chunk5OZGHLDHcjs.classifyError; exports.competitorContent = _chunk5PCDG7VAcjs.competitorContent; exports.competitorCost = _chunk5PCDG7VAcjs.competitorCost; exports.competitorOutputTokens = _chunk5PCDG7VAcjs.competitorOutputTokens; exports.competitorPromptPolicyHash = _chunkSMG3PORScjs.competitorPromptPolicyHash; exports.competitorReasoningTokens = _chunk5PCDG7VAcjs.competitorReasoningTokens; exports.createRunManifest = _chunk5OZGHLDHcjs.createRunManifest; exports.defaultTaskRoot = _chunkSMG3PORScjs.defaultTaskRoot; exports.detectResponseAnomalies = _chunk5OZGHLDHcjs.detectResponseAnomalies; exports.expectedScore = _chunk5OZGHLDHcjs.expectedScore; exports.formatTriage = _chunk5OZGHLDHcjs.formatTriage; exports.getModel = _chunkSMG3PORScjs.getModel; exports.isArenaCancellationError = _chunkSMG3PORScjs.isArenaCancellationError; exports.isRetryableError = _chunkSMG3PORScjs.isRetryableError; exports.judgePromptPolicyHash = _chunkSMG3PORScjs.judgePromptPolicyHash; exports.judgeSystemPrompt = _chunkSMG3PORScjs.judgeSystemPrompt; exports.judgeVerdictJsonSchema = _chunkSMG3PORScjs.judgeVerdictJsonSchema; exports.listModels = _chunkSMG3PORScjs.listModels; exports.mergePrivateHalves = _chunkSMG3PORScjs.mergePrivateHalves; exports.noopLogger = _chunkSMG3PORScjs.noopLogger; exports.parseJudgeVerdict = _chunkSMG3PORScjs.parseJudgeVerdict; exports.redactSecrets = _chunkSMG3PORScjs.redactSecrets; exports.renderMarkdown = _chunk5OZGHLDHcjs.renderMarkdown; exports.resolveCompetitorRoster = _chunkSMG3PORScjs.resolveCompetitorRoster; exports.runIdFromManifest = _chunk5OZGHLDHcjs.runIdFromManifest; exports.runManifestHash = _chunk5OZGHLDHcjs.runManifestHash; exports.runOpenRouterAttempt = _chunkSMG3PORScjs.runOpenRouterAttempt; exports.sanitizeError = _chunkSMG3PORScjs.sanitizeError; exports.scheduleMatches = _chunk5OZGHLDHcjs.scheduleMatches; exports.throwIfCancelled = _chunkSMG3PORScjs.throwIfCancelled; exports.triageJournal = _chunk5OZGHLDHcjs.triageJournal; exports.validatePublicTaskFile = _chunkSMG3PORScjs.validatePublicTaskFile; exports.verifiedEloState = _chunk5OZGHLDHcjs.verifiedEloState; exports.verifyJournal = _chunk5OZGHLDHcjs.verifyJournal; exports.writeReports = _chunk5OZGHLDHcjs.writeReports;
|
package/dist/index.js
CHANGED
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
verifiedEloState,
|
|
28
28
|
verifyJournal,
|
|
29
29
|
writeReports
|
|
30
|
-
} from "./chunk-
|
|
30
|
+
} from "./chunk-KGZEQ6R5.js";
|
|
31
31
|
import {
|
|
32
32
|
ArenaCancellationError,
|
|
33
33
|
FileArenaLogger,
|
|
@@ -61,7 +61,7 @@ import {
|
|
|
61
61
|
sanitizeError,
|
|
62
62
|
throwIfCancelled,
|
|
63
63
|
validatePublicTaskFile
|
|
64
|
-
} from "./chunk-
|
|
64
|
+
} from "./chunk-3CX4FIFZ.js";
|
|
65
65
|
import {
|
|
66
66
|
BenchmarkCategorySchema,
|
|
67
67
|
CATEGORIES,
|
|
@@ -84,7 +84,7 @@ import {
|
|
|
84
84
|
competitorCost,
|
|
85
85
|
competitorOutputTokens,
|
|
86
86
|
competitorReasoningTokens
|
|
87
|
-
} from "./chunk-
|
|
87
|
+
} from "./chunk-UXFKTSQS.js";
|
|
88
88
|
export {
|
|
89
89
|
ArenaCancellationError,
|
|
90
90
|
ArenaRunner,
|
|
@@ -168,4 +168,3 @@ export {
|
|
|
168
168
|
verifyJournal,
|
|
169
169
|
writeReports
|
|
170
170
|
};
|
|
171
|
-
//# sourceMappingURL=index.js.map
|
package/dist/tasks.cjs
CHANGED
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
var
|
|
11
|
-
require('./chunk-
|
|
10
|
+
var _chunkSMG3PORScjs = require('./chunk-SMG3PORS.cjs');
|
|
11
|
+
require('./chunk-5PCDG7VA.cjs');
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
|
|
@@ -18,5 +18,4 @@ require('./chunk-CIXITJW6.cjs');
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
exports.TASKS_PER_CATEGORY =
|
|
22
|
-
//# sourceMappingURL=tasks.cjs.map
|
|
21
|
+
exports.TASKS_PER_CATEGORY = _chunkSMG3PORScjs.TASKS_PER_CATEGORY; exports.TASKS_PER_CLUSTER = _chunkSMG3PORScjs.TASKS_PER_CLUSTER; exports.TaskLoader = _chunkSMG3PORScjs.TaskLoader; exports.buildCompetitorPrompt = _chunkSMG3PORScjs.buildCompetitorPrompt; exports.competitorPromptPolicyHash = _chunkSMG3PORScjs.competitorPromptPolicyHash; exports.defaultTaskRoot = _chunkSMG3PORScjs.defaultTaskRoot; exports.mergePrivateHalves = _chunkSMG3PORScjs.mergePrivateHalves; exports.validatePublicTaskFile = _chunkSMG3PORScjs.validatePublicTaskFile;
|
package/dist/tasks.js
CHANGED
|
@@ -7,8 +7,8 @@ import {
|
|
|
7
7
|
defaultTaskRoot,
|
|
8
8
|
mergePrivateHalves,
|
|
9
9
|
validatePublicTaskFile
|
|
10
|
-
} from "./chunk-
|
|
11
|
-
import "./chunk-
|
|
10
|
+
} from "./chunk-3CX4FIFZ.js";
|
|
11
|
+
import "./chunk-UXFKTSQS.js";
|
|
12
12
|
export {
|
|
13
13
|
TASKS_PER_CATEGORY,
|
|
14
14
|
TASKS_PER_CLUSTER,
|
|
@@ -19,4 +19,3 @@ export {
|
|
|
19
19
|
mergePrivateHalves,
|
|
20
20
|
validatePublicTaskFile
|
|
21
21
|
};
|
|
22
|
-
//# sourceMappingURL=tasks.js.map
|