@artemiskit/sdk 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/__tests__/experiment-fixtures.d.ts +44 -0
- package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
- package/dist/builders/index.js +19 -66
- package/dist/contracts/index.js +3 -50
- package/dist/experiment-files.d.ts +54 -0
- package/dist/experiment-files.d.ts.map +1 -0
- package/dist/experiment-fixture.d.ts +20 -0
- package/dist/experiment-fixture.d.ts.map +1 -0
- package/dist/experiment-summary.d.ts +44 -0
- package/dist/experiment-summary.d.ts.map +1 -0
- package/dist/experiment-validation.d.ts +48 -0
- package/dist/experiment-validation.d.ts.map +1 -0
- package/dist/experiment.d.ts +162 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +6 -1
- package/dist/guardian/circuit-breaker.d.ts.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2557 -1863
- package/dist/matchers/index.js +13 -60
- package/dist/matchers/jest.js +0 -47
- package/dist/matchers/vitest.js +0 -47
- package/dist/utils/index.js +11 -58
- package/package.json +6 -5
- package/src/__tests__/artemiskit.test.ts +1 -2
- package/src/__tests__/builders.test.ts +0 -2
- package/src/__tests__/circuit-breaker.test.ts +1 -1
- package/src/__tests__/compare.test.ts +1 -1
- package/src/__tests__/contracts.test.ts +1 -1
- package/src/__tests__/experiment-case-level.test.ts +212 -0
- package/src/__tests__/experiment-fixtures.ts +194 -0
- package/src/__tests__/experiment.test.ts +1262 -0
- package/src/__tests__/guardian-modes.test.ts +2 -2
- package/src/__tests__/guardrails.test.ts +1 -1
- package/src/__tests__/matchers.test.ts +1 -1
- package/src/__tests__/semantic-validator.test.ts +15 -10
- package/src/__tests__/validate.test.ts +0 -4
- package/src/__tests__/workflow-recovery.test.ts +7 -3
- package/src/__tests__/workflow-report.test.ts +6 -3
- package/src/__tests__/workflow.test.ts +11 -5
- package/src/experiment-files.ts +190 -0
- package/src/experiment-fixture.ts +70 -0
- package/src/experiment-summary.ts +172 -0
- package/src/experiment-validation.ts +426 -0
- package/src/experiment.ts +630 -0
- package/src/guardian/circuit-breaker.ts +6 -1
- package/src/index.ts +52 -0
- package/tsconfig.typecheck.json +12 -0
|
@@ -0,0 +1,1262 @@
|
|
|
1
|
+
import { afterEach, describe, expect, mock, setDefaultTimeout, test } from 'bun:test';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { readFile, readdir, stat, writeFile } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import type { ExperimentRunResult, GenerateResult, ModelClient } from '@artemiskit/core';
|
|
6
|
+
import * as realCore from '../../../core/src/index';
|
|
7
|
+
|
|
8
|
+
// Bun shares module mocks across test files in one run, and earlier SDK test files stub
|
|
9
|
+
// `@artemiskit/core` (for example `runScenario` and `createAdapter`) without restoring them.
|
|
10
|
+
// These tests exercise the real experiment pipeline, so rebind the real implementations.
|
|
11
|
+
mock.module('@artemiskit/core', () => ({ ...realCore }));
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
EXPERIMENT_EXIT_CODES,
|
|
15
|
+
ExperimentSurfaceError,
|
|
16
|
+
LOCAL_ENDPOINT_PLACEHOLDER_API_KEY,
|
|
17
|
+
LOOPBACK_REQUIRED_MESSAGE,
|
|
18
|
+
createExperimentLiveAuthorization,
|
|
19
|
+
formatExperimentRunSummary,
|
|
20
|
+
reportExperimentResult,
|
|
21
|
+
runExperimentFile,
|
|
22
|
+
validateExperimentFile,
|
|
23
|
+
} from '../index';
|
|
24
|
+
import {
|
|
25
|
+
FIXTURE_WORKFLOW,
|
|
26
|
+
LIVE_CONTROLS,
|
|
27
|
+
LOOPBACK_BASE_URL,
|
|
28
|
+
sha256,
|
|
29
|
+
writeExperimentFixture,
|
|
30
|
+
} from './experiment-fixtures';
|
|
31
|
+
|
|
32
|
+
// Tests here run up to a dozen real scenario attempts and exceeded the 5 s default with machine
|
|
33
|
+
// load ~31. This file-scoped timeout keeps them from failing on time alone.
|
|
34
|
+
setDefaultTimeout(30_000);
|
|
35
|
+
|
|
36
|
+
const originalFetch = globalThis.fetch;
|
|
37
|
+
afterEach(() => {
|
|
38
|
+
globalThis.fetch = originalFetch;
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
/** Replace fetch with a recorder that refuses every request. */
|
|
42
|
+
function forbidNetwork(): { calls: number } {
|
|
43
|
+
const counter = { calls: 0 };
|
|
44
|
+
globalThis.fetch = (() => {
|
|
45
|
+
counter.calls += 1;
|
|
46
|
+
throw new Error('NETWORK-FORBIDDEN');
|
|
47
|
+
}) as unknown as typeof fetch;
|
|
48
|
+
return counter;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function liveOverrides(extra: Record<string, unknown> = {}) {
|
|
52
|
+
return {
|
|
53
|
+
mode: 'live',
|
|
54
|
+
budgets: { max_requests: 20, max_tokens: 10_000 },
|
|
55
|
+
live: LIVE_CONTROLS,
|
|
56
|
+
...extra,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function inMemoryClient(text: string, calls: string[]): (target: { id: string }) => ModelClient {
|
|
61
|
+
return (target) => ({
|
|
62
|
+
provider: 'in-memory',
|
|
63
|
+
capabilities: async () => ({
|
|
64
|
+
streaming: false,
|
|
65
|
+
functionCalling: false,
|
|
66
|
+
toolUse: false,
|
|
67
|
+
maxContext: 1000,
|
|
68
|
+
}),
|
|
69
|
+
generate: async (): Promise<GenerateResult> => {
|
|
70
|
+
calls.push(target.id);
|
|
71
|
+
return {
|
|
72
|
+
id: 'in-memory',
|
|
73
|
+
model: 'in-memory',
|
|
74
|
+
text,
|
|
75
|
+
tokens: { prompt: 3, completion: 2, total: 5 },
|
|
76
|
+
usageAvailable: true,
|
|
77
|
+
latencyMs: 1,
|
|
78
|
+
};
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describe('validateExperimentFile', () => {
|
|
84
|
+
test('accepts a two-target local experiment and verifies every source and target', async () => {
|
|
85
|
+
const fixture = await writeExperimentFixture();
|
|
86
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
87
|
+
|
|
88
|
+
expect(validation.valid).toBe(true);
|
|
89
|
+
expect(validation.exitCode).toBe(0);
|
|
90
|
+
expect(validation.root).toBe(fixture.directory);
|
|
91
|
+
expect(validation.experimentId).toBe('two-target-fixture');
|
|
92
|
+
expect(validation.mode).toBe('fixture');
|
|
93
|
+
expect(validation.checks.map((check) => [check.subject, check.id, check.code])).toEqual([
|
|
94
|
+
['manifest', undefined, 'manifest_valid'],
|
|
95
|
+
['task', 'pass', 'source_verified'],
|
|
96
|
+
['task', 'fail', 'source_verified'],
|
|
97
|
+
['target', 'qwen-small', 'loopback_verified'],
|
|
98
|
+
['target', 'qwen-large', 'loopback_verified'],
|
|
99
|
+
]);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('reports a digest mismatch without echoing file contents', async () => {
|
|
103
|
+
const fixture = await writeExperimentFixture({ digestOverride: { pass: 'f'.repeat(64) } });
|
|
104
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
105
|
+
|
|
106
|
+
expect(validation.valid).toBe(false);
|
|
107
|
+
expect(validation.exitCode).toBe(2);
|
|
108
|
+
const check = validation.checks.find((item) => item.id === 'pass');
|
|
109
|
+
expect(check?.status).toBe('error');
|
|
110
|
+
expect(check?.code).toBe('source_digest_mismatch');
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test('reports a missing task source', async () => {
|
|
114
|
+
const fixture = await writeExperimentFixture({ omitSources: ['tasks/fail.yaml'] });
|
|
115
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
116
|
+
|
|
117
|
+
expect(validation.exitCode).toBe(2);
|
|
118
|
+
expect(validation.checks.find((item) => item.id === 'fail')?.code).toBe('source_missing');
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('refuses cost-not-applicable targets off loopback and never echoes the URL', async () => {
|
|
122
|
+
const fixture = await writeExperimentFixture({
|
|
123
|
+
target: (target) =>
|
|
124
|
+
target.id === 'qwen-small'
|
|
125
|
+
? { ...target, settings: { base_url: 'https://user:pw@models.example.com/v1' } }
|
|
126
|
+
: { ...target, settings: {} },
|
|
127
|
+
});
|
|
128
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
129
|
+
|
|
130
|
+
expect(validation.valid).toBe(false);
|
|
131
|
+
const targets = validation.checks.filter((check) => check.subject === 'target');
|
|
132
|
+
expect(targets.map((check) => check.code)).toEqual([
|
|
133
|
+
'cost_not_applicable_requires_loopback',
|
|
134
|
+
'cost_not_applicable_requires_loopback',
|
|
135
|
+
]);
|
|
136
|
+
for (const check of targets) expect(check.message).toContain(LOOPBACK_REQUIRED_MESSAGE);
|
|
137
|
+
expect(JSON.stringify(validation.checks)).not.toContain('models.example.com');
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test('rejects an unknown reasoning_effort target setting', async () => {
|
|
141
|
+
const fixture = await writeExperimentFixture({
|
|
142
|
+
target: (target) =>
|
|
143
|
+
target.id === 'qwen-small'
|
|
144
|
+
? {
|
|
145
|
+
...target,
|
|
146
|
+
settings: { ...(target.settings as object), reasoning_effort: 'maximum' },
|
|
147
|
+
}
|
|
148
|
+
: target,
|
|
149
|
+
});
|
|
150
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
151
|
+
|
|
152
|
+
expect(validation.valid).toBe(false);
|
|
153
|
+
expect(validation.checks.find((check) => check.id === 'qwen-small')?.code).toBe(
|
|
154
|
+
'invalid_reasoning_effort'
|
|
155
|
+
);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test('accepts in-range target generation settings and lists them', async () => {
|
|
159
|
+
const fixture = await writeExperimentFixture({
|
|
160
|
+
target: (target) =>
|
|
161
|
+
target.id === 'qwen-large'
|
|
162
|
+
? {
|
|
163
|
+
...target,
|
|
164
|
+
settings: {
|
|
165
|
+
base_url: LOOPBACK_BASE_URL,
|
|
166
|
+
reasoning_effort: 'medium',
|
|
167
|
+
temperature: 1,
|
|
168
|
+
top_p: 0.95,
|
|
169
|
+
presence_penalty: 1.5,
|
|
170
|
+
max_output_tokens: 32768,
|
|
171
|
+
},
|
|
172
|
+
}
|
|
173
|
+
: target,
|
|
174
|
+
});
|
|
175
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
176
|
+
|
|
177
|
+
expect(validation.valid).toBe(true);
|
|
178
|
+
expect(validation.checks.filter((check) => check.code === 'generation_settings')).toEqual([
|
|
179
|
+
{
|
|
180
|
+
subject: 'target',
|
|
181
|
+
id: 'qwen-large',
|
|
182
|
+
status: 'ok',
|
|
183
|
+
code: 'generation_settings',
|
|
184
|
+
message:
|
|
185
|
+
'temperature 1, top_p 0.95, presence_penalty 1.5, max_output_tokens 32768 replace task values for this target',
|
|
186
|
+
},
|
|
187
|
+
]);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test.each([
|
|
191
|
+
[
|
|
192
|
+
{ temperature: 2.5 },
|
|
193
|
+
'invalid_generation_setting',
|
|
194
|
+
'settings.temperature must be a number from 0 to 2',
|
|
195
|
+
],
|
|
196
|
+
[{ top_p: -0.1 }, 'invalid_generation_setting', 'settings.top_p must be a number from 0 to 1'],
|
|
197
|
+
[{ top_k: 0 }, 'invalid_generation_setting', 'settings.top_k must be an integer of at least 1'],
|
|
198
|
+
[{ min_p: 2 }, 'invalid_generation_setting', 'settings.min_p must be a number from 0 to 1'],
|
|
199
|
+
[
|
|
200
|
+
{ presence_penalty: 3 },
|
|
201
|
+
'invalid_generation_setting',
|
|
202
|
+
'settings.presence_penalty must be a number from -2 to 2',
|
|
203
|
+
],
|
|
204
|
+
[
|
|
205
|
+
{ max_output_tokens: 200000 },
|
|
206
|
+
'invalid_generation_setting',
|
|
207
|
+
'settings.max_output_tokens must be an integer from 1 to 131072',
|
|
208
|
+
],
|
|
209
|
+
[
|
|
210
|
+
{ max_tokens: 4096 },
|
|
211
|
+
'unknown_generation_setting',
|
|
212
|
+
'settings.max_tokens is not supported; use max_output_tokens',
|
|
213
|
+
],
|
|
214
|
+
[
|
|
215
|
+
{ num_predict: 512 },
|
|
216
|
+
'unknown_generation_setting',
|
|
217
|
+
'settings.num_predict is not supported; use max_output_tokens',
|
|
218
|
+
],
|
|
219
|
+
[{ seed: 1 }, 'unknown_generation_setting', 'declare the experiment seed'],
|
|
220
|
+
[
|
|
221
|
+
{ frequency_penalty: 0.5 },
|
|
222
|
+
'unknown_generation_setting',
|
|
223
|
+
'not a supported generation setting',
|
|
224
|
+
],
|
|
225
|
+
[{ top_k: 20 }, 'generation_setting_unsupported', "Ollama's /v1 endpoint) have no top_k field"],
|
|
226
|
+
[{ min_p: 0 }, 'generation_setting_unsupported', 'Modelfile PARAMETER min_p'],
|
|
227
|
+
])('rejects target settings %p with %s', async (extra, code, message) => {
|
|
228
|
+
const fixture = await writeExperimentFixture({
|
|
229
|
+
target: (target) =>
|
|
230
|
+
target.id === 'qwen-small'
|
|
231
|
+
? { ...target, settings: { base_url: LOOPBACK_BASE_URL, ...extra } }
|
|
232
|
+
: target,
|
|
233
|
+
});
|
|
234
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
235
|
+
|
|
236
|
+
expect(validation.valid).toBe(false);
|
|
237
|
+
expect(validation.exitCode).toBe(2);
|
|
238
|
+
const errors = validation.checks.filter((check) => check.status === 'error');
|
|
239
|
+
expect(errors).toHaveLength(1);
|
|
240
|
+
expect(errors[0]).toMatchObject({ subject: 'target', id: 'qwen-small', code });
|
|
241
|
+
expect(errors[0].message).toContain(message);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
test('rejects presence_penalty for a provider whose adapter does not send it', async () => {
|
|
245
|
+
const fixture = await writeExperimentFixture({
|
|
246
|
+
target: (target) =>
|
|
247
|
+
target.id === 'qwen-small'
|
|
248
|
+
? {
|
|
249
|
+
...target,
|
|
250
|
+
provider: 'anthropic',
|
|
251
|
+
settings: { base_url: LOOPBACK_BASE_URL, presence_penalty: 1 },
|
|
252
|
+
}
|
|
253
|
+
: target,
|
|
254
|
+
});
|
|
255
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
256
|
+
|
|
257
|
+
expect(validation.valid).toBe(false);
|
|
258
|
+
expect(validation.checks.find((check) => check.status === 'error')).toMatchObject({
|
|
259
|
+
code: 'generation_setting_unsupported',
|
|
260
|
+
message: 'settings.presence_penalty is not sent by the anthropic adapter; remove it',
|
|
261
|
+
});
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test('warns about a non-generation setting that is never sent', async () => {
|
|
265
|
+
const fixture = await writeExperimentFixture({
|
|
266
|
+
target: (target) =>
|
|
267
|
+
target.id === 'qwen-small'
|
|
268
|
+
? { ...target, settings: { base_url: LOOPBACK_BASE_URL, quantization: 'q4_K_M' } }
|
|
269
|
+
: target,
|
|
270
|
+
});
|
|
271
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
272
|
+
|
|
273
|
+
expect(validation.valid).toBe(true);
|
|
274
|
+
expect(validation.checks.find((check) => check.status === 'warning')).toMatchObject({
|
|
275
|
+
code: 'target_setting_unused',
|
|
276
|
+
message: 'settings.quantization is recorded with the target but not sent to the model',
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
test('distinguishes an unreadable file (1) from an invalid manifest (2)', async () => {
|
|
281
|
+
const fixture = await writeExperimentFixture({ overrides: { concurrency: 9 } });
|
|
282
|
+
const missing = await validateExperimentFile(path.join(fixture.directory, 'absent.yaml'));
|
|
283
|
+
const invalid = await validateExperimentFile(fixture.file);
|
|
284
|
+
|
|
285
|
+
expect(missing.exitCode).toBe(1);
|
|
286
|
+
expect(missing.checks[0]?.code).toBe('file_unreadable');
|
|
287
|
+
expect(invalid.exitCode).toBe(2);
|
|
288
|
+
expect(invalid.checks.some((check) => check.message.startsWith('concurrency:'))).toBe(true);
|
|
289
|
+
});
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
describe('runExperimentFile (fixture mode)', () => {
|
|
293
|
+
test('runs a two-target experiment end to end with zero network calls', async () => {
|
|
294
|
+
const network = forbidNetwork();
|
|
295
|
+
const fixture = await writeExperimentFixture();
|
|
296
|
+
const outputDir = path.join(fixture.directory, 'out');
|
|
297
|
+
|
|
298
|
+
const outcome = await runExperimentFile({ experiment: fixture.file, outputDir });
|
|
299
|
+
|
|
300
|
+
expect(network.calls).toBe(0);
|
|
301
|
+
expect(outcome.status).toBe('complete');
|
|
302
|
+
expect(outcome.exitCode).toBe(0);
|
|
303
|
+
const result = outcome.result as ExperimentRunResult;
|
|
304
|
+
expect(result.mode).toBe('fixture');
|
|
305
|
+
expect(result.summaries.overall).toMatchObject({ planned: 4, valid: 4, passed: 2 });
|
|
306
|
+
expect(result.summaries.overall.task_failed).toBe(2);
|
|
307
|
+
|
|
308
|
+
const saved = JSON.parse(await readFile(path.join(outputDir, 'result.json'), 'utf8'));
|
|
309
|
+
expect(saved).toEqual(JSON.parse(JSON.stringify(result)));
|
|
310
|
+
expect(outcome.resultPath).toBe(path.join(outputDir, 'result.json'));
|
|
311
|
+
|
|
312
|
+
expect(outcome.evidence).toHaveLength(4);
|
|
313
|
+
for (const record of outcome.evidence) {
|
|
314
|
+
const bytes = await readFile(record.path);
|
|
315
|
+
expect(createHash('sha256').update(bytes).digest('hex')).toBe(record.sha256);
|
|
316
|
+
expect(path.basename(record.path)).toBe(`${record.sha256}.json`);
|
|
317
|
+
}
|
|
318
|
+
const attemptDigests = result.results.flatMap((coordinate) =>
|
|
319
|
+
coordinate.attempts.map((attempt) =>
|
|
320
|
+
attempt.evidence.availability === 'available' ? attempt.evidence.artifact.digest : null
|
|
321
|
+
)
|
|
322
|
+
);
|
|
323
|
+
expect(attemptDigests.sort()).toEqual(outcome.evidence.map((item) => item.sha256).sort());
|
|
324
|
+
expect((await stat(path.join(outputDir, 'result.json'))).mode & 0o777).toBe(0o600);
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
test('defaults the output directory under artemis-output/experiments/<id>/', async () => {
|
|
328
|
+
forbidNetwork();
|
|
329
|
+
const fixture = await writeExperimentFixture();
|
|
330
|
+
const outcome = await runExperimentFile({ experiment: fixture.file, cwd: fixture.directory });
|
|
331
|
+
|
|
332
|
+
const base = path.join(
|
|
333
|
+
fixture.directory,
|
|
334
|
+
'artemis-output',
|
|
335
|
+
'experiments',
|
|
336
|
+
'two-target-fixture'
|
|
337
|
+
);
|
|
338
|
+
expect(path.dirname(outcome.outputDir as string)).toBe(base);
|
|
339
|
+
expect((await readdir(outcome.outputDir as string)).sort()).toEqual([
|
|
340
|
+
'evidence',
|
|
341
|
+
'result.json',
|
|
342
|
+
]);
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
test('refuses to run an invalid experiment and creates no output', async () => {
|
|
346
|
+
const fixture = await writeExperimentFixture({ digestOverride: { fail: '0'.repeat(64) } });
|
|
347
|
+
const outputDir = path.join(fixture.directory, 'out');
|
|
348
|
+
const outcome = await runExperimentFile({ experiment: fixture.file, outputDir });
|
|
349
|
+
|
|
350
|
+
expect(outcome.status).toBe('invalid');
|
|
351
|
+
expect(outcome.exitCode).toBe(EXPERIMENT_EXIT_CODES.invalid);
|
|
352
|
+
expect(outcome.result).toBeUndefined();
|
|
353
|
+
await expect(stat(outputDir)).rejects.toThrow();
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
test('refuses to reuse a non-empty output directory', async () => {
|
|
357
|
+
forbidNetwork();
|
|
358
|
+
const fixture = await writeExperimentFixture();
|
|
359
|
+
const error = await runExperimentFile({
|
|
360
|
+
experiment: fixture.file,
|
|
361
|
+
outputDir: fixture.directory,
|
|
362
|
+
}).catch((caught) => caught);
|
|
363
|
+
|
|
364
|
+
expect(error).toBeInstanceOf(ExperimentSurfaceError);
|
|
365
|
+
expect(error.exitCode).toBe(1);
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
test('exit code 5 when declared exclusions leave coverage incomplete', async () => {
|
|
369
|
+
forbidNetwork();
|
|
370
|
+
const fixture = await writeExperimentFixture({
|
|
371
|
+
overrides: {
|
|
372
|
+
exclusions: [{ id: 'skip-large', reason: 'declared for test', target_id: 'qwen-large' }],
|
|
373
|
+
},
|
|
374
|
+
});
|
|
375
|
+
const outcome = await runExperimentFile({
|
|
376
|
+
experiment: fixture.file,
|
|
377
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
expect(outcome.status).toBe('coverageIncomplete');
|
|
381
|
+
expect(outcome.exitCode).toBe(5);
|
|
382
|
+
expect(outcome.result?.completion.valid_measurement_coverage_complete).toBe(false);
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
test('exit code 4 when the request budget stops execution early', async () => {
|
|
386
|
+
forbidNetwork();
|
|
387
|
+
const fixture = await writeExperimentFixture({ overrides: { budgets: { max_requests: 1 } } });
|
|
388
|
+
const outcome = await runExperimentFile({
|
|
389
|
+
experiment: fixture.file,
|
|
390
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
expect(outcome.status).toBe('stopped');
|
|
394
|
+
expect(outcome.exitCode).toBe(4);
|
|
395
|
+
expect(outcome.result?.stop_reason).toBe('request_budget_exhausted');
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
test('summary labels local targets and makes no ranking claim', async () => {
|
|
399
|
+
forbidNetwork();
|
|
400
|
+
const fixture = await writeExperimentFixture();
|
|
401
|
+
const outcome = await runExperimentFile({
|
|
402
|
+
experiment: fixture.file,
|
|
403
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
404
|
+
});
|
|
405
|
+
const summary = formatExperimentRunSummary(outcome.result as ExperimentRunResult);
|
|
406
|
+
|
|
407
|
+
expect(summary).toContain('fixture mode');
|
|
408
|
+
expect(summary).toContain('open-weight, run locally via ollama; cost not applicable');
|
|
409
|
+
expect(summary).toContain('published by Qwen (Alibaba Cloud)');
|
|
410
|
+
expect(summary).toContain('valid 2/2 planned; passed 1 and task-failed 1 of 2 valid');
|
|
411
|
+
expect(summary).toContain('Stop reason: none');
|
|
412
|
+
expect(summary).toContain('cost not applicable (local targets only)');
|
|
413
|
+
expect(summary).toContain('No ranking or winner is implied');
|
|
414
|
+
expect(summary.indexOf('qwen-small')).toBeLessThan(summary.indexOf('qwen-large'));
|
|
415
|
+
});
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
/** A mixed fixture whose workflow task source is replaced by `content` (digest kept valid). */
|
|
419
|
+
async function writeWorkflowFixture(
|
|
420
|
+
content: string,
|
|
421
|
+
overrides: Record<string, unknown> = {}
|
|
422
|
+
): Promise<Awaited<ReturnType<typeof writeExperimentFixture>>> {
|
|
423
|
+
const fixture = await writeExperimentFixture({ workflow: true, overrides });
|
|
424
|
+
await writeFile(path.join(fixture.directory, 'tasks/workflow.yaml'), content);
|
|
425
|
+
const tasks = fixture.manifest.tasks as Array<{ source: { artifact: { digest: string } } }>;
|
|
426
|
+
tasks[1].source.artifact.digest = sha256(content);
|
|
427
|
+
await writeFile(fixture.file, JSON.stringify(fixture.manifest, null, 2));
|
|
428
|
+
return fixture;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
describe('agent_workflow tasks', () => {
|
|
432
|
+
test('validation parses a workflow source and routes nothing to the network', async () => {
|
|
433
|
+
const network = forbidNetwork();
|
|
434
|
+
const fixture = await writeExperimentFixture({ workflow: true });
|
|
435
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
436
|
+
|
|
437
|
+
expect(network.calls).toBe(0);
|
|
438
|
+
expect(validation.valid).toBe(true);
|
|
439
|
+
expect(validation.checks).toContainEqual({
|
|
440
|
+
subject: 'task',
|
|
441
|
+
id: 'workflow',
|
|
442
|
+
status: 'ok',
|
|
443
|
+
code: 'source_verified',
|
|
444
|
+
message:
|
|
445
|
+
'tasks/workflow.yaml: digest verified; agent workflow parsed (simulated environment, network denied)',
|
|
446
|
+
});
|
|
447
|
+
expect(validation.checks.some((check) => check.status === 'warning')).toBe(false);
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
test('validation rejects an invalid workflow without echoing its content', async () => {
|
|
451
|
+
const fixture = await writeWorkflowFixture(
|
|
452
|
+
`${FIXTURE_WORKFLOW.replace('network: denied', 'network: allowed')}# secret-marker\n`
|
|
453
|
+
);
|
|
454
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
455
|
+
|
|
456
|
+
expect(validation.valid).toBe(false);
|
|
457
|
+
expect(validation.exitCode).toBe(2);
|
|
458
|
+
const check = validation.checks.find((item) => item.id === 'workflow');
|
|
459
|
+
expect(check).toMatchObject({ status: 'error', code: 'workflow_invalid' });
|
|
460
|
+
expect(JSON.stringify(validation.checks)).not.toContain('secret-marker');
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
test('validation warns about semantic outcomes but not seeds or sampling for workflows', async () => {
|
|
464
|
+
const semantic = FIXTURE_WORKFLOW.replace(
|
|
465
|
+
'evidence:',
|
|
466
|
+
` semantic:
|
|
467
|
+
- type: llm_judge
|
|
468
|
+
rubric: Replies.
|
|
469
|
+
mode: strict_assurance
|
|
470
|
+
evidence:`
|
|
471
|
+
);
|
|
472
|
+
const fixture = await writeWorkflowFixture(semantic, {
|
|
473
|
+
seed: { value: 7, strategy: 'fixed', require_support: false },
|
|
474
|
+
});
|
|
475
|
+
const manifest = fixture.manifest as { targets: Array<Record<string, unknown>> };
|
|
476
|
+
manifest.targets[0].settings = { base_url: LOOPBACK_BASE_URL, top_p: 0.8 };
|
|
477
|
+
await writeFile(fixture.file, JSON.stringify(manifest, null, 2));
|
|
478
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
479
|
+
|
|
480
|
+
expect(validation.valid).toBe(true);
|
|
481
|
+
const warnings = validation.checks.filter((check) => check.status === 'warning');
|
|
482
|
+
expect(warnings.map((check) => [check.code, check.id])).toEqual([
|
|
483
|
+
['semantic_judge_unavailable', 'workflow'],
|
|
484
|
+
]);
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
test('runs one scenario and one workflow task on two targets with zero network calls', async () => {
|
|
488
|
+
const network = forbidNetwork();
|
|
489
|
+
const fixture = await writeExperimentFixture({ workflow: true });
|
|
490
|
+
const outputDir = path.join(fixture.directory, 'out');
|
|
491
|
+
|
|
492
|
+
const outcome = await runExperimentFile({ experiment: fixture.file, outputDir });
|
|
493
|
+
|
|
494
|
+
expect(network.calls).toBe(0);
|
|
495
|
+
expect(outcome.status).toBe('complete');
|
|
496
|
+
expect(outcome.exitCode).toBe(0);
|
|
497
|
+
const result = outcome.result as ExperimentRunResult;
|
|
498
|
+
expect(result.summaries.overall).toMatchObject({ planned: 4, valid: 4, passed: 4 });
|
|
499
|
+
const kinds = result.results.map((item) => [
|
|
500
|
+
item.coordinate.task_kind,
|
|
501
|
+
item.coordinate.target_id,
|
|
502
|
+
item.status,
|
|
503
|
+
]);
|
|
504
|
+
expect(kinds).toEqual([
|
|
505
|
+
['scenario_evaluation', 'qwen-small', 'passed'],
|
|
506
|
+
['scenario_evaluation', 'qwen-large', 'passed'],
|
|
507
|
+
['agent_workflow', 'qwen-small', 'passed'],
|
|
508
|
+
['agent_workflow', 'qwen-large', 'passed'],
|
|
509
|
+
]);
|
|
510
|
+
|
|
511
|
+
expect(outcome.evidence).toHaveLength(4);
|
|
512
|
+
for (const coordinate of result.results) {
|
|
513
|
+
const [attempt] = coordinate.attempts;
|
|
514
|
+
expect(attempt.evidence).toMatchObject({
|
|
515
|
+
kind: coordinate.coordinate.task_kind,
|
|
516
|
+
availability: 'available',
|
|
517
|
+
});
|
|
518
|
+
const digest = attempt.evidence.availability === 'available' && attempt.evidence.artifact;
|
|
519
|
+
const file = outcome.evidence.find(
|
|
520
|
+
(item) => item.coordinate_id === coordinate.coordinate.coordinate_id
|
|
521
|
+
);
|
|
522
|
+
expect(file?.sha256).toBe((digest as { digest: string }).digest);
|
|
523
|
+
const bytes = await readFile(file?.path as string);
|
|
524
|
+
expect(createHash('sha256').update(bytes).digest('hex')).toBe(file?.sha256 as string);
|
|
525
|
+
const saved = JSON.parse(bytes.toString('utf8'));
|
|
526
|
+
if (coordinate.coordinate.task_kind === 'agent_workflow') {
|
|
527
|
+
expect(saved).toMatchObject({
|
|
528
|
+
engine: 'native',
|
|
529
|
+
execution: 'completed',
|
|
530
|
+
policy: 'passed',
|
|
531
|
+
taskVerification: 'passed',
|
|
532
|
+
environment: 'simulated',
|
|
533
|
+
});
|
|
534
|
+
expect(saved.configuration.model.display).toBe(
|
|
535
|
+
coordinate.coordinate.target_id === 'qwen-small' ? 'qwen3.5:4b' : 'qwen3.5:9b'
|
|
536
|
+
);
|
|
537
|
+
expect(bytes.toString('utf8')).not.toContain('Fixture brief');
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
test('a workflow task whose state fixture is missing is recorded invalid, not run', async () => {
|
|
543
|
+
forbidNetwork();
|
|
544
|
+
const declaredState = ' initial_state:\n documents:\n brief: Fixture brief.\n';
|
|
545
|
+
expect(FIXTURE_WORKFLOW).toContain(declaredState);
|
|
546
|
+
const fixture = await writeWorkflowFixture(
|
|
547
|
+
FIXTURE_WORKFLOW.replace(declaredState, ' initial_state: missing/state.json\n')
|
|
548
|
+
);
|
|
549
|
+
const outcome = await runExperimentFile({
|
|
550
|
+
experiment: fixture.file,
|
|
551
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
552
|
+
});
|
|
553
|
+
const result = outcome.result as ExperimentRunResult;
|
|
554
|
+
const workflow = result.results.filter(
|
|
555
|
+
(item) => item.coordinate.task_kind === 'agent_workflow'
|
|
556
|
+
);
|
|
557
|
+
expect(workflow.map((item) => item.status)).toEqual(['invalid', 'invalid']);
|
|
558
|
+
expect(workflow[0].attempts[0]).toMatchObject({
|
|
559
|
+
error_code: 'workflow_fixture_invalid',
|
|
560
|
+
usage: { requests: 0, tokens: 0 },
|
|
561
|
+
});
|
|
562
|
+
expect(outcome.evidence).toHaveLength(2);
|
|
563
|
+
});
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
describe('runExperimentFile (live mode)', () => {
|
|
567
|
+
test('refuses without approval before creating any client', async () => {
|
|
568
|
+
const calls: string[] = [];
|
|
569
|
+
let created = 0;
|
|
570
|
+
const fixture = await writeExperimentFixture({ overrides: liveOverrides() });
|
|
571
|
+
const outputDir = path.join(fixture.directory, 'out');
|
|
572
|
+
const outcome = await runExperimentFile({
|
|
573
|
+
experiment: fixture.file,
|
|
574
|
+
outputDir,
|
|
575
|
+
createClient: (target) => {
|
|
576
|
+
created += 1;
|
|
577
|
+
return inMemoryClient('x', calls)(target);
|
|
578
|
+
},
|
|
579
|
+
});
|
|
580
|
+
|
|
581
|
+
expect(outcome.status).toBe('unauthorized');
|
|
582
|
+
expect(outcome.exitCode).toBe(3);
|
|
583
|
+
expect(created).toBe(0);
|
|
584
|
+
expect(calls).toEqual([]);
|
|
585
|
+
await expect(stat(outputDir)).rejects.toThrow();
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
test('refuses a malformed approval object', async () => {
|
|
589
|
+
const fixture = await writeExperimentFixture({ overrides: liveOverrides() });
|
|
590
|
+
const outcome = await runExperimentFile({
|
|
591
|
+
experiment: fixture.file,
|
|
592
|
+
liveAuthorization: {
|
|
593
|
+
approved: true,
|
|
594
|
+
decision_id: 'Not Valid',
|
|
595
|
+
decided_at: 'yesterday',
|
|
596
|
+
approved_by: '',
|
|
597
|
+
reason: '',
|
|
598
|
+
},
|
|
599
|
+
createClient: () => {
|
|
600
|
+
throw new Error('must not be called');
|
|
601
|
+
},
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
expect(outcome.exitCode).toBe(3);
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
test('runs with approval and records the authorization in the result', async () => {
|
|
608
|
+
const calls: string[] = [];
|
|
609
|
+
const fixture = await writeExperimentFixture({ overrides: liveOverrides() });
|
|
610
|
+
const authorization = createExperimentLiveAuthorization({
|
|
611
|
+
approvedBy: 'test-operator',
|
|
612
|
+
reason: 'in-memory clients only',
|
|
613
|
+
});
|
|
614
|
+
const outcome = await runExperimentFile({
|
|
615
|
+
experiment: fixture.file,
|
|
616
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
617
|
+
liveAuthorization: authorization,
|
|
618
|
+
createClient: inMemoryClient('Fixture response, Abuja', calls),
|
|
619
|
+
});
|
|
620
|
+
|
|
621
|
+
expect(outcome.exitCode).toBe(0);
|
|
622
|
+
expect(calls).toEqual(['qwen-small', 'qwen-large', 'qwen-small', 'qwen-large']);
|
|
623
|
+
expect(outcome.result?.live_authorization).toEqual(authorization);
|
|
624
|
+
expect(outcome.result?.summaries.overall.passed).toBe(4);
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
test('binds loopback targets with a placeholder key, never the ambient OpenAI key', async () => {
|
|
628
|
+
const seen: { authorization: string | null; model: string }[] = [];
|
|
629
|
+
const server = Bun.serve({
|
|
630
|
+
hostname: '127.0.0.1',
|
|
631
|
+
port: 0,
|
|
632
|
+
async fetch(request) {
|
|
633
|
+
const body = (await request.json()) as { model: string };
|
|
634
|
+
seen.push({ authorization: request.headers.get('authorization'), model: body.model });
|
|
635
|
+
return Response.json({
|
|
636
|
+
id: 'local',
|
|
637
|
+
object: 'chat.completion',
|
|
638
|
+
created: 0,
|
|
639
|
+
model: body.model,
|
|
640
|
+
choices: [
|
|
641
|
+
{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'Abuja' } },
|
|
642
|
+
],
|
|
643
|
+
usage: { prompt_tokens: 4, completion_tokens: 1, total_tokens: 5 },
|
|
644
|
+
});
|
|
645
|
+
},
|
|
646
|
+
});
|
|
647
|
+
const previousKey = process.env.OPENAI_API_KEY;
|
|
648
|
+
process.env.OPENAI_API_KEY = 'sk-ambient-must-not-leak';
|
|
649
|
+
try {
|
|
650
|
+
const baseUrl = `http://127.0.0.1:${server.port}/v1`;
|
|
651
|
+
const fixture = await writeExperimentFixture({
|
|
652
|
+
overrides: liveOverrides(),
|
|
653
|
+
target: (target) => ({ ...target, settings: { base_url: baseUrl } }),
|
|
654
|
+
});
|
|
655
|
+
const outcome = await runExperimentFile({
|
|
656
|
+
experiment: fixture.file,
|
|
657
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
658
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
659
|
+
approvedBy: 'test-operator',
|
|
660
|
+
reason: 'loopback stub server only',
|
|
661
|
+
}),
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
expect(outcome.exitCode).toBe(0);
|
|
665
|
+
expect(seen).toHaveLength(4);
|
|
666
|
+
for (const request of seen) {
|
|
667
|
+
expect(request.authorization).toBe(`Bearer ${LOCAL_ENDPOINT_PLACEHOLDER_API_KEY}`);
|
|
668
|
+
}
|
|
669
|
+
expect(seen.map((request) => request.model).sort()).toEqual([
|
|
670
|
+
'qwen3.5:4b',
|
|
671
|
+
'qwen3.5:4b',
|
|
672
|
+
'qwen3.5:9b',
|
|
673
|
+
'qwen3.5:9b',
|
|
674
|
+
]);
|
|
675
|
+
} finally {
|
|
676
|
+
if (previousKey === undefined) Reflect.deleteProperty(process.env, 'OPENAI_API_KEY');
|
|
677
|
+
else process.env.OPENAI_API_KEY = previousKey;
|
|
678
|
+
server.stop(true);
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
test('raises the adapter HTTP timeout to the per-case timeout for slow loopback targets', async () => {
|
|
682
|
+
// The stub answers after STUB_DELAY_MS; the resolver stands in for an adapter transport
|
|
683
|
+
// default shorter than that, as the OpenAI adapter's 60 s default is for local thinking runs.
|
|
684
|
+
const STUB_DELAY_MS = 600;
|
|
685
|
+
const SHORT_TRANSPORT_TIMEOUT_MS = 150;
|
|
686
|
+
const server = Bun.serve({
|
|
687
|
+
hostname: '127.0.0.1',
|
|
688
|
+
port: 0,
|
|
689
|
+
async fetch(request) {
|
|
690
|
+
const body = (await request.json()) as { model: string };
|
|
691
|
+
await Bun.sleep(STUB_DELAY_MS);
|
|
692
|
+
return Response.json({
|
|
693
|
+
id: 'slow-local',
|
|
694
|
+
object: 'chat.completion',
|
|
695
|
+
created: 0,
|
|
696
|
+
model: body.model,
|
|
697
|
+
choices: [
|
|
698
|
+
{
|
|
699
|
+
index: 0,
|
|
700
|
+
finish_reason: 'stop',
|
|
701
|
+
message: { role: 'assistant', content: 'Fixture response, Abuja' },
|
|
702
|
+
},
|
|
703
|
+
],
|
|
704
|
+
usage: { prompt_tokens: 4, completion_tokens: 3, total_tokens: 7 },
|
|
705
|
+
});
|
|
706
|
+
},
|
|
707
|
+
});
|
|
708
|
+
try {
|
|
709
|
+
const baseUrl = `http://127.0.0.1:${server.port}/v1`;
|
|
710
|
+
const run = async (timeoutMs: number | undefined) => {
|
|
711
|
+
const fixture = await writeExperimentFixture({
|
|
712
|
+
overrides: liveOverrides(),
|
|
713
|
+
target: (target) => ({ ...target, settings: { base_url: baseUrl } }),
|
|
714
|
+
});
|
|
715
|
+
return runExperimentFile({
|
|
716
|
+
experiment: fixture.file,
|
|
717
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
718
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
719
|
+
approvedBy: 'test-operator',
|
|
720
|
+
reason: 'loopback stub server only',
|
|
721
|
+
}),
|
|
722
|
+
resolveTargetConfig: () => ({ timeout: SHORT_TRANSPORT_TIMEOUT_MS }),
|
|
723
|
+
timeoutMs,
|
|
724
|
+
});
|
|
725
|
+
};
|
|
726
|
+
|
|
727
|
+
const withPerCaseTimeout = await run(10_000);
|
|
728
|
+
expect(withPerCaseTimeout.exitCode).toBe(0);
|
|
729
|
+
expect(withPerCaseTimeout.result?.stop_reason).toBeUndefined();
|
|
730
|
+
expect(withPerCaseTimeout.result?.summaries.overall.passed).toBe(4);
|
|
731
|
+
|
|
732
|
+
// Control: without a per-case timeout the transport default stands and aborts the request.
|
|
733
|
+
const withoutPerCaseTimeout = await run(undefined);
|
|
734
|
+
expect(withoutPerCaseTimeout.result?.stop_reason).toBe('usage_unreported');
|
|
735
|
+
expect(withoutPerCaseTimeout.result?.summaries.overall.passed).toBe(0);
|
|
736
|
+
} finally {
|
|
737
|
+
server.stop(true);
|
|
738
|
+
}
|
|
739
|
+
});
|
|
740
|
+
|
|
741
|
+
test('saves the declared reasoning_effort in the evidence of each attempt that sent it', async () => {
|
|
742
|
+
const sent: unknown[] = [];
|
|
743
|
+
const server = Bun.serve({
|
|
744
|
+
hostname: '127.0.0.1',
|
|
745
|
+
port: 0,
|
|
746
|
+
async fetch(request) {
|
|
747
|
+
const body = (await request.json()) as { model: string; reasoning_effort?: string };
|
|
748
|
+
sent.push(body.reasoning_effort);
|
|
749
|
+
return Response.json({
|
|
750
|
+
id: 'local',
|
|
751
|
+
object: 'chat.completion',
|
|
752
|
+
created: 0,
|
|
753
|
+
model: body.model,
|
|
754
|
+
choices: [
|
|
755
|
+
{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'Abuja' } },
|
|
756
|
+
],
|
|
757
|
+
usage: { prompt_tokens: 4, completion_tokens: 1, total_tokens: 5 },
|
|
758
|
+
});
|
|
759
|
+
},
|
|
760
|
+
});
|
|
761
|
+
try {
|
|
762
|
+
const baseUrl = `http://127.0.0.1:${server.port}/v1`;
|
|
763
|
+
const fixture = await writeExperimentFixture({
|
|
764
|
+
overrides: liveOverrides(),
|
|
765
|
+
target: (target) => ({
|
|
766
|
+
...target,
|
|
767
|
+
settings: { base_url: baseUrl, reasoning_effort: 'none' },
|
|
768
|
+
}),
|
|
769
|
+
});
|
|
770
|
+
const outcome = await runExperimentFile({
|
|
771
|
+
experiment: fixture.file,
|
|
772
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
773
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
774
|
+
approvedBy: 'test-operator',
|
|
775
|
+
reason: 'loopback stub server only',
|
|
776
|
+
}),
|
|
777
|
+
});
|
|
778
|
+
|
|
779
|
+
expect(outcome.exitCode).toBe(0);
|
|
780
|
+
expect(sent).toEqual(['none', 'none', 'none', 'none']);
|
|
781
|
+
expect(outcome.evidence).toHaveLength(4);
|
|
782
|
+
for (const file of outcome.evidence) {
|
|
783
|
+
const saved = JSON.parse(await readFile(file.path, 'utf8'));
|
|
784
|
+
expect(saved.resolved_config.reasoning_effort).toBe('none');
|
|
785
|
+
}
|
|
786
|
+
} finally {
|
|
787
|
+
server.stop(true);
|
|
788
|
+
}
|
|
789
|
+
});
|
|
790
|
+
|
|
791
|
+
test('sends each target generation settings and repetition seed, and saves them as evidence', async () => {
|
|
792
|
+
const bodies: Record<string, unknown>[] = [];
|
|
793
|
+
const server = Bun.serve({
|
|
794
|
+
hostname: '127.0.0.1',
|
|
795
|
+
port: 0,
|
|
796
|
+
async fetch(request) {
|
|
797
|
+
const body = (await request.json()) as Record<string, unknown>;
|
|
798
|
+
bodies.push(body);
|
|
799
|
+
return Response.json({
|
|
800
|
+
id: 'local',
|
|
801
|
+
object: 'chat.completion',
|
|
802
|
+
created: 0,
|
|
803
|
+
model: body.model,
|
|
804
|
+
choices: [
|
|
805
|
+
{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'Abuja' } },
|
|
806
|
+
],
|
|
807
|
+
usage: { prompt_tokens: 4, completion_tokens: 1, total_tokens: 5 },
|
|
808
|
+
});
|
|
809
|
+
},
|
|
810
|
+
});
|
|
811
|
+
try {
|
|
812
|
+
const baseUrl = `http://127.0.0.1:${server.port}/v1`;
|
|
813
|
+
const thinking = {
|
|
814
|
+
reasoning_effort: 'medium',
|
|
815
|
+
temperature: 1,
|
|
816
|
+
top_p: 0.95,
|
|
817
|
+
presence_penalty: 1.5,
|
|
818
|
+
max_output_tokens: 32768,
|
|
819
|
+
};
|
|
820
|
+
const fixture = await writeExperimentFixture({
|
|
821
|
+
overrides: liveOverrides({
|
|
822
|
+
repetitions: 3,
|
|
823
|
+
seed: { value: 40, strategy: 'increment_by_repetition', require_support: false },
|
|
824
|
+
}),
|
|
825
|
+
target: (target) => ({
|
|
826
|
+
...target,
|
|
827
|
+
settings:
|
|
828
|
+
target.id === 'qwen-large'
|
|
829
|
+
? { base_url: baseUrl, ...thinking }
|
|
830
|
+
: { base_url: baseUrl, reasoning_effort: 'none' },
|
|
831
|
+
}),
|
|
832
|
+
});
|
|
833
|
+
const outcome = await runExperimentFile({
|
|
834
|
+
experiment: fixture.file,
|
|
835
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
836
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
837
|
+
approvedBy: 'test-operator',
|
|
838
|
+
reason: 'loopback stub server only',
|
|
839
|
+
}),
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
expect(outcome.status).not.toBe('invalid');
|
|
843
|
+
expect(bodies).toHaveLength(12);
|
|
844
|
+
const sampling = (body: Record<string, unknown>) => ({
|
|
845
|
+
temperature: body.temperature,
|
|
846
|
+
top_p: body.top_p,
|
|
847
|
+
top_k: body.top_k,
|
|
848
|
+
min_p: body.min_p,
|
|
849
|
+
presence_penalty: body.presence_penalty,
|
|
850
|
+
max_tokens: body.max_tokens,
|
|
851
|
+
reasoning_effort: body.reasoning_effort,
|
|
852
|
+
});
|
|
853
|
+
const large = bodies.filter((body) => body.model === 'qwen3.5:9b');
|
|
854
|
+
const small = bodies.filter((body) => body.model === 'qwen3.5:4b');
|
|
855
|
+
for (const body of large) {
|
|
856
|
+
expect(sampling(body)).toEqual({
|
|
857
|
+
temperature: 1,
|
|
858
|
+
top_p: 0.95,
|
|
859
|
+
top_k: undefined,
|
|
860
|
+
min_p: undefined,
|
|
861
|
+
presence_penalty: 1.5,
|
|
862
|
+
max_tokens: 32768,
|
|
863
|
+
reasoning_effort: 'medium',
|
|
864
|
+
});
|
|
865
|
+
}
|
|
866
|
+
// The fixture tasks declare no temperature or max tokens, so nothing else is sent.
|
|
867
|
+
for (const body of small) {
|
|
868
|
+
expect(sampling(body)).toEqual({
|
|
869
|
+
temperature: undefined,
|
|
870
|
+
top_p: undefined,
|
|
871
|
+
top_k: undefined,
|
|
872
|
+
min_p: undefined,
|
|
873
|
+
presence_penalty: undefined,
|
|
874
|
+
max_tokens: undefined,
|
|
875
|
+
reasoning_effort: 'none',
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
for (const group of [large, small]) {
|
|
879
|
+
expect(group.map((body) => body.seed).sort()).toEqual([40, 40, 41, 41, 42, 42]);
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
expect(outcome.evidence).toHaveLength(12);
|
|
883
|
+
for (const file of outcome.evidence) {
|
|
884
|
+
const saved = JSON.parse(await readFile(file.path, 'utf8'));
|
|
885
|
+
const repetition = saved.attempt_evidence.repetition.index;
|
|
886
|
+
if (saved.config.model === 'qwen3.5:9b') {
|
|
887
|
+
expect(saved.resolved_config).toMatchObject({
|
|
888
|
+
temperature: 1,
|
|
889
|
+
top_p: 0.95,
|
|
890
|
+
presence_penalty: 1.5,
|
|
891
|
+
max_tokens: 32768,
|
|
892
|
+
seed: 39 + repetition,
|
|
893
|
+
reasoning_effort: 'medium',
|
|
894
|
+
});
|
|
895
|
+
} else {
|
|
896
|
+
expect(saved.resolved_config).toEqual({
|
|
897
|
+
provider: 'openai',
|
|
898
|
+
model: 'qwen3.5:4b',
|
|
899
|
+
seed: 39 + repetition,
|
|
900
|
+
reasoning_effort: 'none',
|
|
901
|
+
source: {},
|
|
902
|
+
});
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
} finally {
|
|
906
|
+
server.stop(true);
|
|
907
|
+
}
|
|
908
|
+
});
|
|
909
|
+
|
|
910
|
+
test('a seeded workflow task measures with each target sampling and repetition seed', async () => {
|
|
911
|
+
const bodies: Record<string, unknown>[] = [];
|
|
912
|
+
const server = Bun.serve({
|
|
913
|
+
hostname: '127.0.0.1',
|
|
914
|
+
port: 0,
|
|
915
|
+
async fetch(request) {
|
|
916
|
+
const body = (await request.json()) as Record<string, unknown>;
|
|
917
|
+
bodies.push(body);
|
|
918
|
+
return Response.json({
|
|
919
|
+
id: 'local',
|
|
920
|
+
object: 'chat.completion',
|
|
921
|
+
created: 0,
|
|
922
|
+
model: body.model,
|
|
923
|
+
choices: [
|
|
924
|
+
{
|
|
925
|
+
index: 0,
|
|
926
|
+
finish_reason: 'stop',
|
|
927
|
+
message: { role: 'assistant', content: 'Fixture response, Abuja' },
|
|
928
|
+
},
|
|
929
|
+
],
|
|
930
|
+
usage: { prompt_tokens: 4, completion_tokens: 1, total_tokens: 5 },
|
|
931
|
+
});
|
|
932
|
+
},
|
|
933
|
+
});
|
|
934
|
+
try {
|
|
935
|
+
const baseUrl = `http://127.0.0.1:${server.port}/v1`;
|
|
936
|
+
// Qwen's recommended sampling per mode, as the benchmark protocol declares it.
|
|
937
|
+
const sampling: Record<string, Record<string, unknown>> = {
|
|
938
|
+
'qwen3.5:4b': {
|
|
939
|
+
temperature: 0.7,
|
|
940
|
+
top_p: 0.8,
|
|
941
|
+
presence_penalty: 1.5,
|
|
942
|
+
max_output_tokens: 4096,
|
|
943
|
+
reasoning_effort: 'none',
|
|
944
|
+
},
|
|
945
|
+
'qwen3.5:9b': {
|
|
946
|
+
temperature: 1,
|
|
947
|
+
top_p: 0.95,
|
|
948
|
+
presence_penalty: 1.5,
|
|
949
|
+
max_output_tokens: 32768,
|
|
950
|
+
reasoning_effort: 'medium',
|
|
951
|
+
},
|
|
952
|
+
};
|
|
953
|
+
const fixture = await writeExperimentFixture({
|
|
954
|
+
workflow: true,
|
|
955
|
+
overrides: liveOverrides({
|
|
956
|
+
repetitions: 3,
|
|
957
|
+
budgets: { max_requests: 40, max_tokens: 10_000 },
|
|
958
|
+
seed: { value: 40, strategy: 'increment_by_repetition', require_support: false },
|
|
959
|
+
}),
|
|
960
|
+
target: (target) => ({
|
|
961
|
+
...target,
|
|
962
|
+
capabilities: ['text', 'tools'],
|
|
963
|
+
settings: { base_url: baseUrl, ...sampling[target.model as string] },
|
|
964
|
+
}),
|
|
965
|
+
});
|
|
966
|
+
const validation = await validateExperimentFile(fixture.file);
|
|
967
|
+
expect(validation.checks.filter((check) => check.status === 'warning')).toEqual([]);
|
|
968
|
+
|
|
969
|
+
const outcome = await runExperimentFile({
|
|
970
|
+
experiment: fixture.file,
|
|
971
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
972
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
973
|
+
approvedBy: 'test-operator',
|
|
974
|
+
reason: 'loopback stub server only',
|
|
975
|
+
}),
|
|
976
|
+
});
|
|
977
|
+
|
|
978
|
+
expect(outcome.exitCode).toBe(0);
|
|
979
|
+
const result = outcome.result as ExperimentRunResult;
|
|
980
|
+
expect(result.summaries.overall).toMatchObject({ planned: 12, valid: 12, passed: 12 });
|
|
981
|
+
expect(result.results.every((item) => item.attempts[0].error_code === undefined)).toBe(true);
|
|
982
|
+
|
|
983
|
+
// One request per coordinate; workflow requests are the ones that offer tools.
|
|
984
|
+
expect(bodies).toHaveLength(12);
|
|
985
|
+
for (const model of Object.keys(sampling)) {
|
|
986
|
+
const declared = sampling[model];
|
|
987
|
+
for (const workflow of [true, false]) {
|
|
988
|
+
const group = bodies.filter(
|
|
989
|
+
(body) => body.model === model && Array.isArray(body.tools) === workflow
|
|
990
|
+
);
|
|
991
|
+
expect(group).toHaveLength(3);
|
|
992
|
+
for (const body of group) {
|
|
993
|
+
expect(body).toMatchObject({
|
|
994
|
+
temperature: declared.temperature,
|
|
995
|
+
top_p: declared.top_p,
|
|
996
|
+
presence_penalty: declared.presence_penalty,
|
|
997
|
+
max_tokens: declared.max_output_tokens,
|
|
998
|
+
reasoning_effort: declared.reasoning_effort,
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
expect(group.map((body) => body.seed).sort()).toEqual([40, 41, 42]);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
const coordinates = new Map(
|
|
1006
|
+
result.results.map((item) => [item.coordinate.coordinate_id, item.coordinate])
|
|
1007
|
+
);
|
|
1008
|
+
const workflowEvidence = outcome.evidence.filter(
|
|
1009
|
+
(file) => coordinates.get(file.coordinate_id)?.task_kind === 'agent_workflow'
|
|
1010
|
+
);
|
|
1011
|
+
expect(workflowEvidence).toHaveLength(6);
|
|
1012
|
+
for (const file of workflowEvidence) {
|
|
1013
|
+
const coordinate = coordinates.get(file.coordinate_id);
|
|
1014
|
+
const saved = JSON.parse(await readFile(file.path, 'utf8'));
|
|
1015
|
+
const declared = sampling[saved.configuration.model.display];
|
|
1016
|
+
expect(coordinate?.target_id).toBe(
|
|
1017
|
+
saved.configuration.model.display === 'qwen3.5:4b' ? 'qwen-small' : 'qwen-large'
|
|
1018
|
+
);
|
|
1019
|
+
expect(saved.configuration.generation).toEqual({
|
|
1020
|
+
maxTokens: declared.max_output_tokens,
|
|
1021
|
+
temperature: declared.temperature,
|
|
1022
|
+
topP: declared.top_p,
|
|
1023
|
+
presencePenalty: declared.presence_penalty,
|
|
1024
|
+
seed: coordinate?.seed,
|
|
1025
|
+
reasoningEffort: declared.reasoning_effort,
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
} finally {
|
|
1029
|
+
server.stop(true);
|
|
1030
|
+
}
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
test('awaits an async target config resolver for non-loopback live targets', async () => {
|
|
1034
|
+
const resolved: string[] = [];
|
|
1035
|
+
const fixture = await writeExperimentFixture({
|
|
1036
|
+
overrides: liveOverrides({
|
|
1037
|
+
budgets: { max_requests: 20, max_tokens: 10_000, max_cost: { amount: 1, currency: 'USD' } },
|
|
1038
|
+
}),
|
|
1039
|
+
target: (target) => ({
|
|
1040
|
+
...target,
|
|
1041
|
+
settings: { base_url: 'https://models.example.com/v1' },
|
|
1042
|
+
deployment: { locality: 'remote', cost_basis: 'metered' },
|
|
1043
|
+
weights: undefined,
|
|
1044
|
+
}),
|
|
1045
|
+
});
|
|
1046
|
+
const outcome = await runExperimentFile({
|
|
1047
|
+
experiment: fixture.file,
|
|
1048
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
1049
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
1050
|
+
approvedBy: 'test-operator',
|
|
1051
|
+
reason: 'resolver failure path only',
|
|
1052
|
+
}),
|
|
1053
|
+
resolveTargetConfig: async (target) => {
|
|
1054
|
+
resolved.push(target.id);
|
|
1055
|
+
throw new Error('no transport configured');
|
|
1056
|
+
},
|
|
1057
|
+
});
|
|
1058
|
+
|
|
1059
|
+
expect(resolved[0]).toBe('qwen-small');
|
|
1060
|
+
// The client is unavailable, so the metered attempt reports neither usage nor cost.
|
|
1061
|
+
expect(outcome.result?.stop_reason).toBe('usage_unreported');
|
|
1062
|
+
expect(outcome.exitCode).toBe(EXPERIMENT_EXIT_CODES.stopped);
|
|
1063
|
+
});
|
|
1064
|
+
});
|
|
1065
|
+
|
|
1066
|
+
describe('runExperimentFile (concurrent live mode)', () => {
|
|
1067
|
+
const concurrentLive = () =>
|
|
1068
|
+
writeExperimentFixture({
|
|
1069
|
+
overrides: liveOverrides({
|
|
1070
|
+
concurrency: 2,
|
|
1071
|
+
live: { ...LIVE_CONTROLS, concurrency_ceiling: 2 },
|
|
1072
|
+
}),
|
|
1073
|
+
target: (target) => ({
|
|
1074
|
+
...target,
|
|
1075
|
+
settings: { ...(target.settings as object), max_output_tokens: 16 },
|
|
1076
|
+
}),
|
|
1077
|
+
});
|
|
1078
|
+
|
|
1079
|
+
test('refuses unless the approval names the declared concurrency', async () => {
|
|
1080
|
+
const fixture = await concurrentLive();
|
|
1081
|
+
for (const maxConcurrency of [undefined, 1]) {
|
|
1082
|
+
let created = 0;
|
|
1083
|
+
const outcome = await runExperimentFile({
|
|
1084
|
+
experiment: fixture.file,
|
|
1085
|
+
outputDir: path.join(fixture.directory, `out-${maxConcurrency}`),
|
|
1086
|
+
liveAuthorization: createExperimentLiveAuthorization({
|
|
1087
|
+
approvedBy: 'test-operator',
|
|
1088
|
+
reason: 'in-memory clients only',
|
|
1089
|
+
maxConcurrency,
|
|
1090
|
+
}),
|
|
1091
|
+
createClient: () => {
|
|
1092
|
+
created += 1;
|
|
1093
|
+
throw new Error('must not be called');
|
|
1094
|
+
},
|
|
1095
|
+
});
|
|
1096
|
+
expect(outcome.status).toBe('unauthorized');
|
|
1097
|
+
expect(outcome.exitCode).toBe(3);
|
|
1098
|
+
expect(outcome.message).toContain('--max-concurrency 2');
|
|
1099
|
+
expect(created).toBe(0);
|
|
1100
|
+
}
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
test('runs with an approved concurrency, records it and lists evidence in matrix order', async () => {
|
|
1104
|
+
const fixture = await concurrentLive();
|
|
1105
|
+
const calls: string[] = [];
|
|
1106
|
+
const base = inMemoryClient('Fixture response, Abuja', calls);
|
|
1107
|
+
const authorization = createExperimentLiveAuthorization({
|
|
1108
|
+
approvedBy: 'test-operator',
|
|
1109
|
+
reason: 'in-memory clients only',
|
|
1110
|
+
maxConcurrency: 2,
|
|
1111
|
+
});
|
|
1112
|
+
const outcome = await runExperimentFile({
|
|
1113
|
+
experiment: fixture.file,
|
|
1114
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
1115
|
+
liveAuthorization: authorization,
|
|
1116
|
+
createClient: (target) => {
|
|
1117
|
+
const client = base(target);
|
|
1118
|
+
return {
|
|
1119
|
+
...client,
|
|
1120
|
+
generate: async (options) => {
|
|
1121
|
+
// The first target answers slowly, so attempts finish out of matrix order.
|
|
1122
|
+
await new Promise((resolve) =>
|
|
1123
|
+
setTimeout(resolve, target.id === 'qwen-small' ? 15 : 1)
|
|
1124
|
+
);
|
|
1125
|
+
return client.generate(options);
|
|
1126
|
+
},
|
|
1127
|
+
};
|
|
1128
|
+
},
|
|
1129
|
+
});
|
|
1130
|
+
|
|
1131
|
+
expect(outcome.exitCode).toBe(0);
|
|
1132
|
+
const result = outcome.result as ExperimentRunResult;
|
|
1133
|
+
expect(result.live_authorization).toEqual(authorization);
|
|
1134
|
+
expect(result.live_authorization?.approved_concurrency).toBe(2);
|
|
1135
|
+
expect(result.execution?.peak_in_flight).toBe(2);
|
|
1136
|
+
expect(calls[0]).toBe('qwen-large');
|
|
1137
|
+
expect(outcome.evidence.map((item) => item.coordinate_id)).toEqual(
|
|
1138
|
+
result.results.map((item) => item.coordinate.coordinate_id)
|
|
1139
|
+
);
|
|
1140
|
+
const summary = formatExperimentRunSummary(result);
|
|
1141
|
+
expect(summary).toContain(
|
|
1142
|
+
'Concurrency: up to 2 coordinates in flight (declared limit); recorded peak 2'
|
|
1143
|
+
);
|
|
1144
|
+
});
|
|
1145
|
+
});
|
|
1146
|
+
|
|
1147
|
+
describe('createExperimentLiveAuthorization', () => {
|
|
1148
|
+
test('builds a bounded decision record', () => {
|
|
1149
|
+
const authorization = createExperimentLiveAuthorization({
|
|
1150
|
+
approvedBy: ' operator ',
|
|
1151
|
+
reason: ' local run ',
|
|
1152
|
+
decidedAt: new Date('2026-11-19T10:00:00Z'),
|
|
1153
|
+
});
|
|
1154
|
+
expect(authorization).toMatchObject({
|
|
1155
|
+
approved: true,
|
|
1156
|
+
approved_by: 'operator',
|
|
1157
|
+
reason: 'local run',
|
|
1158
|
+
decided_at: '2026-11-19T10:00:00.000Z',
|
|
1159
|
+
});
|
|
1160
|
+
expect(authorization.decision_id).toMatch(/^live-20261119100000-[a-f0-9]{8}$/);
|
|
1161
|
+
});
|
|
1162
|
+
|
|
1163
|
+
test('records an approved concurrency from 1 to 8', () => {
|
|
1164
|
+
expect(
|
|
1165
|
+
createExperimentLiveAuthorization({ approvedBy: 'x', reason: 'y', maxConcurrency: 4 })
|
|
1166
|
+
).toMatchObject({ approved_concurrency: 4 });
|
|
1167
|
+
expect(
|
|
1168
|
+
'approved_concurrency' in createExperimentLiveAuthorization({ approvedBy: 'x', reason: 'y' })
|
|
1169
|
+
).toBe(false);
|
|
1170
|
+
});
|
|
1171
|
+
|
|
1172
|
+
test('rejects a missing approver or reason with exit code 3', () => {
|
|
1173
|
+
for (const options of [
|
|
1174
|
+
{ approvedBy: '', reason: 'x' },
|
|
1175
|
+
{ approvedBy: 'x', reason: ' ' },
|
|
1176
|
+
{ approvedBy: 'x', reason: 'y', decisionId: 'UPPER' },
|
|
1177
|
+
{ approvedBy: 'x', reason: 'y', maxConcurrency: 0 },
|
|
1178
|
+
{ approvedBy: 'x', reason: 'y', maxConcurrency: 9 },
|
|
1179
|
+
{ approvedBy: 'x', reason: 'y', maxConcurrency: 1.5 },
|
|
1180
|
+
]) {
|
|
1181
|
+
const error = (() => {
|
|
1182
|
+
try {
|
|
1183
|
+
createExperimentLiveAuthorization(options);
|
|
1184
|
+
} catch (caught) {
|
|
1185
|
+
return caught as ExperimentSurfaceError;
|
|
1186
|
+
}
|
|
1187
|
+
return undefined;
|
|
1188
|
+
})();
|
|
1189
|
+
expect(error?.exitCode).toBe(3);
|
|
1190
|
+
}
|
|
1191
|
+
});
|
|
1192
|
+
});
|
|
1193
|
+
|
|
1194
|
+
describe('reportExperimentResult', () => {
|
|
1195
|
+
test('regenerates all views and formats offline from a saved result', async () => {
|
|
1196
|
+
forbidNetwork();
|
|
1197
|
+
const fixture = await writeExperimentFixture();
|
|
1198
|
+
const run = await runExperimentFile({
|
|
1199
|
+
experiment: fixture.file,
|
|
1200
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
1201
|
+
});
|
|
1202
|
+
const network = forbidNetwork();
|
|
1203
|
+
const reportDir = path.join(fixture.directory, 'reports');
|
|
1204
|
+
|
|
1205
|
+
const reports = await reportExperimentResult({
|
|
1206
|
+
result: run.resultPath as string,
|
|
1207
|
+
outputDir: reportDir,
|
|
1208
|
+
});
|
|
1209
|
+
|
|
1210
|
+
expect(network.calls).toBe(0);
|
|
1211
|
+
expect(reports.map((report) => `${report.view}.${report.format}`)).toEqual([
|
|
1212
|
+
'technical.html',
|
|
1213
|
+
'technical.markdown',
|
|
1214
|
+
'executive.html',
|
|
1215
|
+
'executive.markdown',
|
|
1216
|
+
'comprehensive.html',
|
|
1217
|
+
'comprehensive.markdown',
|
|
1218
|
+
]);
|
|
1219
|
+
expect((await readdir(reportDir)).sort()).toEqual([
|
|
1220
|
+
'comprehensive.html',
|
|
1221
|
+
'comprehensive.md',
|
|
1222
|
+
'executive.html',
|
|
1223
|
+
'executive.md',
|
|
1224
|
+
'technical.html',
|
|
1225
|
+
'technical.md',
|
|
1226
|
+
]);
|
|
1227
|
+
for (const report of reports) {
|
|
1228
|
+
expect(await readFile(report.path as string, 'utf8')).toBe(report.content);
|
|
1229
|
+
}
|
|
1230
|
+
});
|
|
1231
|
+
|
|
1232
|
+
test('selects views and formats', async () => {
|
|
1233
|
+
forbidNetwork();
|
|
1234
|
+
const fixture = await writeExperimentFixture();
|
|
1235
|
+
const run = await runExperimentFile({
|
|
1236
|
+
experiment: fixture.file,
|
|
1237
|
+
outputDir: path.join(fixture.directory, 'out'),
|
|
1238
|
+
});
|
|
1239
|
+
const reports = await reportExperimentResult({
|
|
1240
|
+
result: run.result as ExperimentRunResult,
|
|
1241
|
+
views: ['executive'],
|
|
1242
|
+
formats: ['markdown'],
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
expect(reports).toHaveLength(1);
|
|
1246
|
+
expect(reports[0]?.path).toBeUndefined();
|
|
1247
|
+
expect(reports[0]?.content.startsWith('#')).toBe(true);
|
|
1248
|
+
});
|
|
1249
|
+
|
|
1250
|
+
test('maps unreadable input to exit code 1 and invalid input to exit code 2', async () => {
|
|
1251
|
+
const fixture = await writeExperimentFixture();
|
|
1252
|
+
const missing = await reportExperimentResult({
|
|
1253
|
+
result: path.join(fixture.directory, 'missing.json'),
|
|
1254
|
+
}).catch((error) => error);
|
|
1255
|
+
const invalid = await reportExperimentResult({ result: fixture.file }).catch((error) => error);
|
|
1256
|
+
|
|
1257
|
+
expect(missing).toBeInstanceOf(ExperimentSurfaceError);
|
|
1258
|
+
expect(missing.exitCode).toBe(1);
|
|
1259
|
+
expect(invalid).toBeInstanceOf(ExperimentSurfaceError);
|
|
1260
|
+
expect(invalid.exitCode).toBe(2);
|
|
1261
|
+
});
|
|
1262
|
+
});
|