@artemiskit/sdk 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/__tests__/experiment-fixtures.d.ts +44 -0
- package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
- package/dist/builders/index.js +19 -66
- package/dist/contracts/index.js +3 -50
- package/dist/experiment-files.d.ts +54 -0
- package/dist/experiment-files.d.ts.map +1 -0
- package/dist/experiment-fixture.d.ts +20 -0
- package/dist/experiment-fixture.d.ts.map +1 -0
- package/dist/experiment-summary.d.ts +44 -0
- package/dist/experiment-summary.d.ts.map +1 -0
- package/dist/experiment-validation.d.ts +48 -0
- package/dist/experiment-validation.d.ts.map +1 -0
- package/dist/experiment.d.ts +162 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +6 -1
- package/dist/guardian/circuit-breaker.d.ts.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2557 -1863
- package/dist/matchers/index.js +13 -60
- package/dist/matchers/jest.js +0 -47
- package/dist/matchers/vitest.js +0 -47
- package/dist/utils/index.js +11 -58
- package/package.json +6 -5
- package/src/__tests__/artemiskit.test.ts +1 -2
- package/src/__tests__/builders.test.ts +0 -2
- package/src/__tests__/circuit-breaker.test.ts +1 -1
- package/src/__tests__/compare.test.ts +1 -1
- package/src/__tests__/contracts.test.ts +1 -1
- package/src/__tests__/experiment-case-level.test.ts +212 -0
- package/src/__tests__/experiment-fixtures.ts +194 -0
- package/src/__tests__/experiment.test.ts +1262 -0
- package/src/__tests__/guardian-modes.test.ts +2 -2
- package/src/__tests__/guardrails.test.ts +1 -1
- package/src/__tests__/matchers.test.ts +1 -1
- package/src/__tests__/semantic-validator.test.ts +15 -10
- package/src/__tests__/validate.test.ts +0 -4
- package/src/__tests__/workflow-recovery.test.ts +7 -3
- package/src/__tests__/workflow-report.test.ts +6 -3
- package/src/__tests__/workflow.test.ts +11 -5
- package/src/experiment-files.ts +190 -0
- package/src/experiment-fixture.ts +70 -0
- package/src/experiment-summary.ts +172 -0
- package/src/experiment-validation.ts +426 -0
- package/src/experiment.ts +630 -0
- package/src/guardian/circuit-breaker.ts +6 -1
- package/src/index.ts +52 -0
- package/tsconfig.typecheck.json +12 -0
|
@@ -0,0 +1,630 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Comparative experiment surfaces: validate, run and report.
|
|
3
|
+
*
|
|
4
|
+
* This module is the one engine behind both the SDK functions and the `akit experiment` CLI.
|
|
5
|
+
* Fixture experiments bind every target to a deterministic in-memory client and make no network
|
|
6
|
+
* calls. Live experiments additionally require an explicit `ExperimentLiveAuthorization`.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { randomBytes } from 'node:crypto';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import {
|
|
12
|
+
type AdapterConfig,
|
|
13
|
+
type ContentIdentity,
|
|
14
|
+
type ExperimentAttemptExecutor,
|
|
15
|
+
type ExperimentLiveAuthorization,
|
|
16
|
+
type ExperimentRunResult,
|
|
17
|
+
type ExperimentTarget,
|
|
18
|
+
type ExperimentTaskKind,
|
|
19
|
+
type ModelClient,
|
|
20
|
+
createAdapter,
|
|
21
|
+
createScenarioExperimentExecutor,
|
|
22
|
+
createWorkflowExperimentExecutor,
|
|
23
|
+
isLoopbackEndpoint,
|
|
24
|
+
runExperiment,
|
|
25
|
+
} from '@artemiskit/core';
|
|
26
|
+
import {
|
|
27
|
+
type ExperimentCaseEvidenceFiles,
|
|
28
|
+
type ExperimentCaseLevelSummary,
|
|
29
|
+
type ExperimentReportFormat,
|
|
30
|
+
type ExperimentReportView,
|
|
31
|
+
createExperimentCaseSummary,
|
|
32
|
+
createExperimentReport,
|
|
33
|
+
listExperimentCaseEvidenceDigests,
|
|
34
|
+
renderExperimentReportHTML,
|
|
35
|
+
renderExperimentReportMarkdown,
|
|
36
|
+
} from '@artemiskit/reports';
|
|
37
|
+
import {
|
|
38
|
+
EXPERIMENT_EVIDENCE_DIR,
|
|
39
|
+
ExperimentSurfaceError,
|
|
40
|
+
MAX_EXPERIMENT_RESULT_BYTES,
|
|
41
|
+
defaultExperimentOutputDir,
|
|
42
|
+
prepareEmptyDirectory,
|
|
43
|
+
readBoundedText,
|
|
44
|
+
readExperimentCaseEvidence,
|
|
45
|
+
saveAttemptEvidence,
|
|
46
|
+
saveExperimentResult,
|
|
47
|
+
writeNewFile,
|
|
48
|
+
} from './experiment-files';
|
|
49
|
+
import { createExperimentFixtureClient } from './experiment-fixture';
|
|
50
|
+
import {
|
|
51
|
+
EXPERIMENT_EXIT_CODES,
|
|
52
|
+
type ExperimentExitCode,
|
|
53
|
+
type ExperimentRunStatus,
|
|
54
|
+
experimentRunStatus,
|
|
55
|
+
} from './experiment-summary';
|
|
56
|
+
import {
|
|
57
|
+
type ExperimentValidation,
|
|
58
|
+
isReasoningEffort,
|
|
59
|
+
validateExperimentFile,
|
|
60
|
+
} from './experiment-validation';
|
|
61
|
+
|
|
62
|
+
export { ExperimentSurfaceError } from './experiment-files';
|
|
63
|
+
export {
|
|
64
|
+
EXPERIMENT_EXIT_CODES,
|
|
65
|
+
describeConcurrency,
|
|
66
|
+
describeTargetDeployment,
|
|
67
|
+
experimentRunStatus,
|
|
68
|
+
formatExperimentCaseSummary,
|
|
69
|
+
formatExperimentRunSummary,
|
|
70
|
+
} from './experiment-summary';
|
|
71
|
+
export type { ExperimentExitCode, ExperimentRunStatus } from './experiment-summary';
|
|
72
|
+
export { LOOPBACK_REQUIRED_MESSAGE, validateExperimentFile } from './experiment-validation';
|
|
73
|
+
export type {
|
|
74
|
+
ExperimentCheck,
|
|
75
|
+
ExperimentCheckStatus,
|
|
76
|
+
ExperimentCheckSubject,
|
|
77
|
+
ExperimentValidation,
|
|
78
|
+
} from './experiment-validation';
|
|
79
|
+
export { createExperimentFixtureClient } from './experiment-fixture';
|
|
80
|
+
export type { ExperimentFixtureClientOptions } from './experiment-fixture';
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Non-secret API key sent to loopback OpenAI-compatible servers (such as Ollama) that ignore
|
|
84
|
+
* authentication. It keeps the OpenAI client from falling back to an ambient `OPENAI_API_KEY`,
|
|
85
|
+
* so a real key never reaches a local endpoint unless the host configures one explicitly.
|
|
86
|
+
*/
|
|
87
|
+
export const LOCAL_ENDPOINT_PLACEHOLDER_API_KEY = 'artemiskit-local-no-key';
|
|
88
|
+
|
|
89
|
+
const DECISION_ID = /^[a-z0-9][a-z0-9._:-]{0,63}$/;
|
|
90
|
+
const MAX_APPROVER_LENGTH = 128;
|
|
91
|
+
const MAX_REASON_LENGTH = 256;
|
|
92
|
+
const MAX_CONCURRENCY = 8;
|
|
93
|
+
type ApprovedConcurrency = NonNullable<ExperimentLiveAuthorization['approved_concurrency']>;
|
|
94
|
+
|
|
95
|
+
export interface CreateExperimentLiveAuthorizationOptions {
|
|
96
|
+
/** Who approved the live run. */
|
|
97
|
+
approvedBy: string;
|
|
98
|
+
/** Why the live run is approved. */
|
|
99
|
+
reason: string;
|
|
100
|
+
/** Bounded decision ID; generated when omitted. */
|
|
101
|
+
decisionId?: string;
|
|
102
|
+
/** Decision time; defaults to now. */
|
|
103
|
+
decidedAt?: Date;
|
|
104
|
+
/**
|
|
105
|
+
* Highest experiment concurrency this approval covers (integer 1..8). Required to run a
|
|
106
|
+
* live experiment that declares concurrency above 1; omitted means serial live calls only.
|
|
107
|
+
*/
|
|
108
|
+
maxConcurrency?: number;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Build the explicit live-run approval record that `runExperimentFile` requires for
|
|
113
|
+
* `mode: live` experiments. Throws `ExperimentSurfaceError` (exit code 3) on invalid input.
|
|
114
|
+
*/
|
|
115
|
+
export function createExperimentLiveAuthorization(
|
|
116
|
+
options: CreateExperimentLiveAuthorizationOptions
|
|
117
|
+
): ExperimentLiveAuthorization {
|
|
118
|
+
const decidedAt = options.decidedAt ?? new Date();
|
|
119
|
+
const authorization: ExperimentLiveAuthorization = {
|
|
120
|
+
approved: true,
|
|
121
|
+
decision_id: options.decisionId ?? generateDecisionId(decidedAt),
|
|
122
|
+
decided_at: decidedAt.toISOString(),
|
|
123
|
+
approved_by: String(options.approvedBy ?? '').trim(),
|
|
124
|
+
reason: String(options.reason ?? '').trim(),
|
|
125
|
+
...(options.maxConcurrency === undefined
|
|
126
|
+
? {}
|
|
127
|
+
: { approved_concurrency: options.maxConcurrency as ApprovedConcurrency }),
|
|
128
|
+
};
|
|
129
|
+
if (!isValidLiveAuthorization(authorization)) {
|
|
130
|
+
throw new ExperimentSurfaceError(
|
|
131
|
+
'authorization_invalid',
|
|
132
|
+
`Live approval needs an approver (1-${MAX_APPROVER_LENGTH} characters), a reason (1-${MAX_REASON_LENGTH} characters), a decision ID matching ${DECISION_ID} and, when given, a maximum concurrency from 1 to ${MAX_CONCURRENCY}`,
|
|
133
|
+
EXPERIMENT_EXIT_CODES.unauthorized
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
return authorization;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function generateDecisionId(now: Date): string {
|
|
140
|
+
const stamp = now.toISOString().replace(/\D/g, '').slice(0, 14);
|
|
141
|
+
return `live-${stamp}-${randomBytes(4).toString('hex')}`;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function isValidLiveAuthorization(value: unknown): value is ExperimentLiveAuthorization {
|
|
145
|
+
const candidate = value as Partial<ExperimentLiveAuthorization> | undefined;
|
|
146
|
+
return (
|
|
147
|
+
typeof candidate === 'object' &&
|
|
148
|
+
candidate !== null &&
|
|
149
|
+
candidate.approved === true &&
|
|
150
|
+
typeof candidate.decision_id === 'string' &&
|
|
151
|
+
DECISION_ID.test(candidate.decision_id) &&
|
|
152
|
+
typeof candidate.decided_at === 'string' &&
|
|
153
|
+
!Number.isNaN(Date.parse(candidate.decided_at)) &&
|
|
154
|
+
boundedText(candidate.approved_by, MAX_APPROVER_LENGTH) &&
|
|
155
|
+
boundedText(candidate.reason, MAX_REASON_LENGTH) &&
|
|
156
|
+
(candidate.approved_concurrency === undefined ||
|
|
157
|
+
(Number.isSafeInteger(candidate.approved_concurrency) &&
|
|
158
|
+
candidate.approved_concurrency >= 1 &&
|
|
159
|
+
candidate.approved_concurrency <= MAX_CONCURRENCY))
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function boundedText(value: unknown, maximum: number): boolean {
|
|
164
|
+
return typeof value === 'string' && value.trim().length > 0 && value.trim().length <= maximum;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export interface RunExperimentFileOptions {
|
|
168
|
+
/** Experiment YAML or JSON file. Task sources resolve relative to its directory. */
|
|
169
|
+
experiment: string;
|
|
170
|
+
/**
|
|
171
|
+
* New or empty directory for `result.json` and `evidence/`. Defaults to
|
|
172
|
+
* `<cwd>/artemis-output/experiments/<experiment-id>/<timestamp>/`.
|
|
173
|
+
*/
|
|
174
|
+
outputDir?: string;
|
|
175
|
+
/** Base for the default output directory; defaults to `process.cwd()`. */
|
|
176
|
+
cwd?: string;
|
|
177
|
+
/** Required for `mode: live`; ignored for fixture experiments. */
|
|
178
|
+
liveAuthorization?: ExperimentLiveAuthorization;
|
|
179
|
+
/**
|
|
180
|
+
* Live mode only: provider transport settings for a target (API key, timeout, organization).
|
|
181
|
+
* The target's `settings.base_url`, provider and model always take precedence.
|
|
182
|
+
*/
|
|
183
|
+
resolveTargetConfig?: (
|
|
184
|
+
target: ExperimentTarget
|
|
185
|
+
) => Partial<AdapterConfig> | undefined | Promise<Partial<AdapterConfig> | undefined>;
|
|
186
|
+
/** Live mode only: supply your own client per target instead of the built-in adapters. */
|
|
187
|
+
createClient?: (target: ExperimentTarget) => ModelClient | Promise<ModelClient>;
|
|
188
|
+
/**
|
|
189
|
+
* Per-case model request timeout in milliseconds for scenario tasks. Live targets bound
|
|
190
|
+
* through the built-in adapters also get an HTTP timeout of at least this value. Workflow
|
|
191
|
+
* tasks keep their own declared `timeout_ms`.
|
|
192
|
+
*/
|
|
193
|
+
timeoutMs?: number;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Saved evidence for one executed attempt: the scenario run manifest, or the metadata-only
|
|
198
|
+
* workflow record, as canonical JSON named by its SHA-256.
|
|
199
|
+
*/
|
|
200
|
+
export interface ExperimentEvidenceFile {
|
|
201
|
+
coordinate_id: string;
|
|
202
|
+
attempt_number: number;
|
|
203
|
+
sha256: string;
|
|
204
|
+
path: string;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export interface ExperimentFileRunOutcome {
|
|
208
|
+
status: ExperimentRunStatus;
|
|
209
|
+
/** Same value `akit experiment run` exits with. */
|
|
210
|
+
exitCode: ExperimentExitCode;
|
|
211
|
+
validation: ExperimentValidation;
|
|
212
|
+
/** Present once the run executed. */
|
|
213
|
+
result?: ExperimentRunResult;
|
|
214
|
+
outputDir?: string;
|
|
215
|
+
resultPath?: string;
|
|
216
|
+
evidence: ExperimentEvidenceFile[];
|
|
217
|
+
/**
|
|
218
|
+
* Present once the run executed: individual scenario cases from the saved, SHA-256-verified
|
|
219
|
+
* evidence, or why they are unavailable. Coordinate-level counts stay in `result.summaries`.
|
|
220
|
+
*/
|
|
221
|
+
caseLevel?: ExperimentCaseLevelSummary;
|
|
222
|
+
/** Explanation for refusals (`invalid`, `unauthorized`). */
|
|
223
|
+
message?: string;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Validate, then run an experiment file and save the V1 result plus per-attempt evidence.
|
|
228
|
+
*
|
|
229
|
+
* Each task runs through the built-in executor for its kind: `scenario_evaluation` tasks
|
|
230
|
+
* through the scenario runner and `agent_workflow` tasks through the native workflow engine
|
|
231
|
+
* (simulated or sandbox environment, network denied). Fixture targets never call a tool, so
|
|
232
|
+
* fixture workflow outcomes describe the pipeline, never a model.
|
|
233
|
+
*
|
|
234
|
+
* Refusals (`invalid`, `unauthorized`) return before any client is created or any attempt runs.
|
|
235
|
+
* Output and persistence failures throw `ExperimentSurfaceError` (exit code 1).
|
|
236
|
+
*
|
|
237
|
+
* A target's generation settings (`temperature`, `top_p`, `presence_penalty`,
|
|
238
|
+
* `max_output_tokens`) replace each task's own values for that target's requests, and every
|
|
239
|
+
* request carries the coordinate's seed. `llm_grader` and embedding requests keep their own
|
|
240
|
+
* sampling and output length. `reasoning_effort` is adapter configuration, so it also applies to
|
|
241
|
+
* `llm_grader` requests made through the target's client. Workflow tasks get the same settings
|
|
242
|
+
* and seed on every workflow model request. Each saved run manifest records the values sent in
|
|
243
|
+
* `resolved_config`; each saved workflow record lists them in `configuration.generation`.
|
|
244
|
+
*/
|
|
245
|
+
export async function runExperimentFile(
|
|
246
|
+
options: RunExperimentFileOptions
|
|
247
|
+
): Promise<ExperimentFileRunOutcome> {
|
|
248
|
+
const validation = await validateExperimentFile(options.experiment);
|
|
249
|
+
if (!validation.valid || !validation.manifest) {
|
|
250
|
+
return refusal('invalid', validation, 'Experiment validation failed; nothing was executed');
|
|
251
|
+
}
|
|
252
|
+
const manifest = validation.manifest;
|
|
253
|
+
const live = manifest.mode === 'live';
|
|
254
|
+
if (live && !isValidLiveAuthorization(options.liveAuthorization)) {
|
|
255
|
+
return refusal(
|
|
256
|
+
'unauthorized',
|
|
257
|
+
validation,
|
|
258
|
+
'Live experiments require an explicit approval (approver, reason, decision ID); nothing was executed'
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
const approvedConcurrency = options.liveAuthorization?.approved_concurrency ?? 1;
|
|
262
|
+
if (live && manifest.concurrency > approvedConcurrency) {
|
|
263
|
+
return refusal(
|
|
264
|
+
'unauthorized',
|
|
265
|
+
validation,
|
|
266
|
+
`The experiment declares concurrency ${manifest.concurrency} but the live approval covers ${approvedConcurrency}; approve it explicitly (--max-concurrency ${manifest.concurrency}); nothing was executed`
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const outputDir = path.resolve(
|
|
271
|
+
options.outputDir ??
|
|
272
|
+
defaultExperimentOutputDir(manifest.id, options.cwd ?? process.cwd(), new Date())
|
|
273
|
+
);
|
|
274
|
+
await prepareEmptyDirectory(outputDir);
|
|
275
|
+
|
|
276
|
+
const evidence: ExperimentEvidenceFile[] = [];
|
|
277
|
+
const saveEvidence = async (
|
|
278
|
+
attempt: { coordinate_id: string; attempt_number: number; artifact: ContentIdentity },
|
|
279
|
+
content: unknown
|
|
280
|
+
) => {
|
|
281
|
+
const file = await saveAttemptEvidence(outputDir, content, attempt.artifact.digest);
|
|
282
|
+
evidence.push({
|
|
283
|
+
coordinate_id: attempt.coordinate_id,
|
|
284
|
+
attempt_number: attempt.attempt_number,
|
|
285
|
+
sha256: attempt.artifact.digest,
|
|
286
|
+
path: file,
|
|
287
|
+
});
|
|
288
|
+
};
|
|
289
|
+
const executor = routeByTaskKind({
|
|
290
|
+
scenario_evaluation: createScenarioExperimentExecutor({
|
|
291
|
+
root: validation.root,
|
|
292
|
+
repetitions: manifest.repetitions,
|
|
293
|
+
timeoutMs: options.timeoutMs,
|
|
294
|
+
createClient: live
|
|
295
|
+
? (target) => bindLiveClient(target, options)
|
|
296
|
+
: (target) => createExperimentFixtureClient(target),
|
|
297
|
+
onEvidence: (record) => saveEvidence(record, record.manifest),
|
|
298
|
+
}),
|
|
299
|
+
agent_workflow: createWorkflowExperimentExecutor({
|
|
300
|
+
root: validation.root,
|
|
301
|
+
createClient: live
|
|
302
|
+
? (target) => bindLiveClient(target, options)
|
|
303
|
+
: (target) => createExperimentFixtureClient(target, { toolUse: true }),
|
|
304
|
+
onEvidence: (record) => saveEvidence(record, record.record),
|
|
305
|
+
}),
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
const result = await runExperiment(manifest, {
|
|
309
|
+
execute_attempt: executor,
|
|
310
|
+
...(live ? { live_authorization: pickAuthorization(options.liveAuthorization) } : {}),
|
|
311
|
+
});
|
|
312
|
+
sortEvidence(evidence, result);
|
|
313
|
+
const resultPath = await saveExperimentResult(outputDir, result);
|
|
314
|
+
const status = experimentRunStatus(result);
|
|
315
|
+
return {
|
|
316
|
+
status,
|
|
317
|
+
exitCode: EXPERIMENT_EXIT_CODES[status],
|
|
318
|
+
validation,
|
|
319
|
+
result,
|
|
320
|
+
outputDir,
|
|
321
|
+
resultPath,
|
|
322
|
+
evidence,
|
|
323
|
+
caseLevel: await savedRunCaseLevel(result, outputDir),
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Case-level summary read back from the files just saved, so a run reports exactly what a
|
|
329
|
+
* later `report` of the same directory would. A result the report validator rejects (never
|
|
330
|
+
* expected from the runner) yields an unavailable summary rather than failing the saved run.
|
|
331
|
+
*/
|
|
332
|
+
async function savedRunCaseLevel(
|
|
333
|
+
result: ExperimentRunResult,
|
|
334
|
+
outputDir: string
|
|
335
|
+
): Promise<ExperimentCaseLevelSummary> {
|
|
336
|
+
const caseEvidence = await loadCaseEvidence(
|
|
337
|
+
result,
|
|
338
|
+
path.join(outputDir, EXPERIMENT_EVIDENCE_DIR)
|
|
339
|
+
);
|
|
340
|
+
try {
|
|
341
|
+
return createExperimentCaseSummary(result, { caseEvidence });
|
|
342
|
+
} catch {
|
|
343
|
+
return {
|
|
344
|
+
availability: 'unavailable',
|
|
345
|
+
reason: 'result_invalid',
|
|
346
|
+
detail:
|
|
347
|
+
'Case-level summary unavailable: the saved result did not pass report validation. Coordinate-level counts are unaffected.',
|
|
348
|
+
affectedEvidence: [],
|
|
349
|
+
affectedEvidenceCount: 0,
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Evidence bytes for the attempts a case-level summary reads; `undefined` without a directory. */
|
|
355
|
+
async function loadCaseEvidence(
|
|
356
|
+
input: unknown,
|
|
357
|
+
evidenceDir: string | undefined
|
|
358
|
+
): Promise<ExperimentCaseEvidenceFiles | undefined> {
|
|
359
|
+
if (evidenceDir === undefined) return undefined;
|
|
360
|
+
return readExperimentCaseEvidence(evidenceDir, listExperimentCaseEvidenceDigests(input));
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** Explicit evidence directory, else `evidence/` beside a result file; none for a parsed result. */
|
|
364
|
+
function caseEvidenceDir(
|
|
365
|
+
result: string | ExperimentRunResult,
|
|
366
|
+
evidenceDir: string | undefined
|
|
367
|
+
): string | undefined {
|
|
368
|
+
if (evidenceDir !== undefined) return path.resolve(evidenceDir);
|
|
369
|
+
if (typeof result !== 'string') return undefined;
|
|
370
|
+
return path.join(path.dirname(path.resolve(result)), EXPERIMENT_EVIDENCE_DIR);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** One attempt executor that hands each task, and its demand request, to its kind's executor. */
|
|
374
|
+
function routeByTaskKind(
|
|
375
|
+
executors: Record<ExperimentTaskKind, ExperimentAttemptExecutor>
|
|
376
|
+
): ExperimentAttemptExecutor {
|
|
377
|
+
const execute: ExperimentAttemptExecutor = (input) => executors[input.task.kind](input);
|
|
378
|
+
execute.reserve = (request) => executors[request.task.kind].reserve?.(request);
|
|
379
|
+
return execute;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function refusal(
|
|
383
|
+
status: 'invalid' | 'unauthorized',
|
|
384
|
+
validation: ExperimentValidation,
|
|
385
|
+
message: string
|
|
386
|
+
): ExperimentFileRunOutcome {
|
|
387
|
+
return {
|
|
388
|
+
status,
|
|
389
|
+
exitCode: EXPERIMENT_EXIT_CODES[status],
|
|
390
|
+
validation,
|
|
391
|
+
evidence: [],
|
|
392
|
+
message,
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Core parses the authorization strictly; pass exactly the declared fields. */
|
|
397
|
+
function pickAuthorization(value: ExperimentLiveAuthorization | undefined) {
|
|
398
|
+
const authorization = value as ExperimentLiveAuthorization;
|
|
399
|
+
return {
|
|
400
|
+
approved: authorization.approved,
|
|
401
|
+
decision_id: authorization.decision_id,
|
|
402
|
+
decided_at: authorization.decided_at,
|
|
403
|
+
approved_by: authorization.approved_by,
|
|
404
|
+
reason: authorization.reason,
|
|
405
|
+
...(authorization.approved_concurrency === undefined
|
|
406
|
+
? {}
|
|
407
|
+
: { approved_concurrency: authorization.approved_concurrency }),
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Concurrent attempts save evidence in completion order; list it in matrix order, then by
|
|
413
|
+
* attempt number, so the evidence index is deterministic.
|
|
414
|
+
*/
|
|
415
|
+
function sortEvidence(evidence: ExperimentEvidenceFile[], result: ExperimentRunResult): void {
|
|
416
|
+
const order = new Map(
|
|
417
|
+
result.results.map((item, index) => [item.coordinate.coordinate_id, index])
|
|
418
|
+
);
|
|
419
|
+
const position = (item: ExperimentEvidenceFile) =>
|
|
420
|
+
order.get(item.coordinate_id) ?? Number.MAX_SAFE_INTEGER;
|
|
421
|
+
evidence.sort(
|
|
422
|
+
(left, right) => position(left) - position(right) || left.attempt_number - right.attempt_number
|
|
423
|
+
);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Bind a live target through the adapter registry. Loopback targets never inherit an ambient
|
|
428
|
+
* API key: they get the host's explicit key or a non-secret placeholder.
|
|
429
|
+
*/
|
|
430
|
+
async function bindLiveClient(
|
|
431
|
+
target: ExperimentTarget,
|
|
432
|
+
options: RunExperimentFileOptions
|
|
433
|
+
): Promise<ModelClient> {
|
|
434
|
+
if (options.createClient) return options.createClient(target);
|
|
435
|
+
const configured = (await options.resolveTargetConfig?.(target)) ?? {};
|
|
436
|
+
const settingsBaseUrl = target.settings?.base_url;
|
|
437
|
+
const baseUrl = typeof settingsBaseUrl === 'string' ? settingsBaseUrl : configured.baseUrl;
|
|
438
|
+
const loopback = typeof baseUrl === 'string' && isLoopbackEndpoint(baseUrl);
|
|
439
|
+
if (target.deployment?.cost_basis === 'not_applicable' && !loopback) {
|
|
440
|
+
throw new Error('cost_not_applicable_requires_loopback');
|
|
441
|
+
}
|
|
442
|
+
return createAdapter({
|
|
443
|
+
...configured,
|
|
444
|
+
provider: target.provider,
|
|
445
|
+
defaultModel: target.model,
|
|
446
|
+
baseUrl,
|
|
447
|
+
apiKey: loopback
|
|
448
|
+
? (configured.apiKey ?? LOCAL_ENDPOINT_PLACEHOLDER_API_KEY)
|
|
449
|
+
: configured.apiKey,
|
|
450
|
+
maxRetries: 0,
|
|
451
|
+
...transportTimeoutConfig(configured.timeout, options.timeoutMs),
|
|
452
|
+
...reasoningEffortConfig(target),
|
|
453
|
+
} as AdapterConfig);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* The adapter's HTTP timeout must never be shorter than the run's per-case timeout, or long
|
|
458
|
+
* local generations are aborted by the transport (the OpenAI adapter defaults to 60 s) before
|
|
459
|
+
* the declared per-case limit applies. Without a per-case timeout the adapter default stands.
|
|
460
|
+
*/
|
|
461
|
+
function transportTimeoutConfig(
|
|
462
|
+
configuredMs: number | undefined,
|
|
463
|
+
perCaseMs: number | undefined
|
|
464
|
+
): { timeout?: number } {
|
|
465
|
+
if (perCaseMs === undefined) return {};
|
|
466
|
+
return { timeout: Math.max(configuredMs ?? 0, perCaseMs) };
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** Map a target's declared `settings.reasoning_effort` onto the adapter option. */
|
|
470
|
+
function reasoningEffortConfig(target: ExperimentTarget): { reasoningEffort?: string } {
|
|
471
|
+
const effort = target.settings?.reasoning_effort;
|
|
472
|
+
if (effort === undefined) return {};
|
|
473
|
+
if (!isReasoningEffort(effort)) throw new Error('invalid_reasoning_effort');
|
|
474
|
+
return { reasoningEffort: effort };
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
export interface ReportExperimentResultOptions {
|
|
478
|
+
/** Path to a saved `result.json`, or the parsed V1 result. */
|
|
479
|
+
result: string | ExperimentRunResult;
|
|
480
|
+
/** Views to generate; defaults to all three. */
|
|
481
|
+
views?: ExperimentReportView[];
|
|
482
|
+
/** Formats to generate; defaults to HTML and Markdown. */
|
|
483
|
+
formats?: ExperimentReportFormat[];
|
|
484
|
+
/** New or empty directory to write `<view>.<html|md>` files into; omit to only return content. */
|
|
485
|
+
outputDir?: string;
|
|
486
|
+
/**
|
|
487
|
+
* Directory of saved per-attempt evidence for the case-level summary. Defaults to `evidence/`
|
|
488
|
+
* beside a result file; a parsed result without this option reports case level unavailable.
|
|
489
|
+
*/
|
|
490
|
+
evidenceDir?: string;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
export interface ExperimentReportFile {
|
|
494
|
+
view: ExperimentReportView;
|
|
495
|
+
format: ExperimentReportFormat;
|
|
496
|
+
content: string;
|
|
497
|
+
/** Written file, when `outputDir` was given. */
|
|
498
|
+
path?: string;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
export const EXPERIMENT_REPORT_VIEWS: readonly ExperimentReportView[] = [
|
|
502
|
+
'technical',
|
|
503
|
+
'executive',
|
|
504
|
+
'comprehensive',
|
|
505
|
+
];
|
|
506
|
+
export const EXPERIMENT_REPORT_FORMATS: readonly ExperimentReportFormat[] = ['html', 'markdown'];
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* Generate experiment reports offline from a saved result. Loads no task, provider, credential
|
|
510
|
+
* or network. Throws `ExperimentSurfaceError`: exit code 1 for unreadable input or unwritable
|
|
511
|
+
* output, 2 for an invalid or unsupported saved result.
|
|
512
|
+
*/
|
|
513
|
+
export async function reportExperimentResult(
|
|
514
|
+
options: ReportExperimentResultOptions
|
|
515
|
+
): Promise<ExperimentReportFile[]> {
|
|
516
|
+
const views = uniqueChoices(options.views, EXPERIMENT_REPORT_VIEWS);
|
|
517
|
+
const formats = uniqueChoices(options.formats, EXPERIMENT_REPORT_FORMATS);
|
|
518
|
+
const input =
|
|
519
|
+
typeof options.result === 'string' ? await readResult(options.result) : options.result;
|
|
520
|
+
const caseEvidence = await loadCaseEvidence(
|
|
521
|
+
input,
|
|
522
|
+
caseEvidenceDir(options.result, options.evidenceDir)
|
|
523
|
+
);
|
|
524
|
+
|
|
525
|
+
let reports: ExperimentReportFile[];
|
|
526
|
+
try {
|
|
527
|
+
const report = createExperimentReport(input, { caseEvidence });
|
|
528
|
+
reports = views.flatMap((view) =>
|
|
529
|
+
formats.map((format) => ({
|
|
530
|
+
view,
|
|
531
|
+
format,
|
|
532
|
+
content:
|
|
533
|
+
format === 'html'
|
|
534
|
+
? renderExperimentReportHTML(report, { view })
|
|
535
|
+
: renderExperimentReportMarkdown(report, { view }),
|
|
536
|
+
}))
|
|
537
|
+
);
|
|
538
|
+
} catch {
|
|
539
|
+
throw new ExperimentSurfaceError(
|
|
540
|
+
'result_invalid',
|
|
541
|
+
'Invalid, unsupported or ambiguous saved experiment result; no report generated',
|
|
542
|
+
EXPERIMENT_EXIT_CODES.invalid
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
if (options.outputDir !== undefined) {
|
|
547
|
+
const directory = path.resolve(options.outputDir);
|
|
548
|
+
await prepareEmptyDirectory(directory);
|
|
549
|
+
for (const report of reports) {
|
|
550
|
+
const file = path.join(
|
|
551
|
+
directory,
|
|
552
|
+
`${report.view}.${report.format === 'html' ? 'html' : 'md'}`
|
|
553
|
+
);
|
|
554
|
+
try {
|
|
555
|
+
await writeNewFile(file, report.content);
|
|
556
|
+
} catch {
|
|
557
|
+
throw new ExperimentSurfaceError('report_unwritable', 'Unable to write report file', 1);
|
|
558
|
+
}
|
|
559
|
+
report.path = file;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
return reports;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
export interface SummarizeExperimentCasesOptions {
|
|
566
|
+
/** Path to a saved `result.json`, or the parsed V1 result. */
|
|
567
|
+
result: string | ExperimentRunResult;
|
|
568
|
+
/** Saved per-attempt evidence directory; defaults to `evidence/` beside a result file. */
|
|
569
|
+
evidenceDir?: string;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Case-level summary of a saved result, offline: individual scenario cases from each
|
|
574
|
+
* coordinate's terminal attempt evidence, verified against the recorded SHA-256. Missing or
|
|
575
|
+
* altered evidence yields an `unavailable` summary with its reason; an invalid result throws
|
|
576
|
+
* `ExperimentSurfaceError` (exit code 1 unreadable, 2 invalid).
|
|
577
|
+
*/
|
|
578
|
+
export async function summarizeExperimentCases(
|
|
579
|
+
options: SummarizeExperimentCasesOptions
|
|
580
|
+
): Promise<ExperimentCaseLevelSummary> {
|
|
581
|
+
const input =
|
|
582
|
+
typeof options.result === 'string' ? await readResult(options.result) : options.result;
|
|
583
|
+
const caseEvidence = await loadCaseEvidence(
|
|
584
|
+
input,
|
|
585
|
+
caseEvidenceDir(options.result, options.evidenceDir)
|
|
586
|
+
);
|
|
587
|
+
try {
|
|
588
|
+
return createExperimentCaseSummary(input, { caseEvidence });
|
|
589
|
+
} catch {
|
|
590
|
+
throw new ExperimentSurfaceError(
|
|
591
|
+
'result_invalid',
|
|
592
|
+
'Invalid, unsupported or ambiguous saved experiment result; no case-level summary',
|
|
593
|
+
EXPERIMENT_EXIT_CODES.invalid
|
|
594
|
+
);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
async function readResult(file: string): Promise<unknown> {
|
|
599
|
+
let text: string;
|
|
600
|
+
try {
|
|
601
|
+
text = await readBoundedText(path.resolve(file), MAX_EXPERIMENT_RESULT_BYTES);
|
|
602
|
+
} catch {
|
|
603
|
+
throw new ExperimentSurfaceError(
|
|
604
|
+
'result_unreadable',
|
|
605
|
+
'Saved experiment result is missing, unreadable or too large',
|
|
606
|
+
1
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
try {
|
|
610
|
+
return JSON.parse(text);
|
|
611
|
+
} catch {
|
|
612
|
+
throw new ExperimentSurfaceError(
|
|
613
|
+
'result_invalid',
|
|
614
|
+
'Saved experiment result is not valid JSON',
|
|
615
|
+
EXPERIMENT_EXIT_CODES.invalid
|
|
616
|
+
);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
function uniqueChoices<T extends string>(requested: T[] | undefined, allowed: readonly T[]): T[] {
|
|
621
|
+
if (requested === undefined || requested.length === 0) return [...allowed];
|
|
622
|
+
if (requested.some((choice) => !allowed.includes(choice))) {
|
|
623
|
+
throw new ExperimentSurfaceError(
|
|
624
|
+
'report_option_invalid',
|
|
625
|
+
`Choose from: ${allowed.join(', ')}`,
|
|
626
|
+
EXPERIMENT_EXIT_CODES.invalid
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
return allowed.filter((choice) => requested.includes(choice));
|
|
630
|
+
}
|
|
@@ -46,7 +46,12 @@ export class CircuitBreaker {
|
|
|
46
46
|
private halfOpenSuccesses = 0;
|
|
47
47
|
private eventHandlers: CircuitBreakerEventHandler[] = [];
|
|
48
48
|
|
|
49
|
-
|
|
49
|
+
/**
|
|
50
|
+
* @param config Circuit breaker settings. Every field is optional; omitted fields default to
|
|
51
|
+
* `enabled: true`, `threshold: 5`, `windowMs: 60000`, `cooldownMs: 300000` and
|
|
52
|
+
* `halfOpenRequests: 3`.
|
|
53
|
+
*/
|
|
54
|
+
constructor(config: Partial<CircuitBreakerConfig>) {
|
|
50
55
|
this.config = {
|
|
51
56
|
enabled: config.enabled ?? true,
|
|
52
57
|
threshold: config.threshold ?? 5,
|
package/src/index.ts
CHANGED
|
@@ -506,3 +506,55 @@ export type {
|
|
|
506
506
|
WorkflowReportRow,
|
|
507
507
|
WorkflowReportSection,
|
|
508
508
|
} from '@artemiskit/reports';
|
|
509
|
+
|
|
510
|
+
// Comparative experiments: one engine behind `akit experiment validate | run | report`.
|
|
511
|
+
export {
|
|
512
|
+
EXPERIMENT_EXIT_CODES,
|
|
513
|
+
EXPERIMENT_REPORT_FORMATS,
|
|
514
|
+
EXPERIMENT_REPORT_VIEWS,
|
|
515
|
+
ExperimentSurfaceError,
|
|
516
|
+
LOCAL_ENDPOINT_PLACEHOLDER_API_KEY,
|
|
517
|
+
LOOPBACK_REQUIRED_MESSAGE,
|
|
518
|
+
createExperimentFixtureClient,
|
|
519
|
+
createExperimentLiveAuthorization,
|
|
520
|
+
describeConcurrency,
|
|
521
|
+
describeTargetDeployment,
|
|
522
|
+
experimentRunStatus,
|
|
523
|
+
formatExperimentCaseSummary,
|
|
524
|
+
formatExperimentRunSummary,
|
|
525
|
+
reportExperimentResult,
|
|
526
|
+
runExperimentFile,
|
|
527
|
+
summarizeExperimentCases,
|
|
528
|
+
validateExperimentFile,
|
|
529
|
+
} from './experiment';
|
|
530
|
+
export type {
|
|
531
|
+
CreateExperimentLiveAuthorizationOptions,
|
|
532
|
+
ExperimentCheck,
|
|
533
|
+
ExperimentCheckStatus,
|
|
534
|
+
ExperimentCheckSubject,
|
|
535
|
+
ExperimentEvidenceFile,
|
|
536
|
+
ExperimentExitCode,
|
|
537
|
+
ExperimentFileRunOutcome,
|
|
538
|
+
ExperimentReportFile,
|
|
539
|
+
ExperimentRunStatus,
|
|
540
|
+
ExperimentValidation,
|
|
541
|
+
ReportExperimentResultOptions,
|
|
542
|
+
RunExperimentFileOptions,
|
|
543
|
+
SummarizeExperimentCasesOptions,
|
|
544
|
+
} from './experiment';
|
|
545
|
+
export type {
|
|
546
|
+
ExperimentCaseCounts,
|
|
547
|
+
ExperimentCaseGroup,
|
|
548
|
+
ExperimentCaseLevelAvailable,
|
|
549
|
+
ExperimentCaseLevelSummary,
|
|
550
|
+
ExperimentCaseLevelUnavailable,
|
|
551
|
+
ExperimentCaseUnavailableReason,
|
|
552
|
+
} from '@artemiskit/reports';
|
|
553
|
+
export type {
|
|
554
|
+
ExperimentLiveAuthorization,
|
|
555
|
+
ExperimentManifest,
|
|
556
|
+
ExperimentRunResult,
|
|
557
|
+
ExperimentTarget,
|
|
558
|
+
ExperimentTask,
|
|
559
|
+
} from '@artemiskit/core';
|
|
560
|
+
export type { ExperimentReportFormat, ExperimentReportView } from '@artemiskit/reports';
|