@artemiskit/sdk 0.6.3 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/dist/__tests__/experiment-fixtures.d.ts +44 -0
  3. package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
  4. package/dist/builders/index.js +19 -66
  5. package/dist/contracts/index.js +3 -50
  6. package/dist/experiment-files.d.ts +54 -0
  7. package/dist/experiment-files.d.ts.map +1 -0
  8. package/dist/experiment-fixture.d.ts +20 -0
  9. package/dist/experiment-fixture.d.ts.map +1 -0
  10. package/dist/experiment-summary.d.ts +44 -0
  11. package/dist/experiment-summary.d.ts.map +1 -0
  12. package/dist/experiment-validation.d.ts +48 -0
  13. package/dist/experiment-validation.d.ts.map +1 -0
  14. package/dist/experiment.d.ts +162 -0
  15. package/dist/experiment.d.ts.map +1 -0
  16. package/dist/guardian/circuit-breaker.d.ts +6 -1
  17. package/dist/guardian/circuit-breaker.d.ts.map +1 -1
  18. package/dist/index.d.ts +5 -0
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +2557 -1863
  21. package/dist/matchers/index.js +13 -60
  22. package/dist/matchers/jest.js +0 -47
  23. package/dist/matchers/vitest.js +0 -47
  24. package/dist/utils/index.js +11 -58
  25. package/package.json +6 -5
  26. package/src/__tests__/artemiskit.test.ts +1 -2
  27. package/src/__tests__/builders.test.ts +0 -2
  28. package/src/__tests__/circuit-breaker.test.ts +1 -1
  29. package/src/__tests__/compare.test.ts +1 -1
  30. package/src/__tests__/contracts.test.ts +1 -1
  31. package/src/__tests__/experiment-case-level.test.ts +212 -0
  32. package/src/__tests__/experiment-fixtures.ts +194 -0
  33. package/src/__tests__/experiment.test.ts +1262 -0
  34. package/src/__tests__/guardian-modes.test.ts +2 -2
  35. package/src/__tests__/guardrails.test.ts +1 -1
  36. package/src/__tests__/matchers.test.ts +1 -1
  37. package/src/__tests__/semantic-validator.test.ts +15 -10
  38. package/src/__tests__/validate.test.ts +0 -4
  39. package/src/__tests__/workflow-recovery.test.ts +7 -3
  40. package/src/__tests__/workflow-report.test.ts +6 -3
  41. package/src/__tests__/workflow.test.ts +11 -5
  42. package/src/experiment-files.ts +190 -0
  43. package/src/experiment-fixture.ts +70 -0
  44. package/src/experiment-summary.ts +172 -0
  45. package/src/experiment-validation.ts +426 -0
  46. package/src/experiment.ts +630 -0
  47. package/src/guardian/circuit-breaker.ts +6 -1
  48. package/src/index.ts +52 -0
  49. package/tsconfig.typecheck.json +12 -0
@@ -0,0 +1,630 @@
1
+ /**
2
+ * Comparative experiment surfaces: validate, run and report.
3
+ *
4
+ * This module is the one engine behind both the SDK functions and the `akit experiment` CLI.
5
+ * Fixture experiments bind every target to a deterministic in-memory client and make no network
6
+ * calls. Live experiments additionally require an explicit `ExperimentLiveAuthorization`.
7
+ */
8
+
9
+ import { randomBytes } from 'node:crypto';
10
+ import path from 'node:path';
11
+ import {
12
+ type AdapterConfig,
13
+ type ContentIdentity,
14
+ type ExperimentAttemptExecutor,
15
+ type ExperimentLiveAuthorization,
16
+ type ExperimentRunResult,
17
+ type ExperimentTarget,
18
+ type ExperimentTaskKind,
19
+ type ModelClient,
20
+ createAdapter,
21
+ createScenarioExperimentExecutor,
22
+ createWorkflowExperimentExecutor,
23
+ isLoopbackEndpoint,
24
+ runExperiment,
25
+ } from '@artemiskit/core';
26
+ import {
27
+ type ExperimentCaseEvidenceFiles,
28
+ type ExperimentCaseLevelSummary,
29
+ type ExperimentReportFormat,
30
+ type ExperimentReportView,
31
+ createExperimentCaseSummary,
32
+ createExperimentReport,
33
+ listExperimentCaseEvidenceDigests,
34
+ renderExperimentReportHTML,
35
+ renderExperimentReportMarkdown,
36
+ } from '@artemiskit/reports';
37
+ import {
38
+ EXPERIMENT_EVIDENCE_DIR,
39
+ ExperimentSurfaceError,
40
+ MAX_EXPERIMENT_RESULT_BYTES,
41
+ defaultExperimentOutputDir,
42
+ prepareEmptyDirectory,
43
+ readBoundedText,
44
+ readExperimentCaseEvidence,
45
+ saveAttemptEvidence,
46
+ saveExperimentResult,
47
+ writeNewFile,
48
+ } from './experiment-files';
49
+ import { createExperimentFixtureClient } from './experiment-fixture';
50
+ import {
51
+ EXPERIMENT_EXIT_CODES,
52
+ type ExperimentExitCode,
53
+ type ExperimentRunStatus,
54
+ experimentRunStatus,
55
+ } from './experiment-summary';
56
+ import {
57
+ type ExperimentValidation,
58
+ isReasoningEffort,
59
+ validateExperimentFile,
60
+ } from './experiment-validation';
61
+
62
+ export { ExperimentSurfaceError } from './experiment-files';
63
+ export {
64
+ EXPERIMENT_EXIT_CODES,
65
+ describeConcurrency,
66
+ describeTargetDeployment,
67
+ experimentRunStatus,
68
+ formatExperimentCaseSummary,
69
+ formatExperimentRunSummary,
70
+ } from './experiment-summary';
71
+ export type { ExperimentExitCode, ExperimentRunStatus } from './experiment-summary';
72
+ export { LOOPBACK_REQUIRED_MESSAGE, validateExperimentFile } from './experiment-validation';
73
+ export type {
74
+ ExperimentCheck,
75
+ ExperimentCheckStatus,
76
+ ExperimentCheckSubject,
77
+ ExperimentValidation,
78
+ } from './experiment-validation';
79
+ export { createExperimentFixtureClient } from './experiment-fixture';
80
+ export type { ExperimentFixtureClientOptions } from './experiment-fixture';
81
+
82
+ /**
83
+ * Non-secret API key sent to loopback OpenAI-compatible servers (such as Ollama) that ignore
84
+ * authentication. It keeps the OpenAI client from falling back to an ambient `OPENAI_API_KEY`,
85
+ * so a real key never reaches a local endpoint unless the host configures one explicitly.
86
+ */
87
+ export const LOCAL_ENDPOINT_PLACEHOLDER_API_KEY = 'artemiskit-local-no-key';
88
+
89
+ const DECISION_ID = /^[a-z0-9][a-z0-9._:-]{0,63}$/;
90
+ const MAX_APPROVER_LENGTH = 128;
91
+ const MAX_REASON_LENGTH = 256;
92
+ const MAX_CONCURRENCY = 8;
93
+ type ApprovedConcurrency = NonNullable<ExperimentLiveAuthorization['approved_concurrency']>;
94
+
95
+ export interface CreateExperimentLiveAuthorizationOptions {
96
+ /** Who approved the live run. */
97
+ approvedBy: string;
98
+ /** Why the live run is approved. */
99
+ reason: string;
100
+ /** Bounded decision ID; generated when omitted. */
101
+ decisionId?: string;
102
+ /** Decision time; defaults to now. */
103
+ decidedAt?: Date;
104
+ /**
105
+ * Highest experiment concurrency this approval covers (integer 1..8). Required to run a
106
+ * live experiment that declares concurrency above 1; omitted means serial live calls only.
107
+ */
108
+ maxConcurrency?: number;
109
+ }
110
+
111
+ /**
112
+ * Build the explicit live-run approval record that `runExperimentFile` requires for
113
+ * `mode: live` experiments. Throws `ExperimentSurfaceError` (exit code 3) on invalid input.
114
+ */
115
+ export function createExperimentLiveAuthorization(
116
+ options: CreateExperimentLiveAuthorizationOptions
117
+ ): ExperimentLiveAuthorization {
118
+ const decidedAt = options.decidedAt ?? new Date();
119
+ const authorization: ExperimentLiveAuthorization = {
120
+ approved: true,
121
+ decision_id: options.decisionId ?? generateDecisionId(decidedAt),
122
+ decided_at: decidedAt.toISOString(),
123
+ approved_by: String(options.approvedBy ?? '').trim(),
124
+ reason: String(options.reason ?? '').trim(),
125
+ ...(options.maxConcurrency === undefined
126
+ ? {}
127
+ : { approved_concurrency: options.maxConcurrency as ApprovedConcurrency }),
128
+ };
129
+ if (!isValidLiveAuthorization(authorization)) {
130
+ throw new ExperimentSurfaceError(
131
+ 'authorization_invalid',
132
+ `Live approval needs an approver (1-${MAX_APPROVER_LENGTH} characters), a reason (1-${MAX_REASON_LENGTH} characters), a decision ID matching ${DECISION_ID} and, when given, a maximum concurrency from 1 to ${MAX_CONCURRENCY}`,
133
+ EXPERIMENT_EXIT_CODES.unauthorized
134
+ );
135
+ }
136
+ return authorization;
137
+ }
138
+
139
+ function generateDecisionId(now: Date): string {
140
+ const stamp = now.toISOString().replace(/\D/g, '').slice(0, 14);
141
+ return `live-${stamp}-${randomBytes(4).toString('hex')}`;
142
+ }
143
+
144
+ function isValidLiveAuthorization(value: unknown): value is ExperimentLiveAuthorization {
145
+ const candidate = value as Partial<ExperimentLiveAuthorization> | undefined;
146
+ return (
147
+ typeof candidate === 'object' &&
148
+ candidate !== null &&
149
+ candidate.approved === true &&
150
+ typeof candidate.decision_id === 'string' &&
151
+ DECISION_ID.test(candidate.decision_id) &&
152
+ typeof candidate.decided_at === 'string' &&
153
+ !Number.isNaN(Date.parse(candidate.decided_at)) &&
154
+ boundedText(candidate.approved_by, MAX_APPROVER_LENGTH) &&
155
+ boundedText(candidate.reason, MAX_REASON_LENGTH) &&
156
+ (candidate.approved_concurrency === undefined ||
157
+ (Number.isSafeInteger(candidate.approved_concurrency) &&
158
+ candidate.approved_concurrency >= 1 &&
159
+ candidate.approved_concurrency <= MAX_CONCURRENCY))
160
+ );
161
+ }
162
+
163
+ function boundedText(value: unknown, maximum: number): boolean {
164
+ return typeof value === 'string' && value.trim().length > 0 && value.trim().length <= maximum;
165
+ }
166
+
167
+ export interface RunExperimentFileOptions {
168
+ /** Experiment YAML or JSON file. Task sources resolve relative to its directory. */
169
+ experiment: string;
170
+ /**
171
+ * New or empty directory for `result.json` and `evidence/`. Defaults to
172
+ * `<cwd>/artemis-output/experiments/<experiment-id>/<timestamp>/`.
173
+ */
174
+ outputDir?: string;
175
+ /** Base for the default output directory; defaults to `process.cwd()`. */
176
+ cwd?: string;
177
+ /** Required for `mode: live`; ignored for fixture experiments. */
178
+ liveAuthorization?: ExperimentLiveAuthorization;
179
+ /**
180
+ * Live mode only: provider transport settings for a target (API key, timeout, organization).
181
+ * The target's `settings.base_url`, provider and model always take precedence.
182
+ */
183
+ resolveTargetConfig?: (
184
+ target: ExperimentTarget
185
+ ) => Partial<AdapterConfig> | undefined | Promise<Partial<AdapterConfig> | undefined>;
186
+ /** Live mode only: supply your own client per target instead of the built-in adapters. */
187
+ createClient?: (target: ExperimentTarget) => ModelClient | Promise<ModelClient>;
188
+ /**
189
+ * Per-case model request timeout in milliseconds for scenario tasks. Live targets bound
190
+ * through the built-in adapters also get an HTTP timeout of at least this value. Workflow
191
+ * tasks keep their own declared `timeout_ms`.
192
+ */
193
+ timeoutMs?: number;
194
+ }
195
+
196
+ /**
197
+ * Saved evidence for one executed attempt: the scenario run manifest, or the metadata-only
198
+ * workflow record, as canonical JSON named by its SHA-256.
199
+ */
200
+ export interface ExperimentEvidenceFile {
201
+ coordinate_id: string;
202
+ attempt_number: number;
203
+ sha256: string;
204
+ path: string;
205
+ }
206
+
207
+ export interface ExperimentFileRunOutcome {
208
+ status: ExperimentRunStatus;
209
+ /** Same value `akit experiment run` exits with. */
210
+ exitCode: ExperimentExitCode;
211
+ validation: ExperimentValidation;
212
+ /** Present once the run executed. */
213
+ result?: ExperimentRunResult;
214
+ outputDir?: string;
215
+ resultPath?: string;
216
+ evidence: ExperimentEvidenceFile[];
217
+ /**
218
+ * Present once the run executed: individual scenario cases from the saved, SHA-256-verified
219
+ * evidence, or why they are unavailable. Coordinate-level counts stay in `result.summaries`.
220
+ */
221
+ caseLevel?: ExperimentCaseLevelSummary;
222
+ /** Explanation for refusals (`invalid`, `unauthorized`). */
223
+ message?: string;
224
+ }
225
+
226
+ /**
227
+ * Validate, then run an experiment file and save the V1 result plus per-attempt evidence.
228
+ *
229
+ * Each task runs through the built-in executor for its kind: `scenario_evaluation` tasks
230
+ * through the scenario runner and `agent_workflow` tasks through the native workflow engine
231
+ * (simulated or sandbox environment, network denied). Fixture targets never call a tool, so
232
+ * fixture workflow outcomes describe the pipeline, never a model.
233
+ *
234
+ * Refusals (`invalid`, `unauthorized`) return before any client is created or any attempt runs.
235
+ * Output and persistence failures throw `ExperimentSurfaceError` (exit code 1).
236
+ *
237
+ * A target's generation settings (`temperature`, `top_p`, `presence_penalty`,
238
+ * `max_output_tokens`) replace each task's own values for that target's requests, and every
239
+ * request carries the coordinate's seed. `llm_grader` and embedding requests keep their own
240
+ * sampling and output length. `reasoning_effort` is adapter configuration, so it also applies to
241
+ * `llm_grader` requests made through the target's client. Workflow tasks get the same settings
242
+ * and seed on every workflow model request. Each saved run manifest records the values sent in
243
+ * `resolved_config`; each saved workflow record lists them in `configuration.generation`.
244
+ */
245
+ export async function runExperimentFile(
246
+ options: RunExperimentFileOptions
247
+ ): Promise<ExperimentFileRunOutcome> {
248
+ const validation = await validateExperimentFile(options.experiment);
249
+ if (!validation.valid || !validation.manifest) {
250
+ return refusal('invalid', validation, 'Experiment validation failed; nothing was executed');
251
+ }
252
+ const manifest = validation.manifest;
253
+ const live = manifest.mode === 'live';
254
+ if (live && !isValidLiveAuthorization(options.liveAuthorization)) {
255
+ return refusal(
256
+ 'unauthorized',
257
+ validation,
258
+ 'Live experiments require an explicit approval (approver, reason, decision ID); nothing was executed'
259
+ );
260
+ }
261
+ const approvedConcurrency = options.liveAuthorization?.approved_concurrency ?? 1;
262
+ if (live && manifest.concurrency > approvedConcurrency) {
263
+ return refusal(
264
+ 'unauthorized',
265
+ validation,
266
+ `The experiment declares concurrency ${manifest.concurrency} but the live approval covers ${approvedConcurrency}; approve it explicitly (--max-concurrency ${manifest.concurrency}); nothing was executed`
267
+ );
268
+ }
269
+
270
+ const outputDir = path.resolve(
271
+ options.outputDir ??
272
+ defaultExperimentOutputDir(manifest.id, options.cwd ?? process.cwd(), new Date())
273
+ );
274
+ await prepareEmptyDirectory(outputDir);
275
+
276
+ const evidence: ExperimentEvidenceFile[] = [];
277
+ const saveEvidence = async (
278
+ attempt: { coordinate_id: string; attempt_number: number; artifact: ContentIdentity },
279
+ content: unknown
280
+ ) => {
281
+ const file = await saveAttemptEvidence(outputDir, content, attempt.artifact.digest);
282
+ evidence.push({
283
+ coordinate_id: attempt.coordinate_id,
284
+ attempt_number: attempt.attempt_number,
285
+ sha256: attempt.artifact.digest,
286
+ path: file,
287
+ });
288
+ };
289
+ const executor = routeByTaskKind({
290
+ scenario_evaluation: createScenarioExperimentExecutor({
291
+ root: validation.root,
292
+ repetitions: manifest.repetitions,
293
+ timeoutMs: options.timeoutMs,
294
+ createClient: live
295
+ ? (target) => bindLiveClient(target, options)
296
+ : (target) => createExperimentFixtureClient(target),
297
+ onEvidence: (record) => saveEvidence(record, record.manifest),
298
+ }),
299
+ agent_workflow: createWorkflowExperimentExecutor({
300
+ root: validation.root,
301
+ createClient: live
302
+ ? (target) => bindLiveClient(target, options)
303
+ : (target) => createExperimentFixtureClient(target, { toolUse: true }),
304
+ onEvidence: (record) => saveEvidence(record, record.record),
305
+ }),
306
+ });
307
+
308
+ const result = await runExperiment(manifest, {
309
+ execute_attempt: executor,
310
+ ...(live ? { live_authorization: pickAuthorization(options.liveAuthorization) } : {}),
311
+ });
312
+ sortEvidence(evidence, result);
313
+ const resultPath = await saveExperimentResult(outputDir, result);
314
+ const status = experimentRunStatus(result);
315
+ return {
316
+ status,
317
+ exitCode: EXPERIMENT_EXIT_CODES[status],
318
+ validation,
319
+ result,
320
+ outputDir,
321
+ resultPath,
322
+ evidence,
323
+ caseLevel: await savedRunCaseLevel(result, outputDir),
324
+ };
325
+ }
326
+
327
+ /**
328
+ * Case-level summary read back from the files just saved, so a run reports exactly what a
329
+ * later `report` of the same directory would. A result the report validator rejects (never
330
+ * expected from the runner) yields an unavailable summary rather than failing the saved run.
331
+ */
332
+ async function savedRunCaseLevel(
333
+ result: ExperimentRunResult,
334
+ outputDir: string
335
+ ): Promise<ExperimentCaseLevelSummary> {
336
+ const caseEvidence = await loadCaseEvidence(
337
+ result,
338
+ path.join(outputDir, EXPERIMENT_EVIDENCE_DIR)
339
+ );
340
+ try {
341
+ return createExperimentCaseSummary(result, { caseEvidence });
342
+ } catch {
343
+ return {
344
+ availability: 'unavailable',
345
+ reason: 'result_invalid',
346
+ detail:
347
+ 'Case-level summary unavailable: the saved result did not pass report validation. Coordinate-level counts are unaffected.',
348
+ affectedEvidence: [],
349
+ affectedEvidenceCount: 0,
350
+ };
351
+ }
352
+ }
353
+
354
+ /** Evidence bytes for the attempts a case-level summary reads; `undefined` without a directory. */
355
+ async function loadCaseEvidence(
356
+ input: unknown,
357
+ evidenceDir: string | undefined
358
+ ): Promise<ExperimentCaseEvidenceFiles | undefined> {
359
+ if (evidenceDir === undefined) return undefined;
360
+ return readExperimentCaseEvidence(evidenceDir, listExperimentCaseEvidenceDigests(input));
361
+ }
362
+
363
+ /** Explicit evidence directory, else `evidence/` beside a result file; none for a parsed result. */
364
+ function caseEvidenceDir(
365
+ result: string | ExperimentRunResult,
366
+ evidenceDir: string | undefined
367
+ ): string | undefined {
368
+ if (evidenceDir !== undefined) return path.resolve(evidenceDir);
369
+ if (typeof result !== 'string') return undefined;
370
+ return path.join(path.dirname(path.resolve(result)), EXPERIMENT_EVIDENCE_DIR);
371
+ }
372
+
373
+ /** One attempt executor that hands each task, and its demand request, to its kind's executor. */
374
+ function routeByTaskKind(
375
+ executors: Record<ExperimentTaskKind, ExperimentAttemptExecutor>
376
+ ): ExperimentAttemptExecutor {
377
+ const execute: ExperimentAttemptExecutor = (input) => executors[input.task.kind](input);
378
+ execute.reserve = (request) => executors[request.task.kind].reserve?.(request);
379
+ return execute;
380
+ }
381
+
382
+ function refusal(
383
+ status: 'invalid' | 'unauthorized',
384
+ validation: ExperimentValidation,
385
+ message: string
386
+ ): ExperimentFileRunOutcome {
387
+ return {
388
+ status,
389
+ exitCode: EXPERIMENT_EXIT_CODES[status],
390
+ validation,
391
+ evidence: [],
392
+ message,
393
+ };
394
+ }
395
+
396
+ /** Core parses the authorization strictly; pass exactly the declared fields. */
397
+ function pickAuthorization(value: ExperimentLiveAuthorization | undefined) {
398
+ const authorization = value as ExperimentLiveAuthorization;
399
+ return {
400
+ approved: authorization.approved,
401
+ decision_id: authorization.decision_id,
402
+ decided_at: authorization.decided_at,
403
+ approved_by: authorization.approved_by,
404
+ reason: authorization.reason,
405
+ ...(authorization.approved_concurrency === undefined
406
+ ? {}
407
+ : { approved_concurrency: authorization.approved_concurrency }),
408
+ };
409
+ }
410
+
411
+ /**
412
+ * Concurrent attempts save evidence in completion order; list it in matrix order, then by
413
+ * attempt number, so the evidence index is deterministic.
414
+ */
415
+ function sortEvidence(evidence: ExperimentEvidenceFile[], result: ExperimentRunResult): void {
416
+ const order = new Map(
417
+ result.results.map((item, index) => [item.coordinate.coordinate_id, index])
418
+ );
419
+ const position = (item: ExperimentEvidenceFile) =>
420
+ order.get(item.coordinate_id) ?? Number.MAX_SAFE_INTEGER;
421
+ evidence.sort(
422
+ (left, right) => position(left) - position(right) || left.attempt_number - right.attempt_number
423
+ );
424
+ }
425
+
426
+ /**
427
+ * Bind a live target through the adapter registry. Loopback targets never inherit an ambient
428
+ * API key: they get the host's explicit key or a non-secret placeholder.
429
+ */
430
+ async function bindLiveClient(
431
+ target: ExperimentTarget,
432
+ options: RunExperimentFileOptions
433
+ ): Promise<ModelClient> {
434
+ if (options.createClient) return options.createClient(target);
435
+ const configured = (await options.resolveTargetConfig?.(target)) ?? {};
436
+ const settingsBaseUrl = target.settings?.base_url;
437
+ const baseUrl = typeof settingsBaseUrl === 'string' ? settingsBaseUrl : configured.baseUrl;
438
+ const loopback = typeof baseUrl === 'string' && isLoopbackEndpoint(baseUrl);
439
+ if (target.deployment?.cost_basis === 'not_applicable' && !loopback) {
440
+ throw new Error('cost_not_applicable_requires_loopback');
441
+ }
442
+ return createAdapter({
443
+ ...configured,
444
+ provider: target.provider,
445
+ defaultModel: target.model,
446
+ baseUrl,
447
+ apiKey: loopback
448
+ ? (configured.apiKey ?? LOCAL_ENDPOINT_PLACEHOLDER_API_KEY)
449
+ : configured.apiKey,
450
+ maxRetries: 0,
451
+ ...transportTimeoutConfig(configured.timeout, options.timeoutMs),
452
+ ...reasoningEffortConfig(target),
453
+ } as AdapterConfig);
454
+ }
455
+
456
+ /**
457
+ * The adapter's HTTP timeout must never be shorter than the run's per-case timeout, or long
458
+ * local generations are aborted by the transport (the OpenAI adapter defaults to 60 s) before
459
+ * the declared per-case limit applies. Without a per-case timeout the adapter default stands.
460
+ */
461
+ function transportTimeoutConfig(
462
+ configuredMs: number | undefined,
463
+ perCaseMs: number | undefined
464
+ ): { timeout?: number } {
465
+ if (perCaseMs === undefined) return {};
466
+ return { timeout: Math.max(configuredMs ?? 0, perCaseMs) };
467
+ }
468
+
469
+ /** Map a target's declared `settings.reasoning_effort` onto the adapter option. */
470
+ function reasoningEffortConfig(target: ExperimentTarget): { reasoningEffort?: string } {
471
+ const effort = target.settings?.reasoning_effort;
472
+ if (effort === undefined) return {};
473
+ if (!isReasoningEffort(effort)) throw new Error('invalid_reasoning_effort');
474
+ return { reasoningEffort: effort };
475
+ }
476
+
477
+ export interface ReportExperimentResultOptions {
478
+ /** Path to a saved `result.json`, or the parsed V1 result. */
479
+ result: string | ExperimentRunResult;
480
+ /** Views to generate; defaults to all three. */
481
+ views?: ExperimentReportView[];
482
+ /** Formats to generate; defaults to HTML and Markdown. */
483
+ formats?: ExperimentReportFormat[];
484
+ /** New or empty directory to write `<view>.<html|md>` files into; omit to only return content. */
485
+ outputDir?: string;
486
+ /**
487
+ * Directory of saved per-attempt evidence for the case-level summary. Defaults to `evidence/`
488
+ * beside a result file; a parsed result without this option reports case level unavailable.
489
+ */
490
+ evidenceDir?: string;
491
+ }
492
+
493
+ export interface ExperimentReportFile {
494
+ view: ExperimentReportView;
495
+ format: ExperimentReportFormat;
496
+ content: string;
497
+ /** Written file, when `outputDir` was given. */
498
+ path?: string;
499
+ }
500
+
501
+ export const EXPERIMENT_REPORT_VIEWS: readonly ExperimentReportView[] = [
502
+ 'technical',
503
+ 'executive',
504
+ 'comprehensive',
505
+ ];
506
+ export const EXPERIMENT_REPORT_FORMATS: readonly ExperimentReportFormat[] = ['html', 'markdown'];
507
+
508
+ /**
509
+ * Generate experiment reports offline from a saved result. Loads no task, provider, credential
510
+ * or network. Throws `ExperimentSurfaceError`: exit code 1 for unreadable input or unwritable
511
+ * output, 2 for an invalid or unsupported saved result.
512
+ */
513
+ export async function reportExperimentResult(
514
+ options: ReportExperimentResultOptions
515
+ ): Promise<ExperimentReportFile[]> {
516
+ const views = uniqueChoices(options.views, EXPERIMENT_REPORT_VIEWS);
517
+ const formats = uniqueChoices(options.formats, EXPERIMENT_REPORT_FORMATS);
518
+ const input =
519
+ typeof options.result === 'string' ? await readResult(options.result) : options.result;
520
+ const caseEvidence = await loadCaseEvidence(
521
+ input,
522
+ caseEvidenceDir(options.result, options.evidenceDir)
523
+ );
524
+
525
+ let reports: ExperimentReportFile[];
526
+ try {
527
+ const report = createExperimentReport(input, { caseEvidence });
528
+ reports = views.flatMap((view) =>
529
+ formats.map((format) => ({
530
+ view,
531
+ format,
532
+ content:
533
+ format === 'html'
534
+ ? renderExperimentReportHTML(report, { view })
535
+ : renderExperimentReportMarkdown(report, { view }),
536
+ }))
537
+ );
538
+ } catch {
539
+ throw new ExperimentSurfaceError(
540
+ 'result_invalid',
541
+ 'Invalid, unsupported or ambiguous saved experiment result; no report generated',
542
+ EXPERIMENT_EXIT_CODES.invalid
543
+ );
544
+ }
545
+
546
+ if (options.outputDir !== undefined) {
547
+ const directory = path.resolve(options.outputDir);
548
+ await prepareEmptyDirectory(directory);
549
+ for (const report of reports) {
550
+ const file = path.join(
551
+ directory,
552
+ `${report.view}.${report.format === 'html' ? 'html' : 'md'}`
553
+ );
554
+ try {
555
+ await writeNewFile(file, report.content);
556
+ } catch {
557
+ throw new ExperimentSurfaceError('report_unwritable', 'Unable to write report file', 1);
558
+ }
559
+ report.path = file;
560
+ }
561
+ }
562
+ return reports;
563
+ }
564
+
565
+ export interface SummarizeExperimentCasesOptions {
566
+ /** Path to a saved `result.json`, or the parsed V1 result. */
567
+ result: string | ExperimentRunResult;
568
+ /** Saved per-attempt evidence directory; defaults to `evidence/` beside a result file. */
569
+ evidenceDir?: string;
570
+ }
571
+
572
+ /**
573
+ * Case-level summary of a saved result, offline: individual scenario cases from each
574
+ * coordinate's terminal attempt evidence, verified against the recorded SHA-256. Missing or
575
+ * altered evidence yields an `unavailable` summary with its reason; an invalid result throws
576
+ * `ExperimentSurfaceError` (exit code 1 unreadable, 2 invalid).
577
+ */
578
+ export async function summarizeExperimentCases(
579
+ options: SummarizeExperimentCasesOptions
580
+ ): Promise<ExperimentCaseLevelSummary> {
581
+ const input =
582
+ typeof options.result === 'string' ? await readResult(options.result) : options.result;
583
+ const caseEvidence = await loadCaseEvidence(
584
+ input,
585
+ caseEvidenceDir(options.result, options.evidenceDir)
586
+ );
587
+ try {
588
+ return createExperimentCaseSummary(input, { caseEvidence });
589
+ } catch {
590
+ throw new ExperimentSurfaceError(
591
+ 'result_invalid',
592
+ 'Invalid, unsupported or ambiguous saved experiment result; no case-level summary',
593
+ EXPERIMENT_EXIT_CODES.invalid
594
+ );
595
+ }
596
+ }
597
+
598
+ async function readResult(file: string): Promise<unknown> {
599
+ let text: string;
600
+ try {
601
+ text = await readBoundedText(path.resolve(file), MAX_EXPERIMENT_RESULT_BYTES);
602
+ } catch {
603
+ throw new ExperimentSurfaceError(
604
+ 'result_unreadable',
605
+ 'Saved experiment result is missing, unreadable or too large',
606
+ 1
607
+ );
608
+ }
609
+ try {
610
+ return JSON.parse(text);
611
+ } catch {
612
+ throw new ExperimentSurfaceError(
613
+ 'result_invalid',
614
+ 'Saved experiment result is not valid JSON',
615
+ EXPERIMENT_EXIT_CODES.invalid
616
+ );
617
+ }
618
+ }
619
+
620
+ function uniqueChoices<T extends string>(requested: T[] | undefined, allowed: readonly T[]): T[] {
621
+ if (requested === undefined || requested.length === 0) return [...allowed];
622
+ if (requested.some((choice) => !allowed.includes(choice))) {
623
+ throw new ExperimentSurfaceError(
624
+ 'report_option_invalid',
625
+ `Choose from: ${allowed.join(', ')}`,
626
+ EXPERIMENT_EXIT_CODES.invalid
627
+ );
628
+ }
629
+ return allowed.filter((choice) => requested.includes(choice));
630
+ }
@@ -46,7 +46,12 @@ export class CircuitBreaker {
46
46
  private halfOpenSuccesses = 0;
47
47
  private eventHandlers: CircuitBreakerEventHandler[] = [];
48
48
 
49
- constructor(config: CircuitBreakerConfig) {
49
+ /**
50
+ * @param config Circuit breaker settings. Every field is optional; omitted fields default to
51
+ * `enabled: true`, `threshold: 5`, `windowMs: 60000`, `cooldownMs: 300000` and
52
+ * `halfOpenRequests: 3`.
53
+ */
54
+ constructor(config: Partial<CircuitBreakerConfig>) {
50
55
  this.config = {
51
56
  enabled: config.enabled ?? true,
52
57
  threshold: config.threshold ?? 5,
package/src/index.ts CHANGED
@@ -506,3 +506,55 @@ export type {
506
506
  WorkflowReportRow,
507
507
  WorkflowReportSection,
508
508
  } from '@artemiskit/reports';
509
+
510
+ // Comparative experiments: one engine behind `akit experiment validate | run | report`.
511
+ export {
512
+ EXPERIMENT_EXIT_CODES,
513
+ EXPERIMENT_REPORT_FORMATS,
514
+ EXPERIMENT_REPORT_VIEWS,
515
+ ExperimentSurfaceError,
516
+ LOCAL_ENDPOINT_PLACEHOLDER_API_KEY,
517
+ LOOPBACK_REQUIRED_MESSAGE,
518
+ createExperimentFixtureClient,
519
+ createExperimentLiveAuthorization,
520
+ describeConcurrency,
521
+ describeTargetDeployment,
522
+ experimentRunStatus,
523
+ formatExperimentCaseSummary,
524
+ formatExperimentRunSummary,
525
+ reportExperimentResult,
526
+ runExperimentFile,
527
+ summarizeExperimentCases,
528
+ validateExperimentFile,
529
+ } from './experiment';
530
+ export type {
531
+ CreateExperimentLiveAuthorizationOptions,
532
+ ExperimentCheck,
533
+ ExperimentCheckStatus,
534
+ ExperimentCheckSubject,
535
+ ExperimentEvidenceFile,
536
+ ExperimentExitCode,
537
+ ExperimentFileRunOutcome,
538
+ ExperimentReportFile,
539
+ ExperimentRunStatus,
540
+ ExperimentValidation,
541
+ ReportExperimentResultOptions,
542
+ RunExperimentFileOptions,
543
+ SummarizeExperimentCasesOptions,
544
+ } from './experiment';
545
+ export type {
546
+ ExperimentCaseCounts,
547
+ ExperimentCaseGroup,
548
+ ExperimentCaseLevelAvailable,
549
+ ExperimentCaseLevelSummary,
550
+ ExperimentCaseLevelUnavailable,
551
+ ExperimentCaseUnavailableReason,
552
+ } from '@artemiskit/reports';
553
+ export type {
554
+ ExperimentLiveAuthorization,
555
+ ExperimentManifest,
556
+ ExperimentRunResult,
557
+ ExperimentTarget,
558
+ ExperimentTask,
559
+ } from '@artemiskit/core';
560
+ export type { ExperimentReportFormat, ExperimentReportView } from '@artemiskit/reports';