@artemiskit/sdk 0.6.3 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/dist/__tests__/experiment-fixtures.d.ts +44 -0
  3. package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
  4. package/dist/builders/index.js +19 -66
  5. package/dist/contracts/index.js +3 -50
  6. package/dist/experiment-files.d.ts +54 -0
  7. package/dist/experiment-files.d.ts.map +1 -0
  8. package/dist/experiment-fixture.d.ts +20 -0
  9. package/dist/experiment-fixture.d.ts.map +1 -0
  10. package/dist/experiment-summary.d.ts +44 -0
  11. package/dist/experiment-summary.d.ts.map +1 -0
  12. package/dist/experiment-validation.d.ts +48 -0
  13. package/dist/experiment-validation.d.ts.map +1 -0
  14. package/dist/experiment.d.ts +162 -0
  15. package/dist/experiment.d.ts.map +1 -0
  16. package/dist/guardian/circuit-breaker.d.ts +6 -1
  17. package/dist/guardian/circuit-breaker.d.ts.map +1 -1
  18. package/dist/index.d.ts +5 -0
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +2557 -1863
  21. package/dist/matchers/index.js +13 -60
  22. package/dist/matchers/jest.js +0 -47
  23. package/dist/matchers/vitest.js +0 -47
  24. package/dist/utils/index.js +11 -58
  25. package/package.json +6 -5
  26. package/src/__tests__/artemiskit.test.ts +1 -2
  27. package/src/__tests__/builders.test.ts +0 -2
  28. package/src/__tests__/circuit-breaker.test.ts +1 -1
  29. package/src/__tests__/compare.test.ts +1 -1
  30. package/src/__tests__/contracts.test.ts +1 -1
  31. package/src/__tests__/experiment-case-level.test.ts +212 -0
  32. package/src/__tests__/experiment-fixtures.ts +194 -0
  33. package/src/__tests__/experiment.test.ts +1262 -0
  34. package/src/__tests__/guardian-modes.test.ts +2 -2
  35. package/src/__tests__/guardrails.test.ts +1 -1
  36. package/src/__tests__/matchers.test.ts +1 -1
  37. package/src/__tests__/semantic-validator.test.ts +15 -10
  38. package/src/__tests__/validate.test.ts +0 -4
  39. package/src/__tests__/workflow-recovery.test.ts +7 -3
  40. package/src/__tests__/workflow-report.test.ts +6 -3
  41. package/src/__tests__/workflow.test.ts +11 -5
  42. package/src/experiment-files.ts +190 -0
  43. package/src/experiment-fixture.ts +70 -0
  44. package/src/experiment-summary.ts +172 -0
  45. package/src/experiment-validation.ts +426 -0
  46. package/src/experiment.ts +630 -0
  47. package/src/guardian/circuit-breaker.ts +6 -1
  48. package/src/index.ts +52 -0
  49. package/tsconfig.typecheck.json +12 -0
@@ -2,8 +2,8 @@
2
2
  * Tests for Guardian mode normalization (v0.3.2+)
3
3
  */
4
4
 
5
- import { describe, expect, mock, test } from 'bun:test';
6
- import { Guardian, createGuardian, normalizeGuardianMode } from '../guardian';
5
+ import { describe, expect, test } from 'bun:test';
6
+ import { createGuardian, normalizeGuardianMode } from '../guardian';
7
7
 
8
8
  describe('Guardian Mode Normalization', () => {
9
9
  describe('normalizeGuardianMode', () => {
@@ -327,7 +327,7 @@ describe('Guardrails', () => {
327
327
  const guardrail = createCustomPatternGuardrail(['forbidden', 'prohibited']);
328
328
  const result = await guardrail('This is forbidden content');
329
329
  expect(result.passed).toBe(false);
330
- expect(result.violations[0].details.matchedText).toBe('forbidden');
330
+ expect(result.violations[0].details?.matchedText).toBe('forbidden');
331
331
  });
332
332
 
333
333
  test('should match custom regex patterns', async () => {
@@ -3,7 +3,7 @@
3
3
  * Tests for custom matchers
4
4
  */
5
5
 
6
- import { describe, expect, it } from 'vitest';
6
+ import { describe, expect, it } from 'bun:test';
7
7
  import {
8
8
  toAchieveRPS,
9
9
  toHaveDefenseRate,
@@ -8,7 +8,7 @@
8
8
  * - Error handling and edge cases
9
9
  */
10
10
 
11
- import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
11
+ import { describe, expect, it, mock } from 'bun:test';
12
12
 
13
13
  // Mock nanoid for consistent IDs in tests
14
14
  mock.module('nanoid', () => ({
@@ -17,7 +17,11 @@ mock.module('nanoid', () => ({
17
17
 
18
18
  import type { ModelClient } from '@artemiskit/core';
19
19
  import { SemanticValidator, createSemanticValidator } from '../guardian/semantic-validator';
20
- import type { ContentValidationConfig, ValidationCategory } from '../guardian/types';
20
+ import type {
21
+ ContentValidationConfig,
22
+ ValidationCategory,
23
+ ViolationSeverity,
24
+ } from '../guardian/types';
21
25
 
22
26
  // =============================================================================
23
27
  // Mock LLM Client
@@ -350,14 +354,15 @@ describe('SemanticValidator', () => {
350
354
  });
351
355
 
352
356
  describe('severity mapping', () => {
353
- const testCases: Array<{ category: ValidationCategory; expectedSeverity: string }> = [
354
- { category: 'prompt_injection', expectedSeverity: 'critical' },
355
- { category: 'jailbreak', expectedSeverity: 'critical' },
356
- { category: 'data_extraction', expectedSeverity: 'critical' },
357
- { category: 'pii_disclosure', expectedSeverity: 'high' },
358
- { category: 'role_manipulation', expectedSeverity: 'high' },
359
- { category: 'content_safety', expectedSeverity: 'high' },
360
- ];
357
+ const testCases: Array<{ category: ValidationCategory; expectedSeverity: ViolationSeverity }> =
358
+ [
359
+ { category: 'prompt_injection', expectedSeverity: 'critical' },
360
+ { category: 'jailbreak', expectedSeverity: 'critical' },
361
+ { category: 'data_extraction', expectedSeverity: 'critical' },
362
+ { category: 'pii_disclosure', expectedSeverity: 'high' },
363
+ { category: 'role_manipulation', expectedSeverity: 'high' },
364
+ { category: 'content_safety', expectedSeverity: 'high' },
365
+ ];
361
366
 
362
367
  for (const { category, expectedSeverity } of testCases) {
363
368
  it(`should map ${category} to ${expectedSeverity} severity`, async () => {
@@ -90,10 +90,6 @@ function setParseError(error: Error) {
90
90
  parseScenarioFileError = error;
91
91
  }
92
92
 
93
- function setResolvePaths(paths: string[]) {
94
- resolveScenarioPathsResponse = paths;
95
- }
96
-
97
93
  function resetMocks() {
98
94
  parseScenarioFileResponse = {};
99
95
  parseScenarioFileError = null;
@@ -2,7 +2,11 @@ import { describe, expect, test } from 'bun:test';
2
2
  import { mkdtemp } from 'node:fs/promises';
3
3
  import { tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
- import { type AgentTarget, validateAgentWorkflow } from '@artemiskit/core';
5
+ import {
6
+ type AgentTarget,
7
+ type AgentWorkflowRecord,
8
+ validateAgentWorkflow,
9
+ } from '@artemiskit/core';
6
10
  import { ArtemisKit } from '../artemiskit';
7
11
  import { readWorkflowRecord } from '../index';
8
12
  import type { WorkflowCheckpointOptions } from '../types-only';
@@ -87,7 +91,7 @@ describe('SDK workflow recovery options', () => {
87
91
  expect(paused.record.reason).toBe('checkpoint_paused');
88
92
  expect(paused.record.budgets.actions).toBe(2);
89
93
  expect(calls).toBe(1);
90
- expect(readWorkflowRecord(paused.record)).toEqual(paused.record);
94
+ expect(readWorkflowRecord(paused.record)).toEqual<AgentWorkflowRecord>(paused.record);
91
95
  const bad = await kit.runWorkflow({
92
96
  workflow,
93
97
  target,
@@ -111,6 +115,6 @@ describe('SDK workflow recovery options', () => {
111
115
  expect(resumed.record.recovery?.attempts).toBe(2);
112
116
  expect(JSON.stringify(resumed.record)).not.toContain('PRIVATE-');
113
117
  expect(JSON.stringify(resumed.record)).not.toContain(checkpoint.directory);
114
- expect(readWorkflowRecord(resumed.record)).toEqual(resumed.record);
118
+ expect(readWorkflowRecord(resumed.record)).toEqual<AgentWorkflowRecord>(resumed.record);
115
119
  });
116
120
  });
@@ -63,9 +63,12 @@ describe('SDK offline workflow reports', () => {
63
63
  providerConfig: { apiKey: 'not-a-credential' },
64
64
  });
65
65
  const fetch = globalThis.fetch;
66
- globalThis.fetch = (() => {
67
- throw new Error('Reporting attempted network');
68
- }) as typeof fetch;
66
+ globalThis.fetch = Object.assign(
67
+ () => {
68
+ throw new Error('Reporting attempted network');
69
+ },
70
+ { preconnect: fetch.preconnect }
71
+ );
69
72
  try {
70
73
  const before = JSON.stringify(record);
71
74
  const model = createWorkflowReport(record);
@@ -7,6 +7,7 @@ import {
7
7
  type AgentTarget,
8
8
  type AgentWorkflow,
9
9
  type AgentWorkflowEvent,
10
+ type AgentWorkflowRecord,
10
11
  createSimulatedWorkflowEnvironment,
11
12
  validateAgentWorkflow,
12
13
  } from '@artemiskit/core';
@@ -124,7 +125,9 @@ describe('SDK native workflow session', () => {
124
125
  );
125
126
  expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
126
127
  expect(closed).toBe(0);
127
- expect(readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))).toEqual(result.record);
128
+ expect(
129
+ readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))
130
+ ).toEqual<AgentWorkflowRecord>(result.record);
128
131
  expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
129
132
  }
130
133
  });
@@ -207,7 +210,7 @@ describe('SDK native workflow session', () => {
207
210
  expect(result.record.outcomes.cancelled).toBe(true);
208
211
  expect(result.record.outcomes.reason).toBe('cancelled');
209
212
  expect(result.record.outcomes.semantic.counts.valid).toBe(0);
210
- expect(readWorkflowRecord(result.record)).toEqual(result.record);
213
+ expect(readWorkflowRecord(result.record)).toEqual<AgentWorkflowRecord>(result.record);
211
214
  });
212
215
 
213
216
  test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
@@ -216,7 +219,7 @@ describe('SDK native workflow session', () => {
216
219
  hostname: '127.0.0.1',
217
220
  port: 0,
218
221
  async fetch(request) {
219
- bodies.push(await request.json());
222
+ bodies.push((await request.json()) as Record<string, unknown>);
220
223
  const first = bodies.length === 1;
221
224
  return Response.json({
222
225
  id: 'fixture',
@@ -305,8 +308,11 @@ describe('SDK native workflow session', () => {
305
308
  expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
306
309
  expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
307
310
  expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
308
- expect(events.some((event) => event.type === 'tool_completed')).toBe(true);
309
- });
311
+ expect(events.some((event: AgentWorkflowEvent) => event.type === 'tool_completed')).toBe(true);
312
+ // One Bun child loading core and the OpenAI adapter from source (about 0.5 s on a quiet
313
+ // machine, over 5 s at load averages 30-60). Stay above its 10 s kill timer so a stuck child
314
+ // fails on its own exit evidence.
315
+ }, 30_000);
310
316
  test('file fixture roots, session events and environment overrides use the same engine without persistence', async () => {
311
317
  const directory = await mkdtemp(join(tmpdir(), 'artemis-sdk-workflow-'));
312
318
  const scenario = workflow();
@@ -0,0 +1,190 @@
1
+ /**
2
+ * File-system boundary for experiment surfaces: bounded reads, result directories and
3
+ * write-once evidence files. Shared by the SDK functions and the `akit experiment` CLI.
4
+ */
5
+
6
+ import { createHash } from 'node:crypto';
7
+ import { mkdir, open, readFile, readdir, stat } from 'node:fs/promises';
8
+ import path from 'node:path';
9
+ import type { ExperimentRunResult } from '@artemiskit/core';
10
+ import { MAX_CASE_EVIDENCE_BYTES } from '@artemiskit/reports';
11
+
12
+ /** Largest experiment definition the surfaces will read. */
13
+ export const MAX_EXPERIMENT_FILE_BYTES = 1_048_576;
14
+ /** Largest saved experiment result the report surface will read. */
15
+ export const MAX_EXPERIMENT_RESULT_BYTES = 64 * 1_048_576;
16
+ /** Default output root, relative to the working directory; ignored by git. */
17
+ export const DEFAULT_EXPERIMENT_OUTPUT_ROOT = path.join('artemis-output', 'experiments');
18
+ export const EXPERIMENT_RESULT_FILE = 'result.json';
19
+ export const EXPERIMENT_EVIDENCE_DIR = 'evidence';
20
+
21
+ /** Owner-only permissions: evidence holds prompts and model responses. */
22
+ const PRIVATE_FILE_MODE = 0o600;
23
+
24
+ /**
25
+ * Exit-code-bearing failure for the experiment surfaces. `exitCode` is the CLI exit code the
26
+ * same failure maps to, so SDK callers and the CLI agree.
27
+ */
28
+ export class ExperimentSurfaceError extends Error {
29
+ readonly code: string;
30
+ readonly exitCode: number;
31
+
32
+ constructor(code: string, message: string, exitCode: number) {
33
+ super(message);
34
+ this.name = 'ExperimentSurfaceError';
35
+ this.code = code;
36
+ this.exitCode = exitCode;
37
+ }
38
+ }
39
+
40
+ /** Read a regular file of at most `limit` bytes as strict UTF-8. */
41
+ export async function readBoundedText(file: string, limit: number): Promise<string> {
42
+ const metadata = await stat(file);
43
+ if (!metadata.isFile()) throw new ExperimentSurfaceError('not_a_file', 'Not a regular file', 1);
44
+ if (metadata.size > limit) {
45
+ throw new ExperimentSurfaceError('file_too_large', `File exceeds ${limit} bytes`, 1);
46
+ }
47
+ const bytes = await readFile(file);
48
+ if (bytes.byteLength > limit) {
49
+ throw new ExperimentSurfaceError('file_too_large', `File exceeds ${limit} bytes`, 1);
50
+ }
51
+ return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
52
+ }
53
+
54
+ /** Lowercase hex SHA-256 of exact bytes. */
55
+ export function sha256Hex(bytes: Uint8Array | string): string {
56
+ return createHash('sha256').update(bytes).digest('hex');
57
+ }
58
+
59
+ /**
60
+ * Canonical JSON with recursively sorted object keys. Matches the serialization of core's
61
+ * `runManifestIdentity` and `workflowRecordIdentity`, so a saved evidence file's SHA-256 equals
62
+ * its recorded artifact digest.
63
+ */
64
+ export function canonicalJson(value: unknown): string {
65
+ return JSON.stringify(sortKeys(value));
66
+ }
67
+
68
+ function sortKeys(value: unknown): unknown {
69
+ if (Array.isArray(value)) return value.map(sortKeys);
70
+ if (value === null || typeof value !== 'object') return value;
71
+ const record = value as Record<string, unknown>;
72
+ return Object.fromEntries(
73
+ Object.keys(record)
74
+ .sort()
75
+ .map((key) => [key, sortKeys(record[key])])
76
+ );
77
+ }
78
+
79
+ /** `artemis-output/experiments/<id>/<timestamp>` under `cwd`, with a filename-safe timestamp. */
80
+ export function defaultExperimentOutputDir(experimentId: string, cwd: string, now: Date): string {
81
+ const stamp = now.toISOString().replace(/[:.]/g, '-');
82
+ return path.resolve(cwd, DEFAULT_EXPERIMENT_OUTPUT_ROOT, experimentId, stamp);
83
+ }
84
+
85
+ /** Create `directory` (and parents) and refuse to reuse one that already holds files. */
86
+ export async function prepareEmptyDirectory(directory: string): Promise<void> {
87
+ try {
88
+ await mkdir(directory, { recursive: true });
89
+ if ((await readdir(directory)).length > 0) throw new Error('not_empty');
90
+ } catch {
91
+ throw new ExperimentSurfaceError(
92
+ 'output_unavailable',
93
+ 'Output directory is unavailable or not empty; choose a new directory',
94
+ 1
95
+ );
96
+ }
97
+ }
98
+
99
+ /** Write a new file; never overwrite. */
100
+ export async function writeNewFile(file: string, content: string): Promise<void> {
101
+ const handle = await open(file, 'wx', PRIVATE_FILE_MODE);
102
+ try {
103
+ await handle.writeFile(content, 'utf8');
104
+ await handle.sync();
105
+ } finally {
106
+ await handle.close();
107
+ }
108
+ }
109
+
110
+ /**
111
+ * Save one attempt's evidence (a scenario run manifest or a workflow record) as canonical JSON
112
+ * named by its digest, and prove the written bytes match the identity recorded in the
113
+ * experiment result. A throw marks the attempt `infrastructure_failed` in the executor.
114
+ */
115
+ export async function saveAttemptEvidence(
116
+ directory: string,
117
+ evidence: unknown,
118
+ digest: string
119
+ ): Promise<string> {
120
+ const content = canonicalJson(evidence);
121
+ if (sha256Hex(content) !== digest) throw new Error('evidence_digest_mismatch');
122
+ const evidenceDir = path.join(directory, EXPERIMENT_EVIDENCE_DIR);
123
+ await mkdir(evidenceDir, { recursive: true });
124
+ const file = path.join(evidenceDir, `${digest}.json`);
125
+ try {
126
+ await writeNewFile(file, content);
127
+ } catch (error) {
128
+ // Identical content already stored under its own digest is the same evidence.
129
+ if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
130
+ if (sha256Hex(await readFile(file)) !== digest) throw error;
131
+ }
132
+ return file;
133
+ }
134
+
135
+ /** Persist the V1 experiment result as pretty JSON. */
136
+ export async function saveExperimentResult(
137
+ directory: string,
138
+ result: ExperimentRunResult
139
+ ): Promise<string> {
140
+ const file = path.join(directory, EXPERIMENT_RESULT_FILE);
141
+ try {
142
+ await writeNewFile(file, `${JSON.stringify(result, null, 2)}\n`);
143
+ } catch {
144
+ throw new ExperimentSurfaceError(
145
+ 'result_unwritable',
146
+ 'Unable to save the experiment result',
147
+ 1
148
+ );
149
+ }
150
+ return file;
151
+ }
152
+
153
+ const EVIDENCE_DIGEST = /^[a-f0-9]{64}$/;
154
+
155
+ /**
156
+ * Read the saved evidence files named by `digests` from an `evidence/` directory, as exact bytes
157
+ * keyed by digest. A missing, unreadable or oversized file maps to `null` so the case-level
158
+ * summary reports it; the bytes are verified against the digest by the report model, not here.
159
+ * Returns `undefined` when the directory does not exist (results saved without evidence).
160
+ */
161
+ export async function readExperimentCaseEvidence(
162
+ evidenceDir: string,
163
+ digests: readonly string[]
164
+ ): Promise<Map<string, Uint8Array | null> | undefined> {
165
+ try {
166
+ if (!(await stat(evidenceDir)).isDirectory()) return undefined;
167
+ } catch {
168
+ return undefined;
169
+ }
170
+ const files = new Map<string, Uint8Array | null>();
171
+ for (const digest of digests) {
172
+ files.set(
173
+ digest,
174
+ EVIDENCE_DIGEST.test(digest) ? await readEvidenceBytes(evidenceDir, digest) : null
175
+ );
176
+ }
177
+ return files;
178
+ }
179
+
180
+ async function readEvidenceBytes(evidenceDir: string, digest: string): Promise<Uint8Array | null> {
181
+ const file = path.join(evidenceDir, `${digest}.json`);
182
+ try {
183
+ const metadata = await stat(file);
184
+ if (!metadata.isFile() || metadata.size > MAX_CASE_EVIDENCE_BYTES) return null;
185
+ const bytes = await readFile(file);
186
+ return bytes.byteLength > MAX_CASE_EVIDENCE_BYTES ? null : new Uint8Array(bytes);
187
+ } catch {
188
+ return null;
189
+ }
190
+ }
@@ -0,0 +1,70 @@
1
+ /**
2
+ * Deterministic in-memory model client for fixture-mode experiments.
3
+ *
4
+ * Fixture runs exercise the full experiment pipeline (matrix, executor, evaluators, evidence,
5
+ * reports) without any network access. Responses are fixed text, so fixture outcomes describe
6
+ * the pipeline, never a model.
7
+ */
8
+
9
+ import type {
10
+ ExperimentTarget,
11
+ GenerateOptions,
12
+ GenerateResult,
13
+ ModelClient,
14
+ } from '@artemiskit/core';
15
+ import { sha256Hex } from './experiment-files';
16
+
17
+ export const FIXTURE_PROVIDER = 'artemiskit-fixture';
18
+
19
+ /** Rough token estimate (about four characters per token) so usage is always reported. */
20
+ const CHARACTERS_PER_TOKEN = 4;
21
+
22
+ export interface ExperimentFixtureClientOptions {
23
+ /**
24
+ * Advertise structured tool use. The client accepts tool definitions but never calls a
25
+ * tool, so a fixture workflow can run (the workflow engine requires tool support) and its
26
+ * outcome depends only on the declared state and outcomes.
27
+ */
28
+ toolUse?: boolean;
29
+ }
30
+
31
+ /** Create the offline client bound to one fixture-mode target. */
32
+ export function createExperimentFixtureClient(
33
+ target: ExperimentTarget,
34
+ options: ExperimentFixtureClientOptions = {}
35
+ ): ModelClient {
36
+ const toolUse = options.toolUse === true;
37
+ return {
38
+ provider: FIXTURE_PROVIDER,
39
+ async capabilities() {
40
+ return { streaming: false, functionCalling: false, toolUse, maxContext: 128_000 };
41
+ },
42
+ async generate(options: GenerateOptions): Promise<GenerateResult> {
43
+ const prompt = promptText(options.prompt);
44
+ const text = `Fixture response from ${target.id}.`;
45
+ const promptTokens = estimateTokens(prompt);
46
+ const completionTokens = estimateTokens(text);
47
+ return {
48
+ id: `fixture-${sha256Hex(`${target.id}\n${prompt}`).slice(0, 16)}`,
49
+ model: target.model,
50
+ text,
51
+ tokens: {
52
+ prompt: promptTokens,
53
+ completion: completionTokens,
54
+ total: promptTokens + completionTokens,
55
+ },
56
+ usageAvailable: true,
57
+ latencyMs: 0,
58
+ finishReason: 'stop',
59
+ };
60
+ },
61
+ };
62
+ }
63
+
64
+ function promptText(prompt: GenerateOptions['prompt']): string {
65
+ return typeof prompt === 'string' ? prompt : prompt.map((message) => message.content).join('\n');
66
+ }
67
+
68
+ function estimateTokens(text: string): number {
69
+ return Math.max(1, Math.ceil(text.length / CHARACTERS_PER_TOKEN));
70
+ }
@@ -0,0 +1,172 @@
1
+ /**
2
+ * Exit codes and the plain-text run summary shared by the SDK and `akit experiment run`.
3
+ * The summary is descriptive: it lists targets in declared order and never ranks them.
4
+ */
5
+
6
+ import type { ExperimentRunResult, ExperimentSummary, ExperimentTarget } from '@artemiskit/core';
7
+ import {
8
+ type ExperimentCaseCounts,
9
+ type ExperimentCaseLevelSummary,
10
+ formatCaseRate,
11
+ } from '@artemiskit/reports';
12
+
13
+ /** Exit codes for `akit experiment run`; SDK outcomes carry the same value. */
14
+ export const EXPERIMENT_EXIT_CODES = {
15
+ /** Completed with a valid passed or failed measurement for every declared coordinate. */
16
+ complete: 0,
17
+ /** Unexpected I/O or internal failure, including an unusable output directory. */
18
+ error: 1,
19
+ /** The experiment file, a task source or a target declaration failed validation. */
20
+ invalid: 2,
21
+ /** Live experiment without a valid explicit approval; nothing was executed. */
22
+ unauthorized: 3,
23
+ /** Execution stopped early (budget or usage stop condition); see the stop reason. */
24
+ stopped: 4,
25
+ /** Finished, but some coordinates are invalid, unsupported, excluded, errored or incomplete. */
26
+ coverageIncomplete: 5,
27
+ } as const;
28
+
29
+ export type ExperimentRunStatus = keyof typeof EXPERIMENT_EXIT_CODES;
30
+ export type ExperimentExitCode = (typeof EXPERIMENT_EXIT_CODES)[ExperimentRunStatus];
31
+
32
+ /** Classify a finished run. Early stops take precedence over incomplete coverage. */
33
+ export function experimentRunStatus(
34
+ result: ExperimentRunResult
35
+ ): 'complete' | 'stopped' | 'coverageIncomplete' {
36
+ if (result.stop_reason !== undefined || !result.completion.execution_complete) return 'stopped';
37
+ if (!result.completion.valid_measurement_coverage_complete) return 'coverageIncomplete';
38
+ return 'complete';
39
+ }
40
+
41
+ /** Deployment label for a target, e.g. "open-weight, run locally via ollama; cost not applicable". */
42
+ export function describeTargetDeployment(target: ExperimentTarget): string {
43
+ const deployment = target.deployment;
44
+ const weights = target.weights?.availability === 'open' ? 'open-weight, ' : '';
45
+ if (deployment?.locality !== 'local') return `${weights}remote; metered cost`;
46
+ const runtime = deployment.runtime ?? 'an unspecified runtime';
47
+ const cost = deployment.cost_basis === 'not_applicable' ? 'cost not applicable' : 'metered cost';
48
+ return `${weights}run locally via ${runtime}; ${cost}`;
49
+ }
50
+
51
+ /**
52
+ * Declared concurrency limit and, for concurrent runs, the recorded peak and caveat, e.g.
53
+ * "up to 4 coordinates in flight (declared limit); recorded peak 3; completion order and
54
+ * sampled outputs may differ from a serial run".
55
+ */
56
+ export function describeConcurrency(result: ExperimentRunResult): string {
57
+ const limit = result.manifest.concurrency;
58
+ if (limit === 1) return '1 (serial)';
59
+ const peak = result.execution ? `; recorded peak ${result.execution.peak_in_flight}` : '';
60
+ return `up to ${limit} coordinates in flight (declared limit)${peak}; completion order and sampled outputs may differ from a serial run`;
61
+ }
62
+
63
+ /** Human-readable summary of a saved run. Makes no ranking or winner claim. */
64
+ export function formatExperimentRunSummary(
65
+ result: ExperimentRunResult,
66
+ options: { resultPath?: string; caseLevel?: ExperimentCaseLevelSummary } = {}
67
+ ): string {
68
+ const { manifest, summaries, completion } = result;
69
+ const lines = [
70
+ `Experiment ${result.experiment_id} — ${
71
+ result.mode === 'fixture'
72
+ ? 'fixture mode: deterministic offline client; outcomes measure the pipeline, not any model'
73
+ : `live mode; approved by ${result.live_authorization?.approved_by ?? 'unknown'} (${result.live_authorization?.decision_id ?? 'no decision id'})`
74
+ }`,
75
+ `Matrix: ${summaries.overall.planned} planned coordinate(s) = ${manifest.tasks.length} task(s) × ${manifest.targets.length} target(s) × ${manifest.repetitions} repetition(s)`,
76
+ `Concurrency: ${describeConcurrency(result)}`,
77
+ COORDINATE_LEVEL_LINE,
78
+ 'Targets (declared order; counts describe this run only):',
79
+ ];
80
+ const groups = new Map(summaries.targets.map((group) => [group.key, group.summary]));
81
+ for (const target of manifest.targets) {
82
+ const publisher = target.publisher ? `; published by ${target.publisher.name}` : '';
83
+ lines.push(
84
+ ` - ${target.id}: ${target.provider} ${target.model} — ${describeTargetDeployment(target)}${publisher}`
85
+ );
86
+ const summary = groups.get(target.id);
87
+ if (summary) lines.push(` ${formatCounts(summary)}`);
88
+ }
89
+ lines.push(
90
+ `Overall: ${formatCounts(summaries.overall)}`,
91
+ ...(options.caseLevel
92
+ ? [
93
+ formatExperimentCaseSummary(
94
+ options.caseLevel,
95
+ manifest.targets.map((target) => target.id)
96
+ ),
97
+ ]
98
+ : []),
99
+ `Stop reason: ${result.stop_reason ?? 'none'}`,
100
+ `Completion: matrix ${completeness(completion.matrix_complete)}; execution ${completeness(
101
+ completion.execution_complete
102
+ )}; valid-measurement coverage ${completeness(completion.valid_measurement_coverage_complete)}`,
103
+ `Usage: ${summaries.operational.requests} request(s); ${summaries.operational.tokens} token(s); cost ${formatCost(result)}`,
104
+ 'No ranking or winner is implied. Generate a report for methodology, evidence and limitations.'
105
+ );
106
+ if (options.resultPath) lines.push(`Result: ${options.resultPath}`);
107
+ return lines.join('\n');
108
+ }
109
+
110
+ const COORDINATE_LEVEL_LINE =
111
+ 'Coordinate level: counts below are task × target × repetition coordinates; a scenario coordinate passes only when all of its cases pass.';
112
+
113
+ /**
114
+ * Plain-text case-level summary: individual scenario cases from verified per-attempt evidence,
115
+ * per target (in `targetOrder` when given) and overall, or why it is unavailable. No ranking.
116
+ */
117
+ export function formatExperimentCaseSummary(
118
+ caseLevel: ExperimentCaseLevelSummary,
119
+ targetOrder: readonly string[] = []
120
+ ): string {
121
+ if (caseLevel.availability === 'unavailable')
122
+ return `Case level: unavailable. ${caseLevel.detail}`;
123
+ const position = (key: string) => {
124
+ const index = targetOrder.indexOf(key);
125
+ return index === -1 ? Number.MAX_SAFE_INTEGER : index;
126
+ };
127
+ const targets = [...caseLevel.targets].sort(
128
+ (left, right) => position(left.key) - position(right.key)
129
+ );
130
+ return [
131
+ `Case level: counts below are individual scenario cases from ${caseLevel.verifiedEvidenceFiles} SHA-256-verified evidence file(s) (terminal attempt per coordinate):`,
132
+ ...targets.map((group) => ` - ${group.key}: ${formatCaseCounts(group.counts)}`),
133
+ `Overall cases: ${formatCaseCounts(caseLevel.overall)}`,
134
+ ].join('\n');
135
+ }
136
+
137
+ function formatCaseCounts(counts: ExperimentCaseCounts): string {
138
+ const unknown = (value: number | null) => (value === null ? 'unavailable' : String(value));
139
+ return [
140
+ `passed ${counts.passed} and failed ${counts.failed} of ${counts.valid} valid case(s) (passed/valid ${formatCaseRate(counts)})`,
141
+ `invalid ${counts.invalid}`,
142
+ `planned ${unknown(counts.planned)}`,
143
+ `unmeasured ${unknown(counts.unmeasured)}`,
144
+ `coordinates without case evidence ${counts.coordinatesWithoutCaseEvidence}`,
145
+ `agent-workflow coordinates ${counts.coordinatesNotApplicable}`,
146
+ ].join('; ');
147
+ }
148
+
149
+ function formatCounts(summary: ExperimentSummary): string {
150
+ return [
151
+ `valid ${summary.valid}/${summary.planned} planned`,
152
+ `passed ${summary.passed} and task-failed ${summary.task_failed} of ${summary.valid} valid`,
153
+ `invalid ${summary.invalid}`,
154
+ `infrastructure-failed ${summary.infrastructure_failed}`,
155
+ `unsupported ${summary.unsupported}`,
156
+ `excluded ${summary.excluded}`,
157
+ `incomplete ${summary.incomplete}`,
158
+ ].join('; ');
159
+ }
160
+
161
+ function formatCost(result: ExperimentRunResult): string {
162
+ const allLocal = result.manifest.targets.every(
163
+ (target) => target.deployment?.cost_basis === 'not_applicable'
164
+ );
165
+ if (allLocal) return 'not applicable (local targets only)';
166
+ const cost = result.summaries.operational.cost;
167
+ return cost ? `${cost.amount} ${cost.currency}` : 'not reported';
168
+ }
169
+
170
+ function completeness(value: boolean): string {
171
+ return value ? 'complete' : 'incomplete';
172
+ }