@artemiskit/sdk 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/__tests__/experiment-fixtures.d.ts +44 -0
- package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
- package/dist/builders/index.js +19 -66
- package/dist/contracts/index.js +3 -50
- package/dist/experiment-files.d.ts +54 -0
- package/dist/experiment-files.d.ts.map +1 -0
- package/dist/experiment-fixture.d.ts +20 -0
- package/dist/experiment-fixture.d.ts.map +1 -0
- package/dist/experiment-summary.d.ts +44 -0
- package/dist/experiment-summary.d.ts.map +1 -0
- package/dist/experiment-validation.d.ts +48 -0
- package/dist/experiment-validation.d.ts.map +1 -0
- package/dist/experiment.d.ts +162 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +6 -1
- package/dist/guardian/circuit-breaker.d.ts.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2557 -1863
- package/dist/matchers/index.js +13 -60
- package/dist/matchers/jest.js +0 -47
- package/dist/matchers/vitest.js +0 -47
- package/dist/utils/index.js +11 -58
- package/package.json +6 -5
- package/src/__tests__/artemiskit.test.ts +1 -2
- package/src/__tests__/builders.test.ts +0 -2
- package/src/__tests__/circuit-breaker.test.ts +1 -1
- package/src/__tests__/compare.test.ts +1 -1
- package/src/__tests__/contracts.test.ts +1 -1
- package/src/__tests__/experiment-case-level.test.ts +212 -0
- package/src/__tests__/experiment-fixtures.ts +194 -0
- package/src/__tests__/experiment.test.ts +1262 -0
- package/src/__tests__/guardian-modes.test.ts +2 -2
- package/src/__tests__/guardrails.test.ts +1 -1
- package/src/__tests__/matchers.test.ts +1 -1
- package/src/__tests__/semantic-validator.test.ts +15 -10
- package/src/__tests__/validate.test.ts +0 -4
- package/src/__tests__/workflow-recovery.test.ts +7 -3
- package/src/__tests__/workflow-report.test.ts +6 -3
- package/src/__tests__/workflow.test.ts +11 -5
- package/src/experiment-files.ts +190 -0
- package/src/experiment-fixture.ts +70 -0
- package/src/experiment-summary.ts +172 -0
- package/src/experiment-validation.ts +426 -0
- package/src/experiment.ts +630 -0
- package/src/guardian/circuit-breaker.ts +6 -1
- package/src/index.ts +52 -0
- package/tsconfig.typecheck.json +12 -0
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* Tests for Guardian mode normalization (v0.3.2+)
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import { describe, expect,
|
|
6
|
-
import {
|
|
5
|
+
import { describe, expect, test } from 'bun:test';
|
|
6
|
+
import { createGuardian, normalizeGuardianMode } from '../guardian';
|
|
7
7
|
|
|
8
8
|
describe('Guardian Mode Normalization', () => {
|
|
9
9
|
describe('normalizeGuardianMode', () => {
|
|
@@ -327,7 +327,7 @@ describe('Guardrails', () => {
|
|
|
327
327
|
const guardrail = createCustomPatternGuardrail(['forbidden', 'prohibited']);
|
|
328
328
|
const result = await guardrail('This is forbidden content');
|
|
329
329
|
expect(result.passed).toBe(false);
|
|
330
|
-
expect(result.violations[0].details
|
|
330
|
+
expect(result.violations[0].details?.matchedText).toBe('forbidden');
|
|
331
331
|
});
|
|
332
332
|
|
|
333
333
|
test('should match custom regex patterns', async () => {
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* - Error handling and edge cases
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { describe, expect, it, mock } from 'bun:test';
|
|
12
12
|
|
|
13
13
|
// Mock nanoid for consistent IDs in tests
|
|
14
14
|
mock.module('nanoid', () => ({
|
|
@@ -17,7 +17,11 @@ mock.module('nanoid', () => ({
|
|
|
17
17
|
|
|
18
18
|
import type { ModelClient } from '@artemiskit/core';
|
|
19
19
|
import { SemanticValidator, createSemanticValidator } from '../guardian/semantic-validator';
|
|
20
|
-
import type {
|
|
20
|
+
import type {
|
|
21
|
+
ContentValidationConfig,
|
|
22
|
+
ValidationCategory,
|
|
23
|
+
ViolationSeverity,
|
|
24
|
+
} from '../guardian/types';
|
|
21
25
|
|
|
22
26
|
// =============================================================================
|
|
23
27
|
// Mock LLM Client
|
|
@@ -350,14 +354,15 @@ describe('SemanticValidator', () => {
|
|
|
350
354
|
});
|
|
351
355
|
|
|
352
356
|
describe('severity mapping', () => {
|
|
353
|
-
const testCases: Array<{ category: ValidationCategory; expectedSeverity:
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
357
|
+
const testCases: Array<{ category: ValidationCategory; expectedSeverity: ViolationSeverity }> =
|
|
358
|
+
[
|
|
359
|
+
{ category: 'prompt_injection', expectedSeverity: 'critical' },
|
|
360
|
+
{ category: 'jailbreak', expectedSeverity: 'critical' },
|
|
361
|
+
{ category: 'data_extraction', expectedSeverity: 'critical' },
|
|
362
|
+
{ category: 'pii_disclosure', expectedSeverity: 'high' },
|
|
363
|
+
{ category: 'role_manipulation', expectedSeverity: 'high' },
|
|
364
|
+
{ category: 'content_safety', expectedSeverity: 'high' },
|
|
365
|
+
];
|
|
361
366
|
|
|
362
367
|
for (const { category, expectedSeverity } of testCases) {
|
|
363
368
|
it(`should map ${category} to ${expectedSeverity} severity`, async () => {
|
|
@@ -90,10 +90,6 @@ function setParseError(error: Error) {
|
|
|
90
90
|
parseScenarioFileError = error;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
function setResolvePaths(paths: string[]) {
|
|
94
|
-
resolveScenarioPathsResponse = paths;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
93
|
function resetMocks() {
|
|
98
94
|
parseScenarioFileResponse = {};
|
|
99
95
|
parseScenarioFileError = null;
|
|
@@ -2,7 +2,11 @@ import { describe, expect, test } from 'bun:test';
|
|
|
2
2
|
import { mkdtemp } from 'node:fs/promises';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
type AgentTarget,
|
|
7
|
+
type AgentWorkflowRecord,
|
|
8
|
+
validateAgentWorkflow,
|
|
9
|
+
} from '@artemiskit/core';
|
|
6
10
|
import { ArtemisKit } from '../artemiskit';
|
|
7
11
|
import { readWorkflowRecord } from '../index';
|
|
8
12
|
import type { WorkflowCheckpointOptions } from '../types-only';
|
|
@@ -87,7 +91,7 @@ describe('SDK workflow recovery options', () => {
|
|
|
87
91
|
expect(paused.record.reason).toBe('checkpoint_paused');
|
|
88
92
|
expect(paused.record.budgets.actions).toBe(2);
|
|
89
93
|
expect(calls).toBe(1);
|
|
90
|
-
expect(readWorkflowRecord(paused.record)).toEqual(paused.record);
|
|
94
|
+
expect(readWorkflowRecord(paused.record)).toEqual<AgentWorkflowRecord>(paused.record);
|
|
91
95
|
const bad = await kit.runWorkflow({
|
|
92
96
|
workflow,
|
|
93
97
|
target,
|
|
@@ -111,6 +115,6 @@ describe('SDK workflow recovery options', () => {
|
|
|
111
115
|
expect(resumed.record.recovery?.attempts).toBe(2);
|
|
112
116
|
expect(JSON.stringify(resumed.record)).not.toContain('PRIVATE-');
|
|
113
117
|
expect(JSON.stringify(resumed.record)).not.toContain(checkpoint.directory);
|
|
114
|
-
expect(readWorkflowRecord(resumed.record)).toEqual(resumed.record);
|
|
118
|
+
expect(readWorkflowRecord(resumed.record)).toEqual<AgentWorkflowRecord>(resumed.record);
|
|
115
119
|
});
|
|
116
120
|
});
|
|
@@ -63,9 +63,12 @@ describe('SDK offline workflow reports', () => {
|
|
|
63
63
|
providerConfig: { apiKey: 'not-a-credential' },
|
|
64
64
|
});
|
|
65
65
|
const fetch = globalThis.fetch;
|
|
66
|
-
globalThis.fetch = (
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
globalThis.fetch = Object.assign(
|
|
67
|
+
() => {
|
|
68
|
+
throw new Error('Reporting attempted network');
|
|
69
|
+
},
|
|
70
|
+
{ preconnect: fetch.preconnect }
|
|
71
|
+
);
|
|
69
72
|
try {
|
|
70
73
|
const before = JSON.stringify(record);
|
|
71
74
|
const model = createWorkflowReport(record);
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
type AgentTarget,
|
|
8
8
|
type AgentWorkflow,
|
|
9
9
|
type AgentWorkflowEvent,
|
|
10
|
+
type AgentWorkflowRecord,
|
|
10
11
|
createSimulatedWorkflowEnvironment,
|
|
11
12
|
validateAgentWorkflow,
|
|
12
13
|
} from '@artemiskit/core';
|
|
@@ -124,7 +125,9 @@ describe('SDK native workflow session', () => {
|
|
|
124
125
|
);
|
|
125
126
|
expect(calls).toBe(verdict === 'unconfigured' ? 0 : 1);
|
|
126
127
|
expect(closed).toBe(0);
|
|
127
|
-
expect(
|
|
128
|
+
expect(
|
|
129
|
+
readWorkflowRecord(JSON.parse(JSON.stringify(result.record)))
|
|
130
|
+
).toEqual<AgentWorkflowRecord>(result.record);
|
|
128
131
|
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
129
132
|
}
|
|
130
133
|
});
|
|
@@ -207,7 +210,7 @@ describe('SDK native workflow session', () => {
|
|
|
207
210
|
expect(result.record.outcomes.cancelled).toBe(true);
|
|
208
211
|
expect(result.record.outcomes.reason).toBe('cancelled');
|
|
209
212
|
expect(result.record.outcomes.semantic.counts.valid).toBe(0);
|
|
210
|
-
expect(readWorkflowRecord(result.record)).toEqual(result.record);
|
|
213
|
+
expect(readWorkflowRecord(result.record)).toEqual<AgentWorkflowRecord>(result.record);
|
|
211
214
|
});
|
|
212
215
|
|
|
213
216
|
test('configured adapter uses workflow identity and zero retries with real loopback multi-turn execution', async () => {
|
|
@@ -216,7 +219,7 @@ describe('SDK native workflow session', () => {
|
|
|
216
219
|
hostname: '127.0.0.1',
|
|
217
220
|
port: 0,
|
|
218
221
|
async fetch(request) {
|
|
219
|
-
bodies.push(await request.json());
|
|
222
|
+
bodies.push((await request.json()) as Record<string, unknown>);
|
|
220
223
|
const first = bodies.length === 1;
|
|
221
224
|
return Response.json({
|
|
222
225
|
id: 'fixture',
|
|
@@ -305,8 +308,11 @@ describe('SDK native workflow session', () => {
|
|
|
305
308
|
expect(result.record.budgets).toMatchObject({ modelRequests: 2, toolCalls: 1 });
|
|
306
309
|
expect(result.state?.files).toEqual({ 'result.txt': 'PRIVATE-STATE' });
|
|
307
310
|
expect(JSON.stringify(result.record)).not.toContain('PRIVATE-');
|
|
308
|
-
expect(events.some((event) => event.type === 'tool_completed')).toBe(true);
|
|
309
|
-
|
|
311
|
+
expect(events.some((event: AgentWorkflowEvent) => event.type === 'tool_completed')).toBe(true);
|
|
312
|
+
// One Bun child loading core and the OpenAI adapter from source (about 0.5 s on a quiet
|
|
313
|
+
// machine, over 5 s at load averages 30-60). Stay above its 10 s kill timer so a stuck child
|
|
314
|
+
// fails on its own exit evidence.
|
|
315
|
+
}, 30_000);
|
|
310
316
|
test('file fixture roots, session events and environment overrides use the same engine without persistence', async () => {
|
|
311
317
|
const directory = await mkdtemp(join(tmpdir(), 'artemis-sdk-workflow-'));
|
|
312
318
|
const scenario = workflow();
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File-system boundary for experiment surfaces: bounded reads, result directories and
|
|
3
|
+
* write-once evidence files. Shared by the SDK functions and the `akit experiment` CLI.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import { mkdir, open, readFile, readdir, stat } from 'node:fs/promises';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
import type { ExperimentRunResult } from '@artemiskit/core';
|
|
10
|
+
import { MAX_CASE_EVIDENCE_BYTES } from '@artemiskit/reports';
|
|
11
|
+
|
|
12
|
+
/** Largest experiment definition the surfaces will read. */
|
|
13
|
+
export const MAX_EXPERIMENT_FILE_BYTES = 1_048_576;
|
|
14
|
+
/** Largest saved experiment result the report surface will read. */
|
|
15
|
+
export const MAX_EXPERIMENT_RESULT_BYTES = 64 * 1_048_576;
|
|
16
|
+
/** Default output root, relative to the working directory; ignored by git. */
|
|
17
|
+
export const DEFAULT_EXPERIMENT_OUTPUT_ROOT = path.join('artemis-output', 'experiments');
|
|
18
|
+
export const EXPERIMENT_RESULT_FILE = 'result.json';
|
|
19
|
+
export const EXPERIMENT_EVIDENCE_DIR = 'evidence';
|
|
20
|
+
|
|
21
|
+
/** Owner-only permissions: evidence holds prompts and model responses. */
|
|
22
|
+
const PRIVATE_FILE_MODE = 0o600;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Exit-code-bearing failure for the experiment surfaces. `exitCode` is the CLI exit code the
|
|
26
|
+
* same failure maps to, so SDK callers and the CLI agree.
|
|
27
|
+
*/
|
|
28
|
+
export class ExperimentSurfaceError extends Error {
|
|
29
|
+
readonly code: string;
|
|
30
|
+
readonly exitCode: number;
|
|
31
|
+
|
|
32
|
+
constructor(code: string, message: string, exitCode: number) {
|
|
33
|
+
super(message);
|
|
34
|
+
this.name = 'ExperimentSurfaceError';
|
|
35
|
+
this.code = code;
|
|
36
|
+
this.exitCode = exitCode;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Read a regular file of at most `limit` bytes as strict UTF-8. */
|
|
41
|
+
export async function readBoundedText(file: string, limit: number): Promise<string> {
|
|
42
|
+
const metadata = await stat(file);
|
|
43
|
+
if (!metadata.isFile()) throw new ExperimentSurfaceError('not_a_file', 'Not a regular file', 1);
|
|
44
|
+
if (metadata.size > limit) {
|
|
45
|
+
throw new ExperimentSurfaceError('file_too_large', `File exceeds ${limit} bytes`, 1);
|
|
46
|
+
}
|
|
47
|
+
const bytes = await readFile(file);
|
|
48
|
+
if (bytes.byteLength > limit) {
|
|
49
|
+
throw new ExperimentSurfaceError('file_too_large', `File exceeds ${limit} bytes`, 1);
|
|
50
|
+
}
|
|
51
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Lowercase hex SHA-256 of exact bytes. */
|
|
55
|
+
export function sha256Hex(bytes: Uint8Array | string): string {
|
|
56
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Canonical JSON with recursively sorted object keys. Matches the serialization of core's
|
|
61
|
+
* `runManifestIdentity` and `workflowRecordIdentity`, so a saved evidence file's SHA-256 equals
|
|
62
|
+
* its recorded artifact digest.
|
|
63
|
+
*/
|
|
64
|
+
export function canonicalJson(value: unknown): string {
|
|
65
|
+
return JSON.stringify(sortKeys(value));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function sortKeys(value: unknown): unknown {
|
|
69
|
+
if (Array.isArray(value)) return value.map(sortKeys);
|
|
70
|
+
if (value === null || typeof value !== 'object') return value;
|
|
71
|
+
const record = value as Record<string, unknown>;
|
|
72
|
+
return Object.fromEntries(
|
|
73
|
+
Object.keys(record)
|
|
74
|
+
.sort()
|
|
75
|
+
.map((key) => [key, sortKeys(record[key])])
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** `artemis-output/experiments/<id>/<timestamp>` under `cwd`, with a filename-safe timestamp. */
|
|
80
|
+
export function defaultExperimentOutputDir(experimentId: string, cwd: string, now: Date): string {
|
|
81
|
+
const stamp = now.toISOString().replace(/[:.]/g, '-');
|
|
82
|
+
return path.resolve(cwd, DEFAULT_EXPERIMENT_OUTPUT_ROOT, experimentId, stamp);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Create `directory` (and parents) and refuse to reuse one that already holds files. */
|
|
86
|
+
export async function prepareEmptyDirectory(directory: string): Promise<void> {
|
|
87
|
+
try {
|
|
88
|
+
await mkdir(directory, { recursive: true });
|
|
89
|
+
if ((await readdir(directory)).length > 0) throw new Error('not_empty');
|
|
90
|
+
} catch {
|
|
91
|
+
throw new ExperimentSurfaceError(
|
|
92
|
+
'output_unavailable',
|
|
93
|
+
'Output directory is unavailable or not empty; choose a new directory',
|
|
94
|
+
1
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Write a new file; never overwrite. */
|
|
100
|
+
export async function writeNewFile(file: string, content: string): Promise<void> {
|
|
101
|
+
const handle = await open(file, 'wx', PRIVATE_FILE_MODE);
|
|
102
|
+
try {
|
|
103
|
+
await handle.writeFile(content, 'utf8');
|
|
104
|
+
await handle.sync();
|
|
105
|
+
} finally {
|
|
106
|
+
await handle.close();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Save one attempt's evidence (a scenario run manifest or a workflow record) as canonical JSON
|
|
112
|
+
* named by its digest, and prove the written bytes match the identity recorded in the
|
|
113
|
+
* experiment result. A throw marks the attempt `infrastructure_failed` in the executor.
|
|
114
|
+
*/
|
|
115
|
+
export async function saveAttemptEvidence(
|
|
116
|
+
directory: string,
|
|
117
|
+
evidence: unknown,
|
|
118
|
+
digest: string
|
|
119
|
+
): Promise<string> {
|
|
120
|
+
const content = canonicalJson(evidence);
|
|
121
|
+
if (sha256Hex(content) !== digest) throw new Error('evidence_digest_mismatch');
|
|
122
|
+
const evidenceDir = path.join(directory, EXPERIMENT_EVIDENCE_DIR);
|
|
123
|
+
await mkdir(evidenceDir, { recursive: true });
|
|
124
|
+
const file = path.join(evidenceDir, `${digest}.json`);
|
|
125
|
+
try {
|
|
126
|
+
await writeNewFile(file, content);
|
|
127
|
+
} catch (error) {
|
|
128
|
+
// Identical content already stored under its own digest is the same evidence.
|
|
129
|
+
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
|
|
130
|
+
if (sha256Hex(await readFile(file)) !== digest) throw error;
|
|
131
|
+
}
|
|
132
|
+
return file;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Persist the V1 experiment result as pretty JSON. */
|
|
136
|
+
export async function saveExperimentResult(
|
|
137
|
+
directory: string,
|
|
138
|
+
result: ExperimentRunResult
|
|
139
|
+
): Promise<string> {
|
|
140
|
+
const file = path.join(directory, EXPERIMENT_RESULT_FILE);
|
|
141
|
+
try {
|
|
142
|
+
await writeNewFile(file, `${JSON.stringify(result, null, 2)}\n`);
|
|
143
|
+
} catch {
|
|
144
|
+
throw new ExperimentSurfaceError(
|
|
145
|
+
'result_unwritable',
|
|
146
|
+
'Unable to save the experiment result',
|
|
147
|
+
1
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
return file;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const EVIDENCE_DIGEST = /^[a-f0-9]{64}$/;
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Read the saved evidence files named by `digests` from an `evidence/` directory, as exact bytes
|
|
157
|
+
* keyed by digest. A missing, unreadable or oversized file maps to `null` so the case-level
|
|
158
|
+
* summary reports it; the bytes are verified against the digest by the report model, not here.
|
|
159
|
+
* Returns `undefined` when the directory does not exist (results saved without evidence).
|
|
160
|
+
*/
|
|
161
|
+
export async function readExperimentCaseEvidence(
|
|
162
|
+
evidenceDir: string,
|
|
163
|
+
digests: readonly string[]
|
|
164
|
+
): Promise<Map<string, Uint8Array | null> | undefined> {
|
|
165
|
+
try {
|
|
166
|
+
if (!(await stat(evidenceDir)).isDirectory()) return undefined;
|
|
167
|
+
} catch {
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
const files = new Map<string, Uint8Array | null>();
|
|
171
|
+
for (const digest of digests) {
|
|
172
|
+
files.set(
|
|
173
|
+
digest,
|
|
174
|
+
EVIDENCE_DIGEST.test(digest) ? await readEvidenceBytes(evidenceDir, digest) : null
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
return files;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function readEvidenceBytes(evidenceDir: string, digest: string): Promise<Uint8Array | null> {
|
|
181
|
+
const file = path.join(evidenceDir, `${digest}.json`);
|
|
182
|
+
try {
|
|
183
|
+
const metadata = await stat(file);
|
|
184
|
+
if (!metadata.isFile() || metadata.size > MAX_CASE_EVIDENCE_BYTES) return null;
|
|
185
|
+
const bytes = await readFile(file);
|
|
186
|
+
return bytes.byteLength > MAX_CASE_EVIDENCE_BYTES ? null : new Uint8Array(bytes);
|
|
187
|
+
} catch {
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic in-memory model client for fixture-mode experiments.
|
|
3
|
+
*
|
|
4
|
+
* Fixture runs exercise the full experiment pipeline (matrix, executor, evaluators, evidence,
|
|
5
|
+
* reports) without any network access. Responses are fixed text, so fixture outcomes describe
|
|
6
|
+
* the pipeline, never a model.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type {
|
|
10
|
+
ExperimentTarget,
|
|
11
|
+
GenerateOptions,
|
|
12
|
+
GenerateResult,
|
|
13
|
+
ModelClient,
|
|
14
|
+
} from '@artemiskit/core';
|
|
15
|
+
import { sha256Hex } from './experiment-files';
|
|
16
|
+
|
|
17
|
+
export const FIXTURE_PROVIDER = 'artemiskit-fixture';
|
|
18
|
+
|
|
19
|
+
/** Rough token estimate (about four characters per token) so usage is always reported. */
|
|
20
|
+
const CHARACTERS_PER_TOKEN = 4;
|
|
21
|
+
|
|
22
|
+
export interface ExperimentFixtureClientOptions {
|
|
23
|
+
/**
|
|
24
|
+
* Advertise structured tool use. The client accepts tool definitions but never calls a
|
|
25
|
+
* tool, so a fixture workflow can run (the workflow engine requires tool support) and its
|
|
26
|
+
* outcome depends only on the declared state and outcomes.
|
|
27
|
+
*/
|
|
28
|
+
toolUse?: boolean;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Create the offline client bound to one fixture-mode target. */
|
|
32
|
+
export function createExperimentFixtureClient(
|
|
33
|
+
target: ExperimentTarget,
|
|
34
|
+
options: ExperimentFixtureClientOptions = {}
|
|
35
|
+
): ModelClient {
|
|
36
|
+
const toolUse = options.toolUse === true;
|
|
37
|
+
return {
|
|
38
|
+
provider: FIXTURE_PROVIDER,
|
|
39
|
+
async capabilities() {
|
|
40
|
+
return { streaming: false, functionCalling: false, toolUse, maxContext: 128_000 };
|
|
41
|
+
},
|
|
42
|
+
async generate(options: GenerateOptions): Promise<GenerateResult> {
|
|
43
|
+
const prompt = promptText(options.prompt);
|
|
44
|
+
const text = `Fixture response from ${target.id}.`;
|
|
45
|
+
const promptTokens = estimateTokens(prompt);
|
|
46
|
+
const completionTokens = estimateTokens(text);
|
|
47
|
+
return {
|
|
48
|
+
id: `fixture-${sha256Hex(`${target.id}\n${prompt}`).slice(0, 16)}`,
|
|
49
|
+
model: target.model,
|
|
50
|
+
text,
|
|
51
|
+
tokens: {
|
|
52
|
+
prompt: promptTokens,
|
|
53
|
+
completion: completionTokens,
|
|
54
|
+
total: promptTokens + completionTokens,
|
|
55
|
+
},
|
|
56
|
+
usageAvailable: true,
|
|
57
|
+
latencyMs: 0,
|
|
58
|
+
finishReason: 'stop',
|
|
59
|
+
};
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function promptText(prompt: GenerateOptions['prompt']): string {
|
|
65
|
+
return typeof prompt === 'string' ? prompt : prompt.map((message) => message.content).join('\n');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function estimateTokens(text: string): number {
|
|
69
|
+
return Math.max(1, Math.ceil(text.length / CHARACTERS_PER_TOKEN));
|
|
70
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exit codes and the plain-text run summary shared by the SDK and `akit experiment run`.
|
|
3
|
+
* The summary is descriptive: it lists targets in declared order and never ranks them.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { ExperimentRunResult, ExperimentSummary, ExperimentTarget } from '@artemiskit/core';
|
|
7
|
+
import {
|
|
8
|
+
type ExperimentCaseCounts,
|
|
9
|
+
type ExperimentCaseLevelSummary,
|
|
10
|
+
formatCaseRate,
|
|
11
|
+
} from '@artemiskit/reports';
|
|
12
|
+
|
|
13
|
+
/** Exit codes for `akit experiment run`; SDK outcomes carry the same value. */
|
|
14
|
+
export const EXPERIMENT_EXIT_CODES = {
|
|
15
|
+
/** Completed with a valid passed or failed measurement for every declared coordinate. */
|
|
16
|
+
complete: 0,
|
|
17
|
+
/** Unexpected I/O or internal failure, including an unusable output directory. */
|
|
18
|
+
error: 1,
|
|
19
|
+
/** The experiment file, a task source or a target declaration failed validation. */
|
|
20
|
+
invalid: 2,
|
|
21
|
+
/** Live experiment without a valid explicit approval; nothing was executed. */
|
|
22
|
+
unauthorized: 3,
|
|
23
|
+
/** Execution stopped early (budget or usage stop condition); see the stop reason. */
|
|
24
|
+
stopped: 4,
|
|
25
|
+
/** Finished, but some coordinates are invalid, unsupported, excluded, errored or incomplete. */
|
|
26
|
+
coverageIncomplete: 5,
|
|
27
|
+
} as const;
|
|
28
|
+
|
|
29
|
+
export type ExperimentRunStatus = keyof typeof EXPERIMENT_EXIT_CODES;
|
|
30
|
+
export type ExperimentExitCode = (typeof EXPERIMENT_EXIT_CODES)[ExperimentRunStatus];
|
|
31
|
+
|
|
32
|
+
/** Classify a finished run. Early stops take precedence over incomplete coverage. */
|
|
33
|
+
export function experimentRunStatus(
|
|
34
|
+
result: ExperimentRunResult
|
|
35
|
+
): 'complete' | 'stopped' | 'coverageIncomplete' {
|
|
36
|
+
if (result.stop_reason !== undefined || !result.completion.execution_complete) return 'stopped';
|
|
37
|
+
if (!result.completion.valid_measurement_coverage_complete) return 'coverageIncomplete';
|
|
38
|
+
return 'complete';
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Deployment label for a target, e.g. "open-weight, run locally via ollama; cost not applicable". */
|
|
42
|
+
export function describeTargetDeployment(target: ExperimentTarget): string {
|
|
43
|
+
const deployment = target.deployment;
|
|
44
|
+
const weights = target.weights?.availability === 'open' ? 'open-weight, ' : '';
|
|
45
|
+
if (deployment?.locality !== 'local') return `${weights}remote; metered cost`;
|
|
46
|
+
const runtime = deployment.runtime ?? 'an unspecified runtime';
|
|
47
|
+
const cost = deployment.cost_basis === 'not_applicable' ? 'cost not applicable' : 'metered cost';
|
|
48
|
+
return `${weights}run locally via ${runtime}; ${cost}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Declared concurrency limit and, for concurrent runs, the recorded peak and caveat, e.g.
|
|
53
|
+
* "up to 4 coordinates in flight (declared limit); recorded peak 3; completion order and
|
|
54
|
+
* sampled outputs may differ from a serial run".
|
|
55
|
+
*/
|
|
56
|
+
export function describeConcurrency(result: ExperimentRunResult): string {
|
|
57
|
+
const limit = result.manifest.concurrency;
|
|
58
|
+
if (limit === 1) return '1 (serial)';
|
|
59
|
+
const peak = result.execution ? `; recorded peak ${result.execution.peak_in_flight}` : '';
|
|
60
|
+
return `up to ${limit} coordinates in flight (declared limit)${peak}; completion order and sampled outputs may differ from a serial run`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Human-readable summary of a saved run. Makes no ranking or winner claim. */
|
|
64
|
+
export function formatExperimentRunSummary(
|
|
65
|
+
result: ExperimentRunResult,
|
|
66
|
+
options: { resultPath?: string; caseLevel?: ExperimentCaseLevelSummary } = {}
|
|
67
|
+
): string {
|
|
68
|
+
const { manifest, summaries, completion } = result;
|
|
69
|
+
const lines = [
|
|
70
|
+
`Experiment ${result.experiment_id} — ${
|
|
71
|
+
result.mode === 'fixture'
|
|
72
|
+
? 'fixture mode: deterministic offline client; outcomes measure the pipeline, not any model'
|
|
73
|
+
: `live mode; approved by ${result.live_authorization?.approved_by ?? 'unknown'} (${result.live_authorization?.decision_id ?? 'no decision id'})`
|
|
74
|
+
}`,
|
|
75
|
+
`Matrix: ${summaries.overall.planned} planned coordinate(s) = ${manifest.tasks.length} task(s) × ${manifest.targets.length} target(s) × ${manifest.repetitions} repetition(s)`,
|
|
76
|
+
`Concurrency: ${describeConcurrency(result)}`,
|
|
77
|
+
COORDINATE_LEVEL_LINE,
|
|
78
|
+
'Targets (declared order; counts describe this run only):',
|
|
79
|
+
];
|
|
80
|
+
const groups = new Map(summaries.targets.map((group) => [group.key, group.summary]));
|
|
81
|
+
for (const target of manifest.targets) {
|
|
82
|
+
const publisher = target.publisher ? `; published by ${target.publisher.name}` : '';
|
|
83
|
+
lines.push(
|
|
84
|
+
` - ${target.id}: ${target.provider} ${target.model} — ${describeTargetDeployment(target)}${publisher}`
|
|
85
|
+
);
|
|
86
|
+
const summary = groups.get(target.id);
|
|
87
|
+
if (summary) lines.push(` ${formatCounts(summary)}`);
|
|
88
|
+
}
|
|
89
|
+
lines.push(
|
|
90
|
+
`Overall: ${formatCounts(summaries.overall)}`,
|
|
91
|
+
...(options.caseLevel
|
|
92
|
+
? [
|
|
93
|
+
formatExperimentCaseSummary(
|
|
94
|
+
options.caseLevel,
|
|
95
|
+
manifest.targets.map((target) => target.id)
|
|
96
|
+
),
|
|
97
|
+
]
|
|
98
|
+
: []),
|
|
99
|
+
`Stop reason: ${result.stop_reason ?? 'none'}`,
|
|
100
|
+
`Completion: matrix ${completeness(completion.matrix_complete)}; execution ${completeness(
|
|
101
|
+
completion.execution_complete
|
|
102
|
+
)}; valid-measurement coverage ${completeness(completion.valid_measurement_coverage_complete)}`,
|
|
103
|
+
`Usage: ${summaries.operational.requests} request(s); ${summaries.operational.tokens} token(s); cost ${formatCost(result)}`,
|
|
104
|
+
'No ranking or winner is implied. Generate a report for methodology, evidence and limitations.'
|
|
105
|
+
);
|
|
106
|
+
if (options.resultPath) lines.push(`Result: ${options.resultPath}`);
|
|
107
|
+
return lines.join('\n');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const COORDINATE_LEVEL_LINE =
|
|
111
|
+
'Coordinate level: counts below are task × target × repetition coordinates; a scenario coordinate passes only when all of its cases pass.';
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Plain-text case-level summary: individual scenario cases from verified per-attempt evidence,
|
|
115
|
+
* per target (in `targetOrder` when given) and overall, or why it is unavailable. No ranking.
|
|
116
|
+
*/
|
|
117
|
+
export function formatExperimentCaseSummary(
|
|
118
|
+
caseLevel: ExperimentCaseLevelSummary,
|
|
119
|
+
targetOrder: readonly string[] = []
|
|
120
|
+
): string {
|
|
121
|
+
if (caseLevel.availability === 'unavailable')
|
|
122
|
+
return `Case level: unavailable. ${caseLevel.detail}`;
|
|
123
|
+
const position = (key: string) => {
|
|
124
|
+
const index = targetOrder.indexOf(key);
|
|
125
|
+
return index === -1 ? Number.MAX_SAFE_INTEGER : index;
|
|
126
|
+
};
|
|
127
|
+
const targets = [...caseLevel.targets].sort(
|
|
128
|
+
(left, right) => position(left.key) - position(right.key)
|
|
129
|
+
);
|
|
130
|
+
return [
|
|
131
|
+
`Case level: counts below are individual scenario cases from ${caseLevel.verifiedEvidenceFiles} SHA-256-verified evidence file(s) (terminal attempt per coordinate):`,
|
|
132
|
+
...targets.map((group) => ` - ${group.key}: ${formatCaseCounts(group.counts)}`),
|
|
133
|
+
`Overall cases: ${formatCaseCounts(caseLevel.overall)}`,
|
|
134
|
+
].join('\n');
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function formatCaseCounts(counts: ExperimentCaseCounts): string {
|
|
138
|
+
const unknown = (value: number | null) => (value === null ? 'unavailable' : String(value));
|
|
139
|
+
return [
|
|
140
|
+
`passed ${counts.passed} and failed ${counts.failed} of ${counts.valid} valid case(s) (passed/valid ${formatCaseRate(counts)})`,
|
|
141
|
+
`invalid ${counts.invalid}`,
|
|
142
|
+
`planned ${unknown(counts.planned)}`,
|
|
143
|
+
`unmeasured ${unknown(counts.unmeasured)}`,
|
|
144
|
+
`coordinates without case evidence ${counts.coordinatesWithoutCaseEvidence}`,
|
|
145
|
+
`agent-workflow coordinates ${counts.coordinatesNotApplicable}`,
|
|
146
|
+
].join('; ');
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function formatCounts(summary: ExperimentSummary): string {
|
|
150
|
+
return [
|
|
151
|
+
`valid ${summary.valid}/${summary.planned} planned`,
|
|
152
|
+
`passed ${summary.passed} and task-failed ${summary.task_failed} of ${summary.valid} valid`,
|
|
153
|
+
`invalid ${summary.invalid}`,
|
|
154
|
+
`infrastructure-failed ${summary.infrastructure_failed}`,
|
|
155
|
+
`unsupported ${summary.unsupported}`,
|
|
156
|
+
`excluded ${summary.excluded}`,
|
|
157
|
+
`incomplete ${summary.incomplete}`,
|
|
158
|
+
].join('; ');
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function formatCost(result: ExperimentRunResult): string {
|
|
162
|
+
const allLocal = result.manifest.targets.every(
|
|
163
|
+
(target) => target.deployment?.cost_basis === 'not_applicable'
|
|
164
|
+
);
|
|
165
|
+
if (allLocal) return 'not applicable (local targets only)';
|
|
166
|
+
const cost = result.summaries.operational.cost;
|
|
167
|
+
return cost ? `${cost.amount} ${cost.currency}` : 'not reported';
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function completeness(value: boolean): string {
|
|
171
|
+
return value ? 'complete' : 'incomplete';
|
|
172
|
+
}
|