@artemiskit/sdk 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +17 -0
- package/dist/__tests__/experiment-fixtures.d.ts +44 -0
- package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
- package/dist/artemiskit.d.ts +6 -1
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/builders/index.js +19 -66
- package/dist/contracts/index.js +3 -50
- package/dist/experiment-files.d.ts +54 -0
- package/dist/experiment-files.d.ts.map +1 -0
- package/dist/experiment-fixture.d.ts +20 -0
- package/dist/experiment-fixture.d.ts.map +1 -0
- package/dist/experiment-summary.d.ts +44 -0
- package/dist/experiment-summary.d.ts.map +1 -0
- package/dist/experiment-validation.d.ts +48 -0
- package/dist/experiment-validation.d.ts.map +1 -0
- package/dist/experiment.d.ts +162 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +6 -1
- package/dist/guardian/circuit-breaker.d.ts.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2516 -1801
- package/dist/matchers/index.js +13 -60
- package/dist/matchers/jest.js +0 -47
- package/dist/matchers/vitest.js +0 -47
- package/dist/types-only.d.ts +1 -0
- package/dist/types-only.d.ts.map +1 -1
- package/dist/utils/index.js +11 -58
- package/package.json +6 -4
- package/src/__tests__/artemiskit.test.ts +1 -2
- package/src/__tests__/builders.test.ts +0 -2
- package/src/__tests__/circuit-breaker.test.ts +1 -1
- package/src/__tests__/compare.test.ts +1 -1
- package/src/__tests__/contracts.test.ts +1 -1
- package/src/__tests__/experiment-case-level.test.ts +212 -0
- package/src/__tests__/experiment-fixtures.ts +194 -0
- package/src/__tests__/experiment.test.ts +1262 -0
- package/src/__tests__/guardian-modes.test.ts +2 -2
- package/src/__tests__/guardrails.test.ts +1 -1
- package/src/__tests__/matchers.test.ts +1 -1
- package/src/__tests__/semantic-validator.test.ts +15 -10
- package/src/__tests__/validate.test.ts +0 -4
- package/src/__tests__/workflow-recovery.test.ts +7 -3
- package/src/__tests__/workflow-report.test.ts +112 -0
- package/src/__tests__/workflow.test.ts +11 -5
- package/src/artemiskit.ts +16 -1
- package/src/experiment-files.ts +190 -0
- package/src/experiment-fixture.ts +70 -0
- package/src/experiment-summary.ts +172 -0
- package/src/experiment-validation.ts +426 -0
- package/src/experiment.ts +630 -0
- package/src/guardian/circuit-breaker.ts +6 -1
- package/src/index.ts +71 -0
- package/src/types-only.ts +11 -0
- package/tsconfig.typecheck.json +12 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test helper: write a self-contained two-target experiment directory.
|
|
3
|
+
* Shared by the SDK and CLI experiment tests.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
|
|
8
|
+
import { tmpdir } from 'node:os';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
|
|
11
|
+
/** Passes against the fixture client, whose responses start with "Fixture response". */
|
|
12
|
+
export const PASSING_SCENARIO = `name: fixture-pass
|
|
13
|
+
cases:
|
|
14
|
+
- id: echo
|
|
15
|
+
prompt: "Reply with anything."
|
|
16
|
+
expected:
|
|
17
|
+
type: contains
|
|
18
|
+
values: ["Fixture response"]
|
|
19
|
+
`;
|
|
20
|
+
|
|
21
|
+
/** A valid measurement that fails against the fixture client. */
|
|
22
|
+
export const FAILING_SCENARIO = `name: fixture-fail
|
|
23
|
+
cases:
|
|
24
|
+
- id: capital
|
|
25
|
+
prompt: "What is the capital of Nigeria?"
|
|
26
|
+
expected:
|
|
27
|
+
type: contains
|
|
28
|
+
values: ["Abuja"]
|
|
29
|
+
`;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Agent workflow that passes against the fixture client, which accepts tools but never calls
|
|
33
|
+
* one: the target must answer without reading the brief and without any undeclared tool.
|
|
34
|
+
*/
|
|
35
|
+
export const FIXTURE_WORKFLOW = `version: '1'
|
|
36
|
+
kind: agent_workflow
|
|
37
|
+
name: fixture-workflow
|
|
38
|
+
target:
|
|
39
|
+
provider: openai
|
|
40
|
+
model: declared-model
|
|
41
|
+
environment:
|
|
42
|
+
type: simulated
|
|
43
|
+
policy:
|
|
44
|
+
network: denied
|
|
45
|
+
side_effects: denied
|
|
46
|
+
permissions:
|
|
47
|
+
documents: read
|
|
48
|
+
budgets:
|
|
49
|
+
max_actions: 4
|
|
50
|
+
max_model_requests: 2
|
|
51
|
+
timeout_ms: 10000
|
|
52
|
+
tools:
|
|
53
|
+
- read_document
|
|
54
|
+
workflow:
|
|
55
|
+
system_instructions: Answer from the conversation; read the brief only when asked to.
|
|
56
|
+
initial_state:
|
|
57
|
+
documents:
|
|
58
|
+
brief: Fixture brief.
|
|
59
|
+
turns:
|
|
60
|
+
- role: user
|
|
61
|
+
content: Reply with anything.
|
|
62
|
+
outcomes:
|
|
63
|
+
deterministic:
|
|
64
|
+
- type: tool_trace
|
|
65
|
+
tool: read_document
|
|
66
|
+
minimum_calls: 0
|
|
67
|
+
maximum_calls: 0
|
|
68
|
+
- type: policy
|
|
69
|
+
rule: no_undeclared_tool
|
|
70
|
+
expected: passed
|
|
71
|
+
evidence:
|
|
72
|
+
trace: summary
|
|
73
|
+
artifacts: checksums
|
|
74
|
+
redact: true
|
|
75
|
+
`;
|
|
76
|
+
|
|
77
|
+
export const LOOPBACK_BASE_URL = 'http://127.0.0.1:11434/v1';
|
|
78
|
+
|
|
79
|
+
export interface ExperimentFixtureOptions {
|
|
80
|
+
/** Override manifest fields after the defaults are built. */
|
|
81
|
+
overrides?: Record<string, unknown>;
|
|
82
|
+
/** Replace a task's declared digest. */
|
|
83
|
+
digestOverride?: Record<string, string>;
|
|
84
|
+
/** Skip writing these source files. */
|
|
85
|
+
omitSources?: string[];
|
|
86
|
+
/** Target transforms applied to both default targets. */
|
|
87
|
+
target?: (target: Record<string, unknown>) => Record<string, unknown>;
|
|
88
|
+
/** Manifest file name (default `experiment.yaml`; the content is JSON, which YAML accepts). */
|
|
89
|
+
fileName?: string;
|
|
90
|
+
/**
|
|
91
|
+
* Declare one scenario task (`pass`) and one agent workflow task (`workflow`) instead of the
|
|
92
|
+
* two scenario tasks.
|
|
93
|
+
*/
|
|
94
|
+
workflow?: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface ExperimentFixture {
|
|
98
|
+
directory: string;
|
|
99
|
+
file: string;
|
|
100
|
+
manifest: Record<string, unknown>;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const SOURCES: Record<string, string> = {
|
|
104
|
+
'tasks/pass.yaml': PASSING_SCENARIO,
|
|
105
|
+
'tasks/fail.yaml': FAILING_SCENARIO,
|
|
106
|
+
'tasks/workflow.yaml': FIXTURE_WORKFLOW,
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
export function sha256(text: string): string {
|
|
110
|
+
return createHash('sha256').update(text).digest('hex');
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function localTarget(id: string, model: string): Record<string, unknown> {
|
|
114
|
+
return {
|
|
115
|
+
id,
|
|
116
|
+
provider: 'openai',
|
|
117
|
+
model,
|
|
118
|
+
capabilities: ['text'],
|
|
119
|
+
settings: { base_url: LOOPBACK_BASE_URL },
|
|
120
|
+
deployment: { locality: 'local', runtime: 'ollama', cost_basis: 'not_applicable' },
|
|
121
|
+
publisher: { id: 'qwen', name: 'Qwen (Alibaba Cloud)' },
|
|
122
|
+
weights: { availability: 'open', license: 'Apache-2.0' },
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Live controls every live manifest must declare. */
|
|
127
|
+
export const LIVE_CONTROLS = {
|
|
128
|
+
concurrency_ceiling: 1,
|
|
129
|
+
stop_conditions: [
|
|
130
|
+
'request_budget_exhausted',
|
|
131
|
+
'token_budget_exhausted',
|
|
132
|
+
'cost_budget_exhausted',
|
|
133
|
+
'usage_unreported',
|
|
134
|
+
'usage_invalid',
|
|
135
|
+
'budget_exceeded',
|
|
136
|
+
],
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
export async function writeExperimentFixture(
|
|
140
|
+
options: ExperimentFixtureOptions = {}
|
|
141
|
+
): Promise<ExperimentFixture> {
|
|
142
|
+
const directory = await mkdtemp(path.join(tmpdir(), 'akit-experiment-'));
|
|
143
|
+
await mkdir(path.join(directory, 'tasks'), { recursive: true });
|
|
144
|
+
for (const [relative, content] of Object.entries(SOURCES)) {
|
|
145
|
+
if (!options.omitSources?.includes(relative)) {
|
|
146
|
+
await writeFile(path.join(directory, relative), content);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const task = (id: string, sourcePath: string, kind = 'scenario_evaluation') => ({
|
|
150
|
+
id,
|
|
151
|
+
kind,
|
|
152
|
+
source: {
|
|
153
|
+
kind,
|
|
154
|
+
path: sourcePath,
|
|
155
|
+
artifact: {
|
|
156
|
+
schema_version: '1',
|
|
157
|
+
algorithm: 'sha256',
|
|
158
|
+
digest: options.digestOverride?.[id] ?? sha256(SOURCES[sourcePath]),
|
|
159
|
+
},
|
|
160
|
+
},
|
|
161
|
+
required_capabilities: ['text'],
|
|
162
|
+
});
|
|
163
|
+
const transform = options.target ?? ((target) => target);
|
|
164
|
+
const fixed = (digit: string) => ({
|
|
165
|
+
schema_version: '1',
|
|
166
|
+
algorithm: 'sha256',
|
|
167
|
+
digest: digit.repeat(64),
|
|
168
|
+
});
|
|
169
|
+
const manifest: Record<string, unknown> = {
|
|
170
|
+
schema_version: '1',
|
|
171
|
+
id: 'two-target-fixture',
|
|
172
|
+
mode: 'fixture',
|
|
173
|
+
identities: { schema_version: '1', workload: fixed('a'), rubric: fixed('b') },
|
|
174
|
+
tasks: [
|
|
175
|
+
task('pass', 'tasks/pass.yaml'),
|
|
176
|
+
options.workflow
|
|
177
|
+
? task('workflow', 'tasks/workflow.yaml', 'agent_workflow')
|
|
178
|
+
: task('fail', 'tasks/fail.yaml'),
|
|
179
|
+
],
|
|
180
|
+
targets: [
|
|
181
|
+
transform(localTarget('qwen-small', 'qwen3.5:4b')),
|
|
182
|
+
transform(localTarget('qwen-large', 'qwen3.5:9b')),
|
|
183
|
+
],
|
|
184
|
+
repetitions: 1,
|
|
185
|
+
concurrency: 1,
|
|
186
|
+
exclusions: [],
|
|
187
|
+
retry_policy: { max_attempts: 1, retry_on: [] },
|
|
188
|
+
budgets: { max_requests: 100 },
|
|
189
|
+
...options.overrides,
|
|
190
|
+
};
|
|
191
|
+
const file = path.join(directory, options.fileName ?? 'experiment.yaml');
|
|
192
|
+
await writeFile(file, JSON.stringify(manifest, null, 2));
|
|
193
|
+
return { directory, file, manifest };
|
|
194
|
+
}
|