@artemiskit/sdk 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +17 -0
  3. package/dist/__tests__/experiment-fixtures.d.ts +44 -0
  4. package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
  5. package/dist/artemiskit.d.ts +6 -1
  6. package/dist/artemiskit.d.ts.map +1 -1
  7. package/dist/builders/index.js +19 -66
  8. package/dist/contracts/index.js +3 -50
  9. package/dist/experiment-files.d.ts +54 -0
  10. package/dist/experiment-files.d.ts.map +1 -0
  11. package/dist/experiment-fixture.d.ts +20 -0
  12. package/dist/experiment-fixture.d.ts.map +1 -0
  13. package/dist/experiment-summary.d.ts +44 -0
  14. package/dist/experiment-summary.d.ts.map +1 -0
  15. package/dist/experiment-validation.d.ts +48 -0
  16. package/dist/experiment-validation.d.ts.map +1 -0
  17. package/dist/experiment.d.ts +162 -0
  18. package/dist/experiment.d.ts.map +1 -0
  19. package/dist/guardian/circuit-breaker.d.ts +6 -1
  20. package/dist/guardian/circuit-breaker.d.ts.map +1 -1
  21. package/dist/index.d.ts +7 -0
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +2516 -1801
  24. package/dist/matchers/index.js +13 -60
  25. package/dist/matchers/jest.js +0 -47
  26. package/dist/matchers/vitest.js +0 -47
  27. package/dist/types-only.d.ts +1 -0
  28. package/dist/types-only.d.ts.map +1 -1
  29. package/dist/utils/index.js +11 -58
  30. package/package.json +6 -4
  31. package/src/__tests__/artemiskit.test.ts +1 -2
  32. package/src/__tests__/builders.test.ts +0 -2
  33. package/src/__tests__/circuit-breaker.test.ts +1 -1
  34. package/src/__tests__/compare.test.ts +1 -1
  35. package/src/__tests__/contracts.test.ts +1 -1
  36. package/src/__tests__/experiment-case-level.test.ts +212 -0
  37. package/src/__tests__/experiment-fixtures.ts +194 -0
  38. package/src/__tests__/experiment.test.ts +1262 -0
  39. package/src/__tests__/guardian-modes.test.ts +2 -2
  40. package/src/__tests__/guardrails.test.ts +1 -1
  41. package/src/__tests__/matchers.test.ts +1 -1
  42. package/src/__tests__/semantic-validator.test.ts +15 -10
  43. package/src/__tests__/validate.test.ts +0 -4
  44. package/src/__tests__/workflow-recovery.test.ts +7 -3
  45. package/src/__tests__/workflow-report.test.ts +112 -0
  46. package/src/__tests__/workflow.test.ts +11 -5
  47. package/src/artemiskit.ts +16 -1
  48. package/src/experiment-files.ts +190 -0
  49. package/src/experiment-fixture.ts +70 -0
  50. package/src/experiment-summary.ts +172 -0
  51. package/src/experiment-validation.ts +426 -0
  52. package/src/experiment.ts +630 -0
  53. package/src/guardian/circuit-breaker.ts +6 -1
  54. package/src/index.ts +71 -0
  55. package/src/types-only.ts +11 -0
  56. package/tsconfig.typecheck.json +12 -0
@@ -0,0 +1,194 @@
1
+ /**
2
+ * Test helper: write a self-contained two-target experiment directory.
3
+ * Shared by the SDK and CLI experiment tests.
4
+ */
5
+
6
+ import { createHash } from 'node:crypto';
7
+ import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
8
+ import { tmpdir } from 'node:os';
9
+ import path from 'node:path';
10
+
11
+ /** Passes against the fixture client, whose responses start with "Fixture response". */
12
+ export const PASSING_SCENARIO = `name: fixture-pass
13
+ cases:
14
+ - id: echo
15
+ prompt: "Reply with anything."
16
+ expected:
17
+ type: contains
18
+ values: ["Fixture response"]
19
+ `;
20
+
21
+ /** A valid measurement that fails against the fixture client. */
22
+ export const FAILING_SCENARIO = `name: fixture-fail
23
+ cases:
24
+ - id: capital
25
+ prompt: "What is the capital of Nigeria?"
26
+ expected:
27
+ type: contains
28
+ values: ["Abuja"]
29
+ `;
30
+
31
+ /**
32
+ * Agent workflow that passes against the fixture client, which accepts tools but never calls
33
+ * one: the target must answer without reading the brief and without any undeclared tool.
34
+ */
35
+ export const FIXTURE_WORKFLOW = `version: '1'
36
+ kind: agent_workflow
37
+ name: fixture-workflow
38
+ target:
39
+ provider: openai
40
+ model: declared-model
41
+ environment:
42
+ type: simulated
43
+ policy:
44
+ network: denied
45
+ side_effects: denied
46
+ permissions:
47
+ documents: read
48
+ budgets:
49
+ max_actions: 4
50
+ max_model_requests: 2
51
+ timeout_ms: 10000
52
+ tools:
53
+ - read_document
54
+ workflow:
55
+ system_instructions: Answer from the conversation; read the brief only when asked to.
56
+ initial_state:
57
+ documents:
58
+ brief: Fixture brief.
59
+ turns:
60
+ - role: user
61
+ content: Reply with anything.
62
+ outcomes:
63
+ deterministic:
64
+ - type: tool_trace
65
+ tool: read_document
66
+ minimum_calls: 0
67
+ maximum_calls: 0
68
+ - type: policy
69
+ rule: no_undeclared_tool
70
+ expected: passed
71
+ evidence:
72
+ trace: summary
73
+ artifacts: checksums
74
+ redact: true
75
+ `;
76
+
77
+ export const LOOPBACK_BASE_URL = 'http://127.0.0.1:11434/v1';
78
+
79
+ export interface ExperimentFixtureOptions {
80
+ /** Override manifest fields after the defaults are built. */
81
+ overrides?: Record<string, unknown>;
82
+ /** Replace a task's declared digest. */
83
+ digestOverride?: Record<string, string>;
84
+ /** Skip writing these source files. */
85
+ omitSources?: string[];
86
+ /** Target transforms applied to both default targets. */
87
+ target?: (target: Record<string, unknown>) => Record<string, unknown>;
88
+ /** Manifest file name (default `experiment.yaml`; the content is JSON, which YAML accepts). */
89
+ fileName?: string;
90
+ /**
91
+ * Declare one scenario task (`pass`) and one agent workflow task (`workflow`) instead of the
92
+ * two scenario tasks.
93
+ */
94
+ workflow?: boolean;
95
+ }
96
+
97
+ export interface ExperimentFixture {
98
+ directory: string;
99
+ file: string;
100
+ manifest: Record<string, unknown>;
101
+ }
102
+
103
+ const SOURCES: Record<string, string> = {
104
+ 'tasks/pass.yaml': PASSING_SCENARIO,
105
+ 'tasks/fail.yaml': FAILING_SCENARIO,
106
+ 'tasks/workflow.yaml': FIXTURE_WORKFLOW,
107
+ };
108
+
109
+ export function sha256(text: string): string {
110
+ return createHash('sha256').update(text).digest('hex');
111
+ }
112
+
113
+ function localTarget(id: string, model: string): Record<string, unknown> {
114
+ return {
115
+ id,
116
+ provider: 'openai',
117
+ model,
118
+ capabilities: ['text'],
119
+ settings: { base_url: LOOPBACK_BASE_URL },
120
+ deployment: { locality: 'local', runtime: 'ollama', cost_basis: 'not_applicable' },
121
+ publisher: { id: 'qwen', name: 'Qwen (Alibaba Cloud)' },
122
+ weights: { availability: 'open', license: 'Apache-2.0' },
123
+ };
124
+ }
125
+
126
+ /** Live controls every live manifest must declare. */
127
+ export const LIVE_CONTROLS = {
128
+ concurrency_ceiling: 1,
129
+ stop_conditions: [
130
+ 'request_budget_exhausted',
131
+ 'token_budget_exhausted',
132
+ 'cost_budget_exhausted',
133
+ 'usage_unreported',
134
+ 'usage_invalid',
135
+ 'budget_exceeded',
136
+ ],
137
+ };
138
+
139
+ export async function writeExperimentFixture(
140
+ options: ExperimentFixtureOptions = {}
141
+ ): Promise<ExperimentFixture> {
142
+ const directory = await mkdtemp(path.join(tmpdir(), 'akit-experiment-'));
143
+ await mkdir(path.join(directory, 'tasks'), { recursive: true });
144
+ for (const [relative, content] of Object.entries(SOURCES)) {
145
+ if (!options.omitSources?.includes(relative)) {
146
+ await writeFile(path.join(directory, relative), content);
147
+ }
148
+ }
149
+ const task = (id: string, sourcePath: string, kind = 'scenario_evaluation') => ({
150
+ id,
151
+ kind,
152
+ source: {
153
+ kind,
154
+ path: sourcePath,
155
+ artifact: {
156
+ schema_version: '1',
157
+ algorithm: 'sha256',
158
+ digest: options.digestOverride?.[id] ?? sha256(SOURCES[sourcePath]),
159
+ },
160
+ },
161
+ required_capabilities: ['text'],
162
+ });
163
+ const transform = options.target ?? ((target) => target);
164
+ const fixed = (digit: string) => ({
165
+ schema_version: '1',
166
+ algorithm: 'sha256',
167
+ digest: digit.repeat(64),
168
+ });
169
+ const manifest: Record<string, unknown> = {
170
+ schema_version: '1',
171
+ id: 'two-target-fixture',
172
+ mode: 'fixture',
173
+ identities: { schema_version: '1', workload: fixed('a'), rubric: fixed('b') },
174
+ tasks: [
175
+ task('pass', 'tasks/pass.yaml'),
176
+ options.workflow
177
+ ? task('workflow', 'tasks/workflow.yaml', 'agent_workflow')
178
+ : task('fail', 'tasks/fail.yaml'),
179
+ ],
180
+ targets: [
181
+ transform(localTarget('qwen-small', 'qwen3.5:4b')),
182
+ transform(localTarget('qwen-large', 'qwen3.5:9b')),
183
+ ],
184
+ repetitions: 1,
185
+ concurrency: 1,
186
+ exclusions: [],
187
+ retry_policy: { max_attempts: 1, retry_on: [] },
188
+ budgets: { max_requests: 100 },
189
+ ...options.overrides,
190
+ };
191
+ const file = path.join(directory, options.fileName ?? 'experiment.yaml');
192
+ await writeFile(file, JSON.stringify(manifest, null, 2));
193
+ return { directory, file, manifest };
194
+ }