@artemiskit/sdk 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +34 -0
  2. package/README.md +17 -0
  3. package/dist/__tests__/experiment-fixtures.d.ts +44 -0
  4. package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
  5. package/dist/artemiskit.d.ts +6 -1
  6. package/dist/artemiskit.d.ts.map +1 -1
  7. package/dist/builders/index.js +19 -66
  8. package/dist/contracts/index.js +3 -50
  9. package/dist/experiment-files.d.ts +54 -0
  10. package/dist/experiment-files.d.ts.map +1 -0
  11. package/dist/experiment-fixture.d.ts +20 -0
  12. package/dist/experiment-fixture.d.ts.map +1 -0
  13. package/dist/experiment-summary.d.ts +44 -0
  14. package/dist/experiment-summary.d.ts.map +1 -0
  15. package/dist/experiment-validation.d.ts +48 -0
  16. package/dist/experiment-validation.d.ts.map +1 -0
  17. package/dist/experiment.d.ts +162 -0
  18. package/dist/experiment.d.ts.map +1 -0
  19. package/dist/guardian/circuit-breaker.d.ts +6 -1
  20. package/dist/guardian/circuit-breaker.d.ts.map +1 -1
  21. package/dist/index.d.ts +7 -0
  22. package/dist/index.d.ts.map +1 -1
  23. package/dist/index.js +2516 -1801
  24. package/dist/matchers/index.js +13 -60
  25. package/dist/matchers/jest.js +0 -47
  26. package/dist/matchers/vitest.js +0 -47
  27. package/dist/types-only.d.ts +1 -0
  28. package/dist/types-only.d.ts.map +1 -1
  29. package/dist/utils/index.js +11 -58
  30. package/package.json +6 -4
  31. package/src/__tests__/artemiskit.test.ts +1 -2
  32. package/src/__tests__/builders.test.ts +0 -2
  33. package/src/__tests__/circuit-breaker.test.ts +1 -1
  34. package/src/__tests__/compare.test.ts +1 -1
  35. package/src/__tests__/contracts.test.ts +1 -1
  36. package/src/__tests__/experiment-case-level.test.ts +212 -0
  37. package/src/__tests__/experiment-fixtures.ts +194 -0
  38. package/src/__tests__/experiment.test.ts +1262 -0
  39. package/src/__tests__/guardian-modes.test.ts +2 -2
  40. package/src/__tests__/guardrails.test.ts +1 -1
  41. package/src/__tests__/matchers.test.ts +1 -1
  42. package/src/__tests__/semantic-validator.test.ts +15 -10
  43. package/src/__tests__/validate.test.ts +0 -4
  44. package/src/__tests__/workflow-recovery.test.ts +7 -3
  45. package/src/__tests__/workflow-report.test.ts +112 -0
  46. package/src/__tests__/workflow.test.ts +11 -5
  47. package/src/artemiskit.ts +16 -1
  48. package/src/experiment-files.ts +190 -0
  49. package/src/experiment-fixture.ts +70 -0
  50. package/src/experiment-summary.ts +172 -0
  51. package/src/experiment-validation.ts +426 -0
  52. package/src/experiment.ts +630 -0
  53. package/src/guardian/circuit-breaker.ts +6 -1
  54. package/src/index.ts +71 -0
  55. package/src/types-only.ts +11 -0
  56. package/tsconfig.typecheck.json +12 -0
@@ -0,0 +1,172 @@
1
+ /**
2
+ * Exit codes and the plain-text run summary shared by the SDK and `akit experiment run`.
3
+ * The summary is descriptive: it lists targets in declared order and never ranks them.
4
+ */
5
+
6
+ import type { ExperimentRunResult, ExperimentSummary, ExperimentTarget } from '@artemiskit/core';
7
+ import {
8
+ type ExperimentCaseCounts,
9
+ type ExperimentCaseLevelSummary,
10
+ formatCaseRate,
11
+ } from '@artemiskit/reports';
12
+
13
+ /** Exit codes for `akit experiment run`; SDK outcomes carry the same value. */
14
+ export const EXPERIMENT_EXIT_CODES = {
15
+ /** Completed with a valid passed or failed measurement for every declared coordinate. */
16
+ complete: 0,
17
+ /** Unexpected I/O or internal failure, including an unusable output directory. */
18
+ error: 1,
19
+ /** The experiment file, a task source or a target declaration failed validation. */
20
+ invalid: 2,
21
+ /** Live experiment without a valid explicit approval; nothing was executed. */
22
+ unauthorized: 3,
23
+ /** Execution stopped early (budget or usage stop condition); see the stop reason. */
24
+ stopped: 4,
25
+ /** Finished, but some coordinates are invalid, unsupported, excluded, errored or incomplete. */
26
+ coverageIncomplete: 5,
27
+ } as const;
28
+
29
+ export type ExperimentRunStatus = keyof typeof EXPERIMENT_EXIT_CODES;
30
+ export type ExperimentExitCode = (typeof EXPERIMENT_EXIT_CODES)[ExperimentRunStatus];
31
+
32
+ /** Classify a finished run. Early stops take precedence over incomplete coverage. */
33
+ export function experimentRunStatus(
34
+ result: ExperimentRunResult
35
+ ): 'complete' | 'stopped' | 'coverageIncomplete' {
36
+ if (result.stop_reason !== undefined || !result.completion.execution_complete) return 'stopped';
37
+ if (!result.completion.valid_measurement_coverage_complete) return 'coverageIncomplete';
38
+ return 'complete';
39
+ }
40
+
41
+ /** Deployment label for a target, e.g. "open-weight, run locally via ollama; cost not applicable". */
42
+ export function describeTargetDeployment(target: ExperimentTarget): string {
43
+ const deployment = target.deployment;
44
+ const weights = target.weights?.availability === 'open' ? 'open-weight, ' : '';
45
+ if (deployment?.locality !== 'local') return `${weights}remote; metered cost`;
46
+ const runtime = deployment.runtime ?? 'an unspecified runtime';
47
+ const cost = deployment.cost_basis === 'not_applicable' ? 'cost not applicable' : 'metered cost';
48
+ return `${weights}run locally via ${runtime}; ${cost}`;
49
+ }
50
+
51
+ /**
52
+ * Declared concurrency limit and, for concurrent runs, the recorded peak and caveat, e.g.
53
+ * "up to 4 coordinates in flight (declared limit); recorded peak 3; completion order and
54
+ * sampled outputs may differ from a serial run".
55
+ */
56
+ export function describeConcurrency(result: ExperimentRunResult): string {
57
+ const limit = result.manifest.concurrency;
58
+ if (limit === 1) return '1 (serial)';
59
+ const peak = result.execution ? `; recorded peak ${result.execution.peak_in_flight}` : '';
60
+ return `up to ${limit} coordinates in flight (declared limit)${peak}; completion order and sampled outputs may differ from a serial run`;
61
+ }
62
+
63
+ /** Human-readable summary of a saved run. Makes no ranking or winner claim. */
64
+ export function formatExperimentRunSummary(
65
+ result: ExperimentRunResult,
66
+ options: { resultPath?: string; caseLevel?: ExperimentCaseLevelSummary } = {}
67
+ ): string {
68
+ const { manifest, summaries, completion } = result;
69
+ const lines = [
70
+ `Experiment ${result.experiment_id} — ${
71
+ result.mode === 'fixture'
72
+ ? 'fixture mode: deterministic offline client; outcomes measure the pipeline, not any model'
73
+ : `live mode; approved by ${result.live_authorization?.approved_by ?? 'unknown'} (${result.live_authorization?.decision_id ?? 'no decision id'})`
74
+ }`,
75
+ `Matrix: ${summaries.overall.planned} planned coordinate(s) = ${manifest.tasks.length} task(s) × ${manifest.targets.length} target(s) × ${manifest.repetitions} repetition(s)`,
76
+ `Concurrency: ${describeConcurrency(result)}`,
77
+ COORDINATE_LEVEL_LINE,
78
+ 'Targets (declared order; counts describe this run only):',
79
+ ];
80
+ const groups = new Map(summaries.targets.map((group) => [group.key, group.summary]));
81
+ for (const target of manifest.targets) {
82
+ const publisher = target.publisher ? `; published by ${target.publisher.name}` : '';
83
+ lines.push(
84
+ ` - ${target.id}: ${target.provider} ${target.model} — ${describeTargetDeployment(target)}${publisher}`
85
+ );
86
+ const summary = groups.get(target.id);
87
+ if (summary) lines.push(` ${formatCounts(summary)}`);
88
+ }
89
+ lines.push(
90
+ `Overall: ${formatCounts(summaries.overall)}`,
91
+ ...(options.caseLevel
92
+ ? [
93
+ formatExperimentCaseSummary(
94
+ options.caseLevel,
95
+ manifest.targets.map((target) => target.id)
96
+ ),
97
+ ]
98
+ : []),
99
+ `Stop reason: ${result.stop_reason ?? 'none'}`,
100
+ `Completion: matrix ${completeness(completion.matrix_complete)}; execution ${completeness(
101
+ completion.execution_complete
102
+ )}; valid-measurement coverage ${completeness(completion.valid_measurement_coverage_complete)}`,
103
+ `Usage: ${summaries.operational.requests} request(s); ${summaries.operational.tokens} token(s); cost ${formatCost(result)}`,
104
+ 'No ranking or winner is implied. Generate a report for methodology, evidence and limitations.'
105
+ );
106
+ if (options.resultPath) lines.push(`Result: ${options.resultPath}`);
107
+ return lines.join('\n');
108
+ }
109
+
110
+ const COORDINATE_LEVEL_LINE =
111
+ 'Coordinate level: counts below are task × target × repetition coordinates; a scenario coordinate passes only when all of its cases pass.';
112
+
113
+ /**
114
+ * Plain-text case-level summary: individual scenario cases from verified per-attempt evidence,
115
+ * per target (in `targetOrder` when given) and overall, or why it is unavailable. No ranking.
116
+ */
117
+ export function formatExperimentCaseSummary(
118
+ caseLevel: ExperimentCaseLevelSummary,
119
+ targetOrder: readonly string[] = []
120
+ ): string {
121
+ if (caseLevel.availability === 'unavailable')
122
+ return `Case level: unavailable. ${caseLevel.detail}`;
123
+ const position = (key: string) => {
124
+ const index = targetOrder.indexOf(key);
125
+ return index === -1 ? Number.MAX_SAFE_INTEGER : index;
126
+ };
127
+ const targets = [...caseLevel.targets].sort(
128
+ (left, right) => position(left.key) - position(right.key)
129
+ );
130
+ return [
131
+ `Case level: counts below are individual scenario cases from ${caseLevel.verifiedEvidenceFiles} SHA-256-verified evidence file(s) (terminal attempt per coordinate):`,
132
+ ...targets.map((group) => ` - ${group.key}: ${formatCaseCounts(group.counts)}`),
133
+ `Overall cases: ${formatCaseCounts(caseLevel.overall)}`,
134
+ ].join('\n');
135
+ }
136
+
137
+ function formatCaseCounts(counts: ExperimentCaseCounts): string {
138
+ const unknown = (value: number | null) => (value === null ? 'unavailable' : String(value));
139
+ return [
140
+ `passed ${counts.passed} and failed ${counts.failed} of ${counts.valid} valid case(s) (passed/valid ${formatCaseRate(counts)})`,
141
+ `invalid ${counts.invalid}`,
142
+ `planned ${unknown(counts.planned)}`,
143
+ `unmeasured ${unknown(counts.unmeasured)}`,
144
+ `coordinates without case evidence ${counts.coordinatesWithoutCaseEvidence}`,
145
+ `agent-workflow coordinates ${counts.coordinatesNotApplicable}`,
146
+ ].join('; ');
147
+ }
148
+
149
+ function formatCounts(summary: ExperimentSummary): string {
150
+ return [
151
+ `valid ${summary.valid}/${summary.planned} planned`,
152
+ `passed ${summary.passed} and task-failed ${summary.task_failed} of ${summary.valid} valid`,
153
+ `invalid ${summary.invalid}`,
154
+ `infrastructure-failed ${summary.infrastructure_failed}`,
155
+ `unsupported ${summary.unsupported}`,
156
+ `excluded ${summary.excluded}`,
157
+ `incomplete ${summary.incomplete}`,
158
+ ].join('; ');
159
+ }
160
+
161
+ function formatCost(result: ExperimentRunResult): string {
162
+ const allLocal = result.manifest.targets.every(
163
+ (target) => target.deployment?.cost_basis === 'not_applicable'
164
+ );
165
+ if (allLocal) return 'not applicable (local targets only)';
166
+ const cost = result.summaries.operational.cost;
167
+ return cost ? `${cost.amount} ${cost.currency}` : 'not reported';
168
+ }
169
+
170
+ function completeness(value: boolean): string {
171
+ return value ? 'complete' : 'incomplete';
172
+ }
@@ -0,0 +1,426 @@
1
+ /**
2
+ * Offline validation of an experiment file: manifest contract, task source identities and the
3
+ * host-side locality check for targets that declare no metered cost. Makes no model calls.
4
+ */
5
+
6
+ import { readFile, realpath, stat } from 'node:fs/promises';
7
+ import path from 'node:path';
8
+ import {
9
+ type ExperimentManifest,
10
+ type ExperimentMode,
11
+ type ExperimentTarget,
12
+ type ExperimentTask,
13
+ GENERATION_SETTING_KEYS,
14
+ type GenerationSettingKey,
15
+ isGenerationSettingKey,
16
+ isLoopbackEndpoint,
17
+ parseAgentWorkflow,
18
+ parseExperimentManifest,
19
+ readGenerationSettings,
20
+ validateScenario,
21
+ } from '@artemiskit/core';
22
+ import { parse as parseYaml } from 'yaml';
23
+ import { MAX_EXPERIMENT_FILE_BYTES, readBoundedText, sha256Hex } from './experiment-files';
24
+
25
+ export type ExperimentCheckSubject = 'file' | 'manifest' | 'task' | 'target';
26
+ export type ExperimentCheckStatus = 'ok' | 'warning' | 'error';
27
+
28
+ /** One validation finding. `code` is stable and machine-readable. */
29
+ export interface ExperimentCheck {
30
+ subject: ExperimentCheckSubject;
31
+ /** Task or target ID for task and target checks. */
32
+ id?: string;
33
+ status: ExperimentCheckStatus;
34
+ code: string;
35
+ message: string;
36
+ }
37
+
38
+ export interface ExperimentValidation {
39
+ /** True when no check has status `error`. Warnings do not block a run. */
40
+ valid: boolean;
41
+ /** Absolute path of the experiment file. */
42
+ file: string;
43
+ /** Directory that task source paths resolve under (the experiment file's directory). */
44
+ root: string;
45
+ experimentId?: string;
46
+ mode?: ExperimentMode;
47
+ /** Declared concurrency limit (coordinates in flight), when the manifest is valid. */
48
+ concurrency?: number;
49
+ /** Parsed manifest, present when the manifest contract itself is valid. */
50
+ manifest?: ExperimentManifest;
51
+ checks: ExperimentCheck[];
52
+ /** CLI exit code for `akit experiment validate`: 0 valid, 1 unreadable file, 2 invalid. */
53
+ exitCode: 0 | 1 | 2;
54
+ }
55
+
56
+ export const LOOPBACK_REQUIRED_MESSAGE =
57
+ 'cost not applicable is only accepted for loopback endpoints';
58
+
59
+ /**
60
+ * Validate an experiment file without contacting any model or network endpoint.
61
+ *
62
+ * Checks the manifest contract; that each task source exists under the experiment directory and
63
+ * its exact bytes match the declared SHA-256; that scenario and workflow sources parse; that
64
+ * workflow tasks can run as declared (warnings otherwise); that every target
65
+ * declaring `cost_basis: not_applicable` names a loopback `settings.base_url`; and that target
66
+ * generation settings are in range and sent by the target's provider adapter.
67
+ */
68
+ export async function validateExperimentFile(
69
+ experimentFile: string
70
+ ): Promise<ExperimentValidation> {
71
+ const file = path.resolve(experimentFile);
72
+ const root = path.dirname(file);
73
+ const checks: ExperimentCheck[] = [];
74
+ const finish = (manifest?: ExperimentManifest): ExperimentValidation => {
75
+ const unreadable = checks.some((check) => check.code === 'file_unreadable');
76
+ const valid = checks.every((check) => check.status !== 'error');
77
+ return {
78
+ valid,
79
+ file,
80
+ root,
81
+ experimentId: manifest?.id,
82
+ mode: manifest?.mode,
83
+ concurrency: manifest?.concurrency,
84
+ manifest,
85
+ checks,
86
+ exitCode: valid ? 0 : unreadable ? 1 : 2,
87
+ };
88
+ };
89
+
90
+ let text: string;
91
+ try {
92
+ text = await readBoundedText(file, MAX_EXPERIMENT_FILE_BYTES);
93
+ } catch {
94
+ checks.push(
95
+ fileError('file_unreadable', 'Experiment file is missing, unreadable or too large')
96
+ );
97
+ return finish();
98
+ }
99
+
100
+ let raw: unknown;
101
+ try {
102
+ // YAML is a superset of JSON. No `${VAR}` expansion: the manifest is taken as written.
103
+ raw = parseYaml(text);
104
+ } catch {
105
+ checks.push(fileError('file_unparseable', 'Experiment file is not valid YAML or JSON'));
106
+ return finish();
107
+ }
108
+
109
+ let manifest: ExperimentManifest;
110
+ try {
111
+ manifest = parseExperimentManifest(raw);
112
+ } catch (error) {
113
+ for (const message of manifestIssues(error)) {
114
+ checks.push({ subject: 'manifest', status: 'error', code: 'manifest_invalid', message });
115
+ }
116
+ return finish();
117
+ }
118
+ checks.push({
119
+ subject: 'manifest',
120
+ status: 'ok',
121
+ code: 'manifest_valid',
122
+ message: `${manifest.mode} experiment: ${manifest.tasks.length} task(s) × ${manifest.targets.length} target(s) × ${manifest.repetitions} repetition(s)`,
123
+ });
124
+
125
+ for (const task of manifest.tasks) checks.push(...(await checkTask(root, task)));
126
+ for (const target of manifest.targets) {
127
+ checks.push(checkTarget(target), ...checkTargetSettings(target));
128
+ }
129
+ return finish(manifest);
130
+ }
131
+
132
+ async function checkTask(root: string, task: ExperimentTask): Promise<ExperimentCheck[]> {
133
+ const fail = (code: string, message: string): ExperimentCheck => ({
134
+ subject: 'task',
135
+ id: task.id,
136
+ status: 'error',
137
+ code,
138
+ message,
139
+ });
140
+ const resolved = path.resolve(root, task.source.path);
141
+
142
+ let bytes: Uint8Array;
143
+ try {
144
+ const realRoot = await realpath(root);
145
+ const realFile = await realpath(resolved);
146
+ if (!isInside(realRoot, realFile)) {
147
+ return [fail('source_outside_root', `${task.source.path} resolves outside ${root}`)];
148
+ }
149
+ if (!(await stat(realFile)).isFile()) {
150
+ return [fail('source_not_file', `${task.source.path} is not a regular file`)];
151
+ }
152
+ bytes = await readFile(realFile);
153
+ } catch (error) {
154
+ return [
155
+ (error as NodeJS.ErrnoException).code === 'ENOENT'
156
+ ? fail('source_missing', `${task.source.path} does not exist under ${root}`)
157
+ : fail('source_unreadable', `${task.source.path} is unreadable`),
158
+ ];
159
+ }
160
+
161
+ if (sha256Hex(bytes) !== task.source.artifact.digest) {
162
+ return [
163
+ fail(
164
+ 'source_digest_mismatch',
165
+ `${task.source.path}: SHA-256 of the file bytes does not match the declared digest`
166
+ ),
167
+ ];
168
+ }
169
+ if (task.kind === 'agent_workflow') return checkWorkflowSource(task, bytes);
170
+ try {
171
+ const source = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
172
+ validateScenario(parseYaml(source));
173
+ } catch {
174
+ return [
175
+ fail('scenario_invalid', `${task.source.path}: digest verified but scenario is invalid`),
176
+ ];
177
+ }
178
+ return [
179
+ {
180
+ subject: 'task',
181
+ id: task.id,
182
+ status: 'ok',
183
+ code: 'source_verified',
184
+ message: `${task.source.path}: digest verified; scenario parsed`,
185
+ },
186
+ ];
187
+ }
188
+
189
+ /**
190
+ * Parse a verified workflow source exactly as the executor does: no `${VAR}` expansion and no
191
+ * fixture dereferencing. Semantic outcomes need an independent judge that experiment runs do
192
+ * not configure, so they are reported as a warning.
193
+ */
194
+ function checkWorkflowSource(task: ExperimentTask, bytes: Uint8Array): ExperimentCheck[] {
195
+ const check = (
196
+ status: ExperimentCheckStatus,
197
+ code: string,
198
+ message: string
199
+ ): ExperimentCheck => ({ subject: 'task', id: task.id, status, code, message });
200
+ let semanticOutcomes: number;
201
+ let environment: string;
202
+ try {
203
+ const workflow = parseAgentWorkflow(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
204
+ semanticOutcomes = workflow.outcomes.semantic?.length ?? 0;
205
+ environment = workflow.environment.type;
206
+ } catch {
207
+ return [
208
+ check(
209
+ 'error',
210
+ 'workflow_invalid',
211
+ `${task.source.path}: digest verified but agent workflow is invalid`
212
+ ),
213
+ ];
214
+ }
215
+ const checks = [
216
+ check(
217
+ 'ok',
218
+ 'source_verified',
219
+ `${task.source.path}: digest verified; agent workflow parsed (${environment} environment, network denied)`
220
+ ),
221
+ ];
222
+ if (semanticOutcomes > 0) {
223
+ checks.push(
224
+ check(
225
+ 'warning',
226
+ 'semantic_judge_unavailable',
227
+ `${task.source.path}: declares semantic outcomes, which need an independent judge that experiment runs do not configure; its coordinates will be recorded invalid`
228
+ )
229
+ );
230
+ }
231
+ return checks;
232
+ }
233
+
234
+ /** Values accepted for a target's `settings.reasoning_effort`. */
235
+ export const REASONING_EFFORTS = ['none', 'minimal', 'low', 'medium', 'high'] as const;
236
+
237
+ export function isReasoningEffort(value: unknown): value is (typeof REASONING_EFFORTS)[number] {
238
+ return typeof value === 'string' && (REASONING_EFFORTS as readonly string[]).includes(value);
239
+ }
240
+
241
+ function checkTarget(target: ExperimentTarget): ExperimentCheck {
242
+ const effort = target.settings?.reasoning_effort;
243
+ if (effort !== undefined && !isReasoningEffort(effort)) {
244
+ return {
245
+ subject: 'target',
246
+ id: target.id,
247
+ status: 'error',
248
+ code: 'invalid_reasoning_effort',
249
+ message: `settings.reasoning_effort must be one of ${REASONING_EFFORTS.join(', ')}`,
250
+ };
251
+ }
252
+ if (target.deployment?.cost_basis !== 'not_applicable') {
253
+ return {
254
+ subject: 'target',
255
+ id: target.id,
256
+ status: 'ok',
257
+ code: 'target_declared',
258
+ message: `${target.provider} ${target.model}`,
259
+ };
260
+ }
261
+ const baseUrl = target.settings?.base_url;
262
+ if (typeof baseUrl !== 'string' || !isLoopbackEndpoint(baseUrl)) {
263
+ // The URL itself is not echoed: a rejected URL may carry userinfo credentials.
264
+ return {
265
+ subject: 'target',
266
+ id: target.id,
267
+ status: 'error',
268
+ code: 'cost_not_applicable_requires_loopback',
269
+ message: `${LOOPBACK_REQUIRED_MESSAGE}; settings.base_url is ${
270
+ typeof baseUrl === 'string' ? 'not a loopback http(s) URL' : 'missing'
271
+ }`,
272
+ };
273
+ }
274
+ return {
275
+ subject: 'target',
276
+ id: target.id,
277
+ status: 'ok',
278
+ code: 'loopback_verified',
279
+ message: `${target.provider} ${target.model} on a loopback endpoint; cost not applicable`,
280
+ };
281
+ }
282
+
283
+ /** Target settings keys the built-in executor and adapters read, besides generation keys. */
284
+ const TRANSPORT_SETTING_KEYS = new Set(['base_url', 'reasoning_effort']);
285
+
286
+ /**
287
+ * Providers whose built-in adapter sends each generation setting. `temperature` and
288
+ * `max_output_tokens` replace the scenario's own values, which every adapter receives. No
289
+ * adapter sends `top_k` or `min_p`: the OpenAI chat completions request has no such field, and
290
+ * Ollama's /v1/chat/completions decodes neither (https://docs.ollama.com/api/openai-compatibility).
291
+ */
292
+ const GENERATION_SETTING_PROVIDERS: Readonly<
293
+ Record<GenerationSettingKey, readonly string[] | 'all'>
294
+ > = {
295
+ temperature: 'all',
296
+ max_output_tokens: 'all',
297
+ top_p: ['openai', 'azure-openai', 'anthropic', 'vercel-ai'],
298
+ presence_penalty: ['openai', 'azure-openai'],
299
+ top_k: [],
300
+ min_p: [],
301
+ };
302
+
303
+ /**
304
+ * Common spellings of generation settings and the declared key they mean. Setting names are
305
+ * lowercase identifiers, so camelCase spellings are already refused by the manifest parser.
306
+ */
307
+ const GENERATION_SETTING_ALIASES: Readonly<Record<string, GenerationSettingKey>> = {
308
+ max_tokens: 'max_output_tokens',
309
+ maxtokens: 'max_output_tokens',
310
+ max_completion_tokens: 'max_output_tokens',
311
+ maxoutputtokens: 'max_output_tokens',
312
+ num_predict: 'max_output_tokens',
313
+ topp: 'top_p',
314
+ topk: 'top_k',
315
+ minp: 'min_p',
316
+ presencepenalty: 'presence_penalty',
317
+ };
318
+
319
+ /** Names that look like sampling or output controls but are not supported target settings. */
320
+ const GENERATION_LIKE_KEY =
321
+ /temperature|top_?[pk]|min_?p|penalty|tokens|predict|seed|^stop|mirostat|typical|sampl|num_ctx/i;
322
+
323
+ /**
324
+ * Check target generation settings: values in range, sent by the target's provider adapter, and
325
+ * no unrecognised generation-like keys. Other unknown keys are reported as unused.
326
+ */
327
+ function checkTargetSettings(target: ExperimentTarget): ExperimentCheck[] {
328
+ const settings = target.settings ?? {};
329
+ const checks: ExperimentCheck[] = [];
330
+ const error = (code: string, message: string): ExperimentCheck => ({
331
+ subject: 'target',
332
+ id: target.id,
333
+ status: 'error',
334
+ code,
335
+ message,
336
+ });
337
+
338
+ for (const key of Object.keys(settings)) {
339
+ if (isGenerationSettingKey(key) || TRANSPORT_SETTING_KEYS.has(key)) continue;
340
+ const alias = GENERATION_SETTING_ALIASES[key];
341
+ if (alias) {
342
+ checks.push(
343
+ error('unknown_generation_setting', `settings.${key} is not supported; use ${alias}`)
344
+ );
345
+ } else if (key === 'seed') {
346
+ checks.push(
347
+ error(
348
+ 'unknown_generation_setting',
349
+ 'settings.seed is not supported; declare the experiment seed (seed.value and seed.strategy) instead'
350
+ )
351
+ );
352
+ } else if (GENERATION_LIKE_KEY.test(key)) {
353
+ checks.push(
354
+ error(
355
+ 'unknown_generation_setting',
356
+ `settings.${key} is not a supported generation setting; supported: ${GENERATION_SETTING_KEYS.join(', ')}`
357
+ )
358
+ );
359
+ } else {
360
+ checks.push({
361
+ subject: 'target',
362
+ id: target.id,
363
+ status: 'warning',
364
+ code: 'target_setting_unused',
365
+ message: `settings.${key} is recorded with the target but not sent to the model`,
366
+ });
367
+ }
368
+ }
369
+
370
+ const { settings: generation, issues } = readGenerationSettings(settings);
371
+ for (const issue of issues) checks.push(error('invalid_generation_setting', issue.message));
372
+ for (const key of GENERATION_SETTING_KEYS) {
373
+ if (generation[key] === undefined || providerSends(key, target.provider)) continue;
374
+ checks.push(error('generation_setting_unsupported', unsupportedMessage(key, target)));
375
+ }
376
+
377
+ const declared = GENERATION_SETTING_KEYS.filter((key) => generation[key] !== undefined);
378
+ if (declared.length > 0 && checks.every((check) => check.status !== 'error')) {
379
+ checks.push({
380
+ subject: 'target',
381
+ id: target.id,
382
+ status: 'ok',
383
+ code: 'generation_settings',
384
+ message: `${declared.map((key) => `${key} ${generation[key]}`).join(', ')} replace task values for this target`,
385
+ });
386
+ }
387
+ return checks;
388
+ }
389
+
390
+ function providerSends(key: GenerationSettingKey, provider: string): boolean {
391
+ const providers = GENERATION_SETTING_PROVIDERS[key];
392
+ return providers === 'all' || providers.includes(provider);
393
+ }
394
+
395
+ function unsupportedMessage(key: GenerationSettingKey, target: ExperimentTarget): string {
396
+ if (key !== 'top_k' && key !== 'min_p') {
397
+ return `settings.${key} is not sent by the ${target.provider} adapter; remove it`;
398
+ }
399
+ const ollama = target.deployment?.runtime === 'ollama' || target.provider === 'openai';
400
+ return ollama
401
+ ? `settings.${key} cannot be sent: OpenAI-compatible chat completions (including Ollama's /v1 endpoint) have no ${key} field, so the model's server default would apply silently. Remove it and record the model's default (for Ollama, its Modelfile PARAMETER ${key}) instead`
402
+ : `settings.${key} cannot be sent: no built-in adapter carries ${key}, so the model's default would apply silently. Remove it`;
403
+ }
404
+
405
+ function fileError(code: string, message: string): ExperimentCheck {
406
+ return { subject: 'file', status: 'error', code, message };
407
+ }
408
+
409
+ /** Bounded, path-qualified messages from a manifest parse failure. */
410
+ function manifestIssues(error: unknown): string[] {
411
+ const issues = (error as { issues?: { path: (string | number)[]; message: string }[] })?.issues;
412
+ if (!Array.isArray(issues) || issues.length === 0) return ['Experiment manifest is invalid'];
413
+ return issues
414
+ .slice(0, 20)
415
+ .map((issue) => `${issue.path.length > 0 ? issue.path.join('.') : '(root)'}: ${issue.message}`);
416
+ }
417
+
418
+ function isInside(parent: string, child: string): boolean {
419
+ const relative = path.relative(parent, child);
420
+ return (
421
+ relative !== '' &&
422
+ relative !== '..' &&
423
+ !relative.startsWith(`..${path.sep}`) &&
424
+ !path.isAbsolute(relative)
425
+ );
426
+ }