@artemiskit/sdk 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/__tests__/experiment-fixtures.d.ts +44 -0
- package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
- package/dist/builders/index.js +19 -66
- package/dist/contracts/index.js +3 -50
- package/dist/experiment-files.d.ts +54 -0
- package/dist/experiment-files.d.ts.map +1 -0
- package/dist/experiment-fixture.d.ts +20 -0
- package/dist/experiment-fixture.d.ts.map +1 -0
- package/dist/experiment-summary.d.ts +44 -0
- package/dist/experiment-summary.d.ts.map +1 -0
- package/dist/experiment-validation.d.ts +48 -0
- package/dist/experiment-validation.d.ts.map +1 -0
- package/dist/experiment.d.ts +162 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +6 -1
- package/dist/guardian/circuit-breaker.d.ts.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2557 -1863
- package/dist/matchers/index.js +13 -60
- package/dist/matchers/jest.js +0 -47
- package/dist/matchers/vitest.js +0 -47
- package/dist/utils/index.js +11 -58
- package/package.json +6 -5
- package/src/__tests__/artemiskit.test.ts +1 -2
- package/src/__tests__/builders.test.ts +0 -2
- package/src/__tests__/circuit-breaker.test.ts +1 -1
- package/src/__tests__/compare.test.ts +1 -1
- package/src/__tests__/contracts.test.ts +1 -1
- package/src/__tests__/experiment-case-level.test.ts +212 -0
- package/src/__tests__/experiment-fixtures.ts +194 -0
- package/src/__tests__/experiment.test.ts +1262 -0
- package/src/__tests__/guardian-modes.test.ts +2 -2
- package/src/__tests__/guardrails.test.ts +1 -1
- package/src/__tests__/matchers.test.ts +1 -1
- package/src/__tests__/semantic-validator.test.ts +15 -10
- package/src/__tests__/validate.test.ts +0 -4
- package/src/__tests__/workflow-recovery.test.ts +7 -3
- package/src/__tests__/workflow-report.test.ts +6 -3
- package/src/__tests__/workflow.test.ts +11 -5
- package/src/experiment-files.ts +190 -0
- package/src/experiment-fixture.ts +70 -0
- package/src/experiment-summary.ts +172 -0
- package/src/experiment-validation.ts +426 -0
- package/src/experiment.ts +630 -0
- package/src/guardian/circuit-breaker.ts +6 -1
- package/src/index.ts +52 -0
- package/tsconfig.typecheck.json +12 -0
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Offline validation of an experiment file: manifest contract, task source identities and the
|
|
3
|
+
* host-side locality check for targets that declare no metered cost. Makes no model calls.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { readFile, realpath, stat } from 'node:fs/promises';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
import {
|
|
9
|
+
type ExperimentManifest,
|
|
10
|
+
type ExperimentMode,
|
|
11
|
+
type ExperimentTarget,
|
|
12
|
+
type ExperimentTask,
|
|
13
|
+
GENERATION_SETTING_KEYS,
|
|
14
|
+
type GenerationSettingKey,
|
|
15
|
+
isGenerationSettingKey,
|
|
16
|
+
isLoopbackEndpoint,
|
|
17
|
+
parseAgentWorkflow,
|
|
18
|
+
parseExperimentManifest,
|
|
19
|
+
readGenerationSettings,
|
|
20
|
+
validateScenario,
|
|
21
|
+
} from '@artemiskit/core';
|
|
22
|
+
import { parse as parseYaml } from 'yaml';
|
|
23
|
+
import { MAX_EXPERIMENT_FILE_BYTES, readBoundedText, sha256Hex } from './experiment-files';
|
|
24
|
+
|
|
25
|
+
export type ExperimentCheckSubject = 'file' | 'manifest' | 'task' | 'target';
|
|
26
|
+
export type ExperimentCheckStatus = 'ok' | 'warning' | 'error';
|
|
27
|
+
|
|
28
|
+
/** One validation finding. `code` is stable and machine-readable. */
|
|
29
|
+
export interface ExperimentCheck {
|
|
30
|
+
subject: ExperimentCheckSubject;
|
|
31
|
+
/** Task or target ID for task and target checks. */
|
|
32
|
+
id?: string;
|
|
33
|
+
status: ExperimentCheckStatus;
|
|
34
|
+
code: string;
|
|
35
|
+
message: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface ExperimentValidation {
|
|
39
|
+
/** True when no check has status `error`. Warnings do not block a run. */
|
|
40
|
+
valid: boolean;
|
|
41
|
+
/** Absolute path of the experiment file. */
|
|
42
|
+
file: string;
|
|
43
|
+
/** Directory that task source paths resolve under (the experiment file's directory). */
|
|
44
|
+
root: string;
|
|
45
|
+
experimentId?: string;
|
|
46
|
+
mode?: ExperimentMode;
|
|
47
|
+
/** Declared concurrency limit (coordinates in flight), when the manifest is valid. */
|
|
48
|
+
concurrency?: number;
|
|
49
|
+
/** Parsed manifest, present when the manifest contract itself is valid. */
|
|
50
|
+
manifest?: ExperimentManifest;
|
|
51
|
+
checks: ExperimentCheck[];
|
|
52
|
+
/** CLI exit code for `akit experiment validate`: 0 valid, 1 unreadable file, 2 invalid. */
|
|
53
|
+
exitCode: 0 | 1 | 2;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export const LOOPBACK_REQUIRED_MESSAGE =
|
|
57
|
+
'cost not applicable is only accepted for loopback endpoints';
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Validate an experiment file without contacting any model or network endpoint.
|
|
61
|
+
*
|
|
62
|
+
* Checks the manifest contract; that each task source exists under the experiment directory and
|
|
63
|
+
* its exact bytes match the declared SHA-256; that scenario and workflow sources parse; that
|
|
64
|
+
* workflow tasks can run as declared (warnings otherwise); that every target
|
|
65
|
+
* declaring `cost_basis: not_applicable` names a loopback `settings.base_url`; and that target
|
|
66
|
+
* generation settings are in range and sent by the target's provider adapter.
|
|
67
|
+
*/
|
|
68
|
+
export async function validateExperimentFile(
|
|
69
|
+
experimentFile: string
|
|
70
|
+
): Promise<ExperimentValidation> {
|
|
71
|
+
const file = path.resolve(experimentFile);
|
|
72
|
+
const root = path.dirname(file);
|
|
73
|
+
const checks: ExperimentCheck[] = [];
|
|
74
|
+
const finish = (manifest?: ExperimentManifest): ExperimentValidation => {
|
|
75
|
+
const unreadable = checks.some((check) => check.code === 'file_unreadable');
|
|
76
|
+
const valid = checks.every((check) => check.status !== 'error');
|
|
77
|
+
return {
|
|
78
|
+
valid,
|
|
79
|
+
file,
|
|
80
|
+
root,
|
|
81
|
+
experimentId: manifest?.id,
|
|
82
|
+
mode: manifest?.mode,
|
|
83
|
+
concurrency: manifest?.concurrency,
|
|
84
|
+
manifest,
|
|
85
|
+
checks,
|
|
86
|
+
exitCode: valid ? 0 : unreadable ? 1 : 2,
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
let text: string;
|
|
91
|
+
try {
|
|
92
|
+
text = await readBoundedText(file, MAX_EXPERIMENT_FILE_BYTES);
|
|
93
|
+
} catch {
|
|
94
|
+
checks.push(
|
|
95
|
+
fileError('file_unreadable', 'Experiment file is missing, unreadable or too large')
|
|
96
|
+
);
|
|
97
|
+
return finish();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
let raw: unknown;
|
|
101
|
+
try {
|
|
102
|
+
// YAML is a superset of JSON. No `${VAR}` expansion: the manifest is taken as written.
|
|
103
|
+
raw = parseYaml(text);
|
|
104
|
+
} catch {
|
|
105
|
+
checks.push(fileError('file_unparseable', 'Experiment file is not valid YAML or JSON'));
|
|
106
|
+
return finish();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
let manifest: ExperimentManifest;
|
|
110
|
+
try {
|
|
111
|
+
manifest = parseExperimentManifest(raw);
|
|
112
|
+
} catch (error) {
|
|
113
|
+
for (const message of manifestIssues(error)) {
|
|
114
|
+
checks.push({ subject: 'manifest', status: 'error', code: 'manifest_invalid', message });
|
|
115
|
+
}
|
|
116
|
+
return finish();
|
|
117
|
+
}
|
|
118
|
+
checks.push({
|
|
119
|
+
subject: 'manifest',
|
|
120
|
+
status: 'ok',
|
|
121
|
+
code: 'manifest_valid',
|
|
122
|
+
message: `${manifest.mode} experiment: ${manifest.tasks.length} task(s) × ${manifest.targets.length} target(s) × ${manifest.repetitions} repetition(s)`,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
for (const task of manifest.tasks) checks.push(...(await checkTask(root, task)));
|
|
126
|
+
for (const target of manifest.targets) {
|
|
127
|
+
checks.push(checkTarget(target), ...checkTargetSettings(target));
|
|
128
|
+
}
|
|
129
|
+
return finish(manifest);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async function checkTask(root: string, task: ExperimentTask): Promise<ExperimentCheck[]> {
|
|
133
|
+
const fail = (code: string, message: string): ExperimentCheck => ({
|
|
134
|
+
subject: 'task',
|
|
135
|
+
id: task.id,
|
|
136
|
+
status: 'error',
|
|
137
|
+
code,
|
|
138
|
+
message,
|
|
139
|
+
});
|
|
140
|
+
const resolved = path.resolve(root, task.source.path);
|
|
141
|
+
|
|
142
|
+
let bytes: Uint8Array;
|
|
143
|
+
try {
|
|
144
|
+
const realRoot = await realpath(root);
|
|
145
|
+
const realFile = await realpath(resolved);
|
|
146
|
+
if (!isInside(realRoot, realFile)) {
|
|
147
|
+
return [fail('source_outside_root', `${task.source.path} resolves outside ${root}`)];
|
|
148
|
+
}
|
|
149
|
+
if (!(await stat(realFile)).isFile()) {
|
|
150
|
+
return [fail('source_not_file', `${task.source.path} is not a regular file`)];
|
|
151
|
+
}
|
|
152
|
+
bytes = await readFile(realFile);
|
|
153
|
+
} catch (error) {
|
|
154
|
+
return [
|
|
155
|
+
(error as NodeJS.ErrnoException).code === 'ENOENT'
|
|
156
|
+
? fail('source_missing', `${task.source.path} does not exist under ${root}`)
|
|
157
|
+
: fail('source_unreadable', `${task.source.path} is unreadable`),
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (sha256Hex(bytes) !== task.source.artifact.digest) {
|
|
162
|
+
return [
|
|
163
|
+
fail(
|
|
164
|
+
'source_digest_mismatch',
|
|
165
|
+
`${task.source.path}: SHA-256 of the file bytes does not match the declared digest`
|
|
166
|
+
),
|
|
167
|
+
];
|
|
168
|
+
}
|
|
169
|
+
if (task.kind === 'agent_workflow') return checkWorkflowSource(task, bytes);
|
|
170
|
+
try {
|
|
171
|
+
const source = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
172
|
+
validateScenario(parseYaml(source));
|
|
173
|
+
} catch {
|
|
174
|
+
return [
|
|
175
|
+
fail('scenario_invalid', `${task.source.path}: digest verified but scenario is invalid`),
|
|
176
|
+
];
|
|
177
|
+
}
|
|
178
|
+
return [
|
|
179
|
+
{
|
|
180
|
+
subject: 'task',
|
|
181
|
+
id: task.id,
|
|
182
|
+
status: 'ok',
|
|
183
|
+
code: 'source_verified',
|
|
184
|
+
message: `${task.source.path}: digest verified; scenario parsed`,
|
|
185
|
+
},
|
|
186
|
+
];
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Parse a verified workflow source exactly as the executor does: no `${VAR}` expansion and no
|
|
191
|
+
* fixture dereferencing. Semantic outcomes need an independent judge that experiment runs do
|
|
192
|
+
* not configure, so they are reported as a warning.
|
|
193
|
+
*/
|
|
194
|
+
function checkWorkflowSource(task: ExperimentTask, bytes: Uint8Array): ExperimentCheck[] {
|
|
195
|
+
const check = (
|
|
196
|
+
status: ExperimentCheckStatus,
|
|
197
|
+
code: string,
|
|
198
|
+
message: string
|
|
199
|
+
): ExperimentCheck => ({ subject: 'task', id: task.id, status, code, message });
|
|
200
|
+
let semanticOutcomes: number;
|
|
201
|
+
let environment: string;
|
|
202
|
+
try {
|
|
203
|
+
const workflow = parseAgentWorkflow(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
|
|
204
|
+
semanticOutcomes = workflow.outcomes.semantic?.length ?? 0;
|
|
205
|
+
environment = workflow.environment.type;
|
|
206
|
+
} catch {
|
|
207
|
+
return [
|
|
208
|
+
check(
|
|
209
|
+
'error',
|
|
210
|
+
'workflow_invalid',
|
|
211
|
+
`${task.source.path}: digest verified but agent workflow is invalid`
|
|
212
|
+
),
|
|
213
|
+
];
|
|
214
|
+
}
|
|
215
|
+
const checks = [
|
|
216
|
+
check(
|
|
217
|
+
'ok',
|
|
218
|
+
'source_verified',
|
|
219
|
+
`${task.source.path}: digest verified; agent workflow parsed (${environment} environment, network denied)`
|
|
220
|
+
),
|
|
221
|
+
];
|
|
222
|
+
if (semanticOutcomes > 0) {
|
|
223
|
+
checks.push(
|
|
224
|
+
check(
|
|
225
|
+
'warning',
|
|
226
|
+
'semantic_judge_unavailable',
|
|
227
|
+
`${task.source.path}: declares semantic outcomes, which need an independent judge that experiment runs do not configure; its coordinates will be recorded invalid`
|
|
228
|
+
)
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
return checks;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Values accepted for a target's `settings.reasoning_effort`. */
|
|
235
|
+
export const REASONING_EFFORTS = ['none', 'minimal', 'low', 'medium', 'high'] as const;
|
|
236
|
+
|
|
237
|
+
export function isReasoningEffort(value: unknown): value is (typeof REASONING_EFFORTS)[number] {
|
|
238
|
+
return typeof value === 'string' && (REASONING_EFFORTS as readonly string[]).includes(value);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function checkTarget(target: ExperimentTarget): ExperimentCheck {
|
|
242
|
+
const effort = target.settings?.reasoning_effort;
|
|
243
|
+
if (effort !== undefined && !isReasoningEffort(effort)) {
|
|
244
|
+
return {
|
|
245
|
+
subject: 'target',
|
|
246
|
+
id: target.id,
|
|
247
|
+
status: 'error',
|
|
248
|
+
code: 'invalid_reasoning_effort',
|
|
249
|
+
message: `settings.reasoning_effort must be one of ${REASONING_EFFORTS.join(', ')}`,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
if (target.deployment?.cost_basis !== 'not_applicable') {
|
|
253
|
+
return {
|
|
254
|
+
subject: 'target',
|
|
255
|
+
id: target.id,
|
|
256
|
+
status: 'ok',
|
|
257
|
+
code: 'target_declared',
|
|
258
|
+
message: `${target.provider} ${target.model}`,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
const baseUrl = target.settings?.base_url;
|
|
262
|
+
if (typeof baseUrl !== 'string' || !isLoopbackEndpoint(baseUrl)) {
|
|
263
|
+
// The URL itself is not echoed: a rejected URL may carry userinfo credentials.
|
|
264
|
+
return {
|
|
265
|
+
subject: 'target',
|
|
266
|
+
id: target.id,
|
|
267
|
+
status: 'error',
|
|
268
|
+
code: 'cost_not_applicable_requires_loopback',
|
|
269
|
+
message: `${LOOPBACK_REQUIRED_MESSAGE}; settings.base_url is ${
|
|
270
|
+
typeof baseUrl === 'string' ? 'not a loopback http(s) URL' : 'missing'
|
|
271
|
+
}`,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
return {
|
|
275
|
+
subject: 'target',
|
|
276
|
+
id: target.id,
|
|
277
|
+
status: 'ok',
|
|
278
|
+
code: 'loopback_verified',
|
|
279
|
+
message: `${target.provider} ${target.model} on a loopback endpoint; cost not applicable`,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Target settings keys the built-in executor and adapters read, besides generation keys. */
|
|
284
|
+
const TRANSPORT_SETTING_KEYS = new Set(['base_url', 'reasoning_effort']);
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Providers whose built-in adapter sends each generation setting. `temperature` and
|
|
288
|
+
* `max_output_tokens` replace the scenario's own values, which every adapter receives. No
|
|
289
|
+
* adapter sends `top_k` or `min_p`: the OpenAI chat completions request has no such field, and
|
|
290
|
+
* Ollama's /v1/chat/completions decodes neither (https://docs.ollama.com/api/openai-compatibility).
|
|
291
|
+
*/
|
|
292
|
+
const GENERATION_SETTING_PROVIDERS: Readonly<
|
|
293
|
+
Record<GenerationSettingKey, readonly string[] | 'all'>
|
|
294
|
+
> = {
|
|
295
|
+
temperature: 'all',
|
|
296
|
+
max_output_tokens: 'all',
|
|
297
|
+
top_p: ['openai', 'azure-openai', 'anthropic', 'vercel-ai'],
|
|
298
|
+
presence_penalty: ['openai', 'azure-openai'],
|
|
299
|
+
top_k: [],
|
|
300
|
+
min_p: [],
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Common spellings of generation settings and the declared key they mean. Setting names are
|
|
305
|
+
* lowercase identifiers, so camelCase spellings are already refused by the manifest parser.
|
|
306
|
+
*/
|
|
307
|
+
const GENERATION_SETTING_ALIASES: Readonly<Record<string, GenerationSettingKey>> = {
|
|
308
|
+
max_tokens: 'max_output_tokens',
|
|
309
|
+
maxtokens: 'max_output_tokens',
|
|
310
|
+
max_completion_tokens: 'max_output_tokens',
|
|
311
|
+
maxoutputtokens: 'max_output_tokens',
|
|
312
|
+
num_predict: 'max_output_tokens',
|
|
313
|
+
topp: 'top_p',
|
|
314
|
+
topk: 'top_k',
|
|
315
|
+
minp: 'min_p',
|
|
316
|
+
presencepenalty: 'presence_penalty',
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
/** Names that look like sampling or output controls but are not supported target settings. */
|
|
320
|
+
const GENERATION_LIKE_KEY =
|
|
321
|
+
/temperature|top_?[pk]|min_?p|penalty|tokens|predict|seed|^stop|mirostat|typical|sampl|num_ctx/i;
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Check target generation settings: values in range, sent by the target's provider adapter, and
|
|
325
|
+
* no unrecognised generation-like keys. Other unknown keys are reported as unused.
|
|
326
|
+
*/
|
|
327
|
+
function checkTargetSettings(target: ExperimentTarget): ExperimentCheck[] {
|
|
328
|
+
const settings = target.settings ?? {};
|
|
329
|
+
const checks: ExperimentCheck[] = [];
|
|
330
|
+
const error = (code: string, message: string): ExperimentCheck => ({
|
|
331
|
+
subject: 'target',
|
|
332
|
+
id: target.id,
|
|
333
|
+
status: 'error',
|
|
334
|
+
code,
|
|
335
|
+
message,
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
for (const key of Object.keys(settings)) {
|
|
339
|
+
if (isGenerationSettingKey(key) || TRANSPORT_SETTING_KEYS.has(key)) continue;
|
|
340
|
+
const alias = GENERATION_SETTING_ALIASES[key];
|
|
341
|
+
if (alias) {
|
|
342
|
+
checks.push(
|
|
343
|
+
error('unknown_generation_setting', `settings.${key} is not supported; use ${alias}`)
|
|
344
|
+
);
|
|
345
|
+
} else if (key === 'seed') {
|
|
346
|
+
checks.push(
|
|
347
|
+
error(
|
|
348
|
+
'unknown_generation_setting',
|
|
349
|
+
'settings.seed is not supported; declare the experiment seed (seed.value and seed.strategy) instead'
|
|
350
|
+
)
|
|
351
|
+
);
|
|
352
|
+
} else if (GENERATION_LIKE_KEY.test(key)) {
|
|
353
|
+
checks.push(
|
|
354
|
+
error(
|
|
355
|
+
'unknown_generation_setting',
|
|
356
|
+
`settings.${key} is not a supported generation setting; supported: ${GENERATION_SETTING_KEYS.join(', ')}`
|
|
357
|
+
)
|
|
358
|
+
);
|
|
359
|
+
} else {
|
|
360
|
+
checks.push({
|
|
361
|
+
subject: 'target',
|
|
362
|
+
id: target.id,
|
|
363
|
+
status: 'warning',
|
|
364
|
+
code: 'target_setting_unused',
|
|
365
|
+
message: `settings.${key} is recorded with the target but not sent to the model`,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const { settings: generation, issues } = readGenerationSettings(settings);
|
|
371
|
+
for (const issue of issues) checks.push(error('invalid_generation_setting', issue.message));
|
|
372
|
+
for (const key of GENERATION_SETTING_KEYS) {
|
|
373
|
+
if (generation[key] === undefined || providerSends(key, target.provider)) continue;
|
|
374
|
+
checks.push(error('generation_setting_unsupported', unsupportedMessage(key, target)));
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const declared = GENERATION_SETTING_KEYS.filter((key) => generation[key] !== undefined);
|
|
378
|
+
if (declared.length > 0 && checks.every((check) => check.status !== 'error')) {
|
|
379
|
+
checks.push({
|
|
380
|
+
subject: 'target',
|
|
381
|
+
id: target.id,
|
|
382
|
+
status: 'ok',
|
|
383
|
+
code: 'generation_settings',
|
|
384
|
+
message: `${declared.map((key) => `${key} ${generation[key]}`).join(', ')} replace task values for this target`,
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
return checks;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function providerSends(key: GenerationSettingKey, provider: string): boolean {
|
|
391
|
+
const providers = GENERATION_SETTING_PROVIDERS[key];
|
|
392
|
+
return providers === 'all' || providers.includes(provider);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function unsupportedMessage(key: GenerationSettingKey, target: ExperimentTarget): string {
|
|
396
|
+
if (key !== 'top_k' && key !== 'min_p') {
|
|
397
|
+
return `settings.${key} is not sent by the ${target.provider} adapter; remove it`;
|
|
398
|
+
}
|
|
399
|
+
const ollama = target.deployment?.runtime === 'ollama' || target.provider === 'openai';
|
|
400
|
+
return ollama
|
|
401
|
+
? `settings.${key} cannot be sent: OpenAI-compatible chat completions (including Ollama's /v1 endpoint) have no ${key} field, so the model's server default would apply silently. Remove it and record the model's default (for Ollama, its Modelfile PARAMETER ${key}) instead`
|
|
402
|
+
: `settings.${key} cannot be sent: no built-in adapter carries ${key}, so the model's default would apply silently. Remove it`;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function fileError(code: string, message: string): ExperimentCheck {
|
|
406
|
+
return { subject: 'file', status: 'error', code, message };
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/** Bounded, path-qualified messages from a manifest parse failure. */
|
|
410
|
+
function manifestIssues(error: unknown): string[] {
|
|
411
|
+
const issues = (error as { issues?: { path: (string | number)[]; message: string }[] })?.issues;
|
|
412
|
+
if (!Array.isArray(issues) || issues.length === 0) return ['Experiment manifest is invalid'];
|
|
413
|
+
return issues
|
|
414
|
+
.slice(0, 20)
|
|
415
|
+
.map((issue) => `${issue.path.length > 0 ? issue.path.join('.') : '(root)'}: ${issue.message}`);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function isInside(parent: string, child: string): boolean {
|
|
419
|
+
const relative = path.relative(parent, child);
|
|
420
|
+
return (
|
|
421
|
+
relative !== '' &&
|
|
422
|
+
relative !== '..' &&
|
|
423
|
+
!relative.startsWith(`..${path.sep}`) &&
|
|
424
|
+
!path.isAbsolute(relative)
|
|
425
|
+
);
|
|
426
|
+
}
|