@artemiskit/sdk 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +17 -0
- package/dist/__tests__/experiment-fixtures.d.ts +44 -0
- package/dist/__tests__/experiment-fixtures.d.ts.map +1 -0
- package/dist/artemiskit.d.ts +6 -1
- package/dist/artemiskit.d.ts.map +1 -1
- package/dist/builders/index.js +19 -66
- package/dist/contracts/index.js +3 -50
- package/dist/experiment-files.d.ts +54 -0
- package/dist/experiment-files.d.ts.map +1 -0
- package/dist/experiment-fixture.d.ts +20 -0
- package/dist/experiment-fixture.d.ts.map +1 -0
- package/dist/experiment-summary.d.ts +44 -0
- package/dist/experiment-summary.d.ts.map +1 -0
- package/dist/experiment-validation.d.ts +48 -0
- package/dist/experiment-validation.d.ts.map +1 -0
- package/dist/experiment.d.ts +162 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +6 -1
- package/dist/guardian/circuit-breaker.d.ts.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2516 -1801
- package/dist/matchers/index.js +13 -60
- package/dist/matchers/jest.js +0 -47
- package/dist/matchers/vitest.js +0 -47
- package/dist/types-only.d.ts +1 -0
- package/dist/types-only.d.ts.map +1 -1
- package/dist/utils/index.js +11 -58
- package/package.json +6 -4
- package/src/__tests__/artemiskit.test.ts +1 -2
- package/src/__tests__/builders.test.ts +0 -2
- package/src/__tests__/circuit-breaker.test.ts +1 -1
- package/src/__tests__/compare.test.ts +1 -1
- package/src/__tests__/contracts.test.ts +1 -1
- package/src/__tests__/experiment-case-level.test.ts +212 -0
- package/src/__tests__/experiment-fixtures.ts +194 -0
- package/src/__tests__/experiment.test.ts +1262 -0
- package/src/__tests__/guardian-modes.test.ts +2 -2
- package/src/__tests__/guardrails.test.ts +1 -1
- package/src/__tests__/matchers.test.ts +1 -1
- package/src/__tests__/semantic-validator.test.ts +15 -10
- package/src/__tests__/validate.test.ts +0 -4
- package/src/__tests__/workflow-recovery.test.ts +7 -3
- package/src/__tests__/workflow-report.test.ts +112 -0
- package/src/__tests__/workflow.test.ts +11 -5
- package/src/artemiskit.ts +16 -1
- package/src/experiment-files.ts +190 -0
- package/src/experiment-fixture.ts +70 -0
- package/src/experiment-summary.ts +172 -0
- package/src/experiment-validation.ts +426 -0
- package/src/experiment.ts +630 -0
- package/src/guardian/circuit-breaker.ts +6 -1
- package/src/index.ts +71 -0
- package/src/types-only.ts +11 -0
- package/tsconfig.typecheck.json +12 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.7.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 4882eff: Experiment reports now show a case-level summary beside the coordinate-level summary. A scenario coordinate (task × target × repetition) passes only when all of its cases pass, so coordinate counts alone understate how many individual cases passed. The case level counts planned, recorded, valid, passed, failed, invalid and unmeasured cases overall and per target, task, language and policy, read from each coordinate's terminal attempt evidence. Each saved evidence file must hash to the SHA-256 recorded in the result, parse as a run manifest and agree with the saved coordinate status; otherwise the whole case level is withheld with its reason instead of showing partial counts. Agent-workflow coordinates have no scenario cases and are counted separately. Results saved without an evidence directory still report, with case level marked unavailable.
|
|
8
|
+
|
|
9
|
+
- `@artemiskit/reports`: `createExperimentReport` and `generateExperimentReport` accept `caseEvidence` (evidence bytes keyed by recorded digest) and add `caseLevel` to the report; new `createExperimentCaseSummary` and `listExperimentCaseEvidenceDigests`. Technical, executive and comprehensive views (HTML and Markdown) label both levels and make no ranking claim; technical and comprehensive views add a per-task, language and policy table.
|
|
10
|
+
- `@artemiskit/sdk`: new `summarizeExperimentCases` and `formatExperimentCaseSummary`; `reportExperimentResult` reads `evidence/` beside a result file or an explicit `evidenceDir`; `runExperimentFile` returns `caseLevel`; `formatExperimentRunSummary` accepts `caseLevel`.
|
|
11
|
+
- `@artemiskit/cli`: `akit experiment run` and `akit experiment report` print the case level after the coordinate counts; `report` gains `--evidence-dir`.
|
|
12
|
+
|
|
13
|
+
- f43d5f2: Add the experiment surfaces. The SDK exports `validateExperimentFile`, `runExperimentFile`, `reportExperimentResult`, `createExperimentLiveAuthorization` and `createExperimentFixtureClient`, with exit codes shared with the CLI. The CLI adds `akit experiment validate`, `run` and `report`. Validation checks the manifest, the SHA-256 of each task source and that every cost-not-applicable target uses a loopback endpoint. `scenario_evaluation` tasks run through the scenario runner and `agent_workflow` tasks through the native workflow engine, both bound to the same targets, with each target's generation settings and the per-repetition seed applied to both kinds; validation parses workflow sources and warns about workflows with semantic outcomes, which experiment runs can't judge. Runs use deterministic fixture clients by default with no network calls. A live run requires `--approve`, `--approved-by` and `--reason`, and `--max-concurrency` for a declared concurrency above 1, before anything executes. Loopback targets get a non-secret placeholder key instead of an ambient API key. Provider configuration is loaded only when a live non-loopback target is bound, and target config resolvers may be async. Live adapters get a timeout of at least the per-case timeout. Results and digest-named evidence are saved under `artemis-output/experiments/`, and validation and run output show the concurrency level. `report` regenerates all three views in HTML and Markdown from a saved result without provider access. The SDK now declares its `yaml` dependency, and the CLI depends on the SDK.
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- cde9742: `new CircuitBreaker(config)` now types its parameter as `Partial<CircuitBreakerConfig>`, matching what it already did at runtime: omitted fields default to `enabled: true`, `threshold: 5`, `windowMs: 60000`, `cooldownMs: 300000` and `halfOpenRequests: 3`. Type-only change; behaviour is unchanged and full configs still type-check.
|
|
18
|
+
- Updated dependencies [4b3a6a9]
|
|
19
|
+
- Updated dependencies [4882eff]
|
|
20
|
+
- Updated dependencies [f20ee54]
|
|
21
|
+
- Updated dependencies [515a69f]
|
|
22
|
+
- Updated dependencies [ae38f94]
|
|
23
|
+
- @artemiskit/core@0.7.0
|
|
24
|
+
- @artemiskit/reports@0.6.0
|
|
25
|
+
- @artemiskit/redteam@0.3.13
|
|
26
|
+
|
|
27
|
+
## 0.6.3
|
|
28
|
+
|
|
29
|
+
### Patch Changes
|
|
30
|
+
|
|
31
|
+
- 5c35b04: Add deterministic offline workflow assessments in technical, executive and comprehensive HTML/Markdown views. Strict saved V1/V2/V3 records share one canonical report model, evidence links, explicit measurement denominators and compatible resume accounting. Expose `akit workflow report` and matching synchronous SDK functions with bounded input and no-overwrite exports. No provider calls, inferred pricing or task rescoring occur. Core receives a release-alignment patch identifying the coherent 0.6.4 milestone; its workflow record contract remains unchanged.
|
|
32
|
+
- Updated dependencies [5c35b04]
|
|
33
|
+
- @artemiskit/reports@0.5.0
|
|
34
|
+
- @artemiskit/core@0.6.4
|
|
35
|
+
- @artemiskit/redteam@0.3.12
|
|
36
|
+
|
|
3
37
|
## 0.6.2
|
|
4
38
|
|
|
5
39
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -247,6 +247,23 @@ const outputResult = await guardian.validateOutput(llmResponse);
|
|
|
247
247
|
| `guardian` | Block violations and log |
|
|
248
248
|
| `hybrid` | Block critical, log others |
|
|
249
249
|
|
|
250
|
+
## Offline workflow reports
|
|
251
|
+
|
|
252
|
+
```ts
|
|
253
|
+
import { ArtemisKit, createWorkflowReport, generateWorkflowReport } from '@artemiskit/sdk';
|
|
254
|
+
|
|
255
|
+
const assessment = createWorkflowReport(savedRecord); // object, JSON string, or record array
|
|
256
|
+
const html = generateWorkflowReport(savedRecord, { view: 'comprehensive', format: 'html' });
|
|
257
|
+
const markdown = new ArtemisKit().generateWorkflowReport(savedRecord, {
|
|
258
|
+
view: 'executive', format: 'markdown',
|
|
259
|
+
});
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
These synchronous functions share `@artemiskit/reports` and never initialize providers, restore
|
|
263
|
+
checkpoints or rescore tasks. `renderWorkflowReportHTML` and `renderWorkflowReportMarkdown` render
|
|
264
|
+
an existing canonical model. Invalid/unavailable measurements, historical evidence and cumulative
|
|
265
|
+
resumed attempts keep their original meaning. See the [guide](../../docs/workflow-reports.md).
|
|
266
|
+
|
|
250
267
|
## License
|
|
251
268
|
|
|
252
269
|
Apache-2.0
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test helper: write a self-contained two-target experiment directory.
|
|
3
|
+
* Shared by the SDK and CLI experiment tests.
|
|
4
|
+
*/
|
|
5
|
+
/** Passes against the fixture client, whose responses start with "Fixture response". */
|
|
6
|
+
export declare const PASSING_SCENARIO = "name: fixture-pass\ncases:\n - id: echo\n prompt: \"Reply with anything.\"\n expected:\n type: contains\n values: [\"Fixture response\"]\n";
|
|
7
|
+
/** A valid measurement that fails against the fixture client. */
|
|
8
|
+
export declare const FAILING_SCENARIO = "name: fixture-fail\ncases:\n - id: capital\n prompt: \"What is the capital of Nigeria?\"\n expected:\n type: contains\n values: [\"Abuja\"]\n";
|
|
9
|
+
/**
|
|
10
|
+
* Agent workflow that passes against the fixture client, which accepts tools but never calls
|
|
11
|
+
* one: the target must answer without reading the brief and without any undeclared tool.
|
|
12
|
+
*/
|
|
13
|
+
export declare const FIXTURE_WORKFLOW = "version: '1'\nkind: agent_workflow\nname: fixture-workflow\ntarget:\n provider: openai\n model: declared-model\nenvironment:\n type: simulated\n policy:\n network: denied\n side_effects: denied\n permissions:\n documents: read\n budgets:\n max_actions: 4\n max_model_requests: 2\n timeout_ms: 10000\ntools:\n - read_document\nworkflow:\n system_instructions: Answer from the conversation; read the brief only when asked to.\n initial_state:\n documents:\n brief: Fixture brief.\n turns:\n - role: user\n content: Reply with anything.\noutcomes:\n deterministic:\n - type: tool_trace\n tool: read_document\n minimum_calls: 0\n maximum_calls: 0\n - type: policy\n rule: no_undeclared_tool\n expected: passed\nevidence:\n trace: summary\n artifacts: checksums\n redact: true\n";
|
|
14
|
+
export declare const LOOPBACK_BASE_URL = "http://127.0.0.1:11434/v1";
|
|
15
|
+
export interface ExperimentFixtureOptions {
|
|
16
|
+
/** Override manifest fields after the defaults are built. */
|
|
17
|
+
overrides?: Record<string, unknown>;
|
|
18
|
+
/** Replace a task's declared digest. */
|
|
19
|
+
digestOverride?: Record<string, string>;
|
|
20
|
+
/** Skip writing these source files. */
|
|
21
|
+
omitSources?: string[];
|
|
22
|
+
/** Target transforms applied to both default targets. */
|
|
23
|
+
target?: (target: Record<string, unknown>) => Record<string, unknown>;
|
|
24
|
+
/** Manifest file name (default `experiment.yaml`; the content is JSON, which YAML accepts). */
|
|
25
|
+
fileName?: string;
|
|
26
|
+
/**
|
|
27
|
+
* Declare one scenario task (`pass`) and one agent workflow task (`workflow`) instead of the
|
|
28
|
+
* two scenario tasks.
|
|
29
|
+
*/
|
|
30
|
+
workflow?: boolean;
|
|
31
|
+
}
|
|
32
|
+
export interface ExperimentFixture {
|
|
33
|
+
directory: string;
|
|
34
|
+
file: string;
|
|
35
|
+
manifest: Record<string, unknown>;
|
|
36
|
+
}
|
|
37
|
+
export declare function sha256(text: string): string;
|
|
38
|
+
/** Live controls every live manifest must declare. */
|
|
39
|
+
export declare const LIVE_CONTROLS: {
|
|
40
|
+
concurrency_ceiling: number;
|
|
41
|
+
stop_conditions: string[];
|
|
42
|
+
};
|
|
43
|
+
export declare function writeExperimentFixture(options?: ExperimentFixtureOptions): Promise<ExperimentFixture>;
|
|
44
|
+
//# sourceMappingURL=experiment-fixtures.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment-fixtures.d.ts","sourceRoot":"","sources":["../../src/__tests__/experiment-fixtures.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAOH,wFAAwF;AACxF,eAAO,MAAM,gBAAgB,gKAO5B,CAAC;AAEF,iEAAiE;AACjE,eAAO,MAAM,gBAAgB,mKAO5B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,q2BAwC5B,CAAC;AAEF,eAAO,MAAM,iBAAiB,8BAA8B,CAAC;AAE7D,MAAM,WAAW,wBAAwB;IACvC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,wCAAwC;IACxC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,yDAAyD;IACzD,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtE,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAQD,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE3C;AAeD,sDAAsD;AACtD,eAAO,MAAM,aAAa;;;CAUzB,CAAC;AAEF,wBAAsB,sBAAsB,CAC1C,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,iBAAiB,CAAC,CAqD5B"}
|
package/dist/artemiskit.d.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* @artemiskit/sdk
|
|
3
3
|
* Main ArtemisKit class - programmatic API for LLM testing
|
|
4
4
|
*/
|
|
5
|
+
import { type GenerateWorkflowReportOptions } from '@artemiskit/reports';
|
|
5
6
|
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult, WorkflowRunOptions } from './types.js';
|
|
6
7
|
/**
|
|
7
8
|
* ArtemisKit SDK - programmatic API for LLM evaluation testing
|
|
@@ -27,8 +28,12 @@ export declare class ArtemisKit {
|
|
|
27
28
|
constructor(config?: ArtemisKitConfig);
|
|
28
29
|
/** Create a cancellable native session without executing model turns or persisting results. */
|
|
29
30
|
createWorkflowSession(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowSession>;
|
|
30
|
-
/** Execute a workflow
|
|
31
|
+
/** Execute a workflow and independently evaluate its declared outcomes. */
|
|
31
32
|
runWorkflow(options: WorkflowRunOptions): Promise<import("@artemiskit/core").AgentWorkflowResult>;
|
|
33
|
+
/** Project saved evidence without running a workflow or loading provider configuration. */
|
|
34
|
+
createWorkflowReport(input: unknown): import("@artemiskit/reports").WorkflowReport;
|
|
35
|
+
/** Generate HTML or Markdown offline; task failures remain visible in a successful export. */
|
|
36
|
+
generateWorkflowReport(input: unknown, options?: GenerateWorkflowReportOptions): string;
|
|
32
37
|
/**
|
|
33
38
|
* Register an event handler
|
|
34
39
|
*/
|
package/dist/artemiskit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuCH,OAAO,EACL,KAAK,6BAA6B,EAGnC,MAAM,qBAAqB,CAAC;AAI7B,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAmBzC,+FAA+F;IACzF,qBAAqB,CAAC,OAAO,EAAE,kBAAkB;IAIvD,2EAA2E;IACrE,WAAW,CAAC,OAAO,EAAE,kBAAkB;IAI7C,2FAA2F;IAC3F,oBAAoB,CAAC,KAAK,EAAE,OAAO;IAInC,8FAA8F;IAC9F,sBAAsB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,6BAA6B;IAQ9E;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|
package/dist/builders/index.js
CHANGED
|
@@ -1,50 +1,3 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
function __accessProp(key) {
|
|
8
|
-
return this[key];
|
|
9
|
-
}
|
|
10
|
-
var __toESMCache_node;
|
|
11
|
-
var __toESMCache_esm;
|
|
12
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
-
var canCache = mod != null && typeof mod === "object";
|
|
14
|
-
if (canCache) {
|
|
15
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
-
var cached = cache.get(mod);
|
|
17
|
-
if (cached)
|
|
18
|
-
return cached;
|
|
19
|
-
}
|
|
20
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
-
for (let key of __getOwnPropNames(mod))
|
|
23
|
-
if (!__hasOwnProp.call(to, key))
|
|
24
|
-
__defProp(to, key, {
|
|
25
|
-
get: __accessProp.bind(mod, key),
|
|
26
|
-
enumerable: true
|
|
27
|
-
});
|
|
28
|
-
if (canCache)
|
|
29
|
-
cache.set(mod, to);
|
|
30
|
-
return to;
|
|
31
|
-
};
|
|
32
|
-
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
-
var __returnValue = (v) => v;
|
|
34
|
-
function __exportSetter(name, newValue) {
|
|
35
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
-
}
|
|
37
|
-
var __export = (target, all) => {
|
|
38
|
-
for (var name in all)
|
|
39
|
-
__defProp(target, name, {
|
|
40
|
-
get: all[name],
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
set: __exportSetter.bind(all, name)
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
47
|
-
|
|
48
1
|
// src/builders/index.ts
|
|
49
2
|
class TestCaseBuilder {
|
|
50
3
|
_id;
|
|
@@ -377,24 +330,24 @@ function anyOf(...expectations) {
|
|
|
377
330
|
return { type: "combined", operator: "or", expectations };
|
|
378
331
|
}
|
|
379
332
|
export {
|
|
380
|
-
|
|
381
|
-
similarity,
|
|
382
|
-
scenario,
|
|
383
|
-
regexCase,
|
|
384
|
-
regex,
|
|
385
|
-
notContains,
|
|
386
|
-
llmGrade,
|
|
387
|
-
jsonSchema,
|
|
388
|
-
jsonCase,
|
|
389
|
-
inline,
|
|
390
|
-
gradedCase,
|
|
391
|
-
fuzzy,
|
|
392
|
-
exactCase,
|
|
393
|
-
exact,
|
|
394
|
-
containsCase,
|
|
395
|
-
contains,
|
|
396
|
-
anyOf,
|
|
397
|
-
allOf,
|
|
333
|
+
ScenarioBuilder,
|
|
398
334
|
TestCaseBuilder,
|
|
399
|
-
|
|
335
|
+
allOf,
|
|
336
|
+
anyOf,
|
|
337
|
+
contains,
|
|
338
|
+
containsCase,
|
|
339
|
+
exact,
|
|
340
|
+
exactCase,
|
|
341
|
+
fuzzy,
|
|
342
|
+
gradedCase,
|
|
343
|
+
inline,
|
|
344
|
+
jsonCase,
|
|
345
|
+
jsonSchema,
|
|
346
|
+
llmGrade,
|
|
347
|
+
notContains,
|
|
348
|
+
regex,
|
|
349
|
+
regexCase,
|
|
350
|
+
scenario,
|
|
351
|
+
similarity,
|
|
352
|
+
testCase
|
|
400
353
|
};
|
package/dist/contracts/index.js
CHANGED
|
@@ -1,50 +1,3 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
function __accessProp(key) {
|
|
8
|
-
return this[key];
|
|
9
|
-
}
|
|
10
|
-
var __toESMCache_node;
|
|
11
|
-
var __toESMCache_esm;
|
|
12
|
-
var __toESM = (mod, isNodeMode, target) => {
|
|
13
|
-
var canCache = mod != null && typeof mod === "object";
|
|
14
|
-
if (canCache) {
|
|
15
|
-
var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
|
|
16
|
-
var cached = cache.get(mod);
|
|
17
|
-
if (cached)
|
|
18
|
-
return cached;
|
|
19
|
-
}
|
|
20
|
-
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
21
|
-
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
22
|
-
for (let key of __getOwnPropNames(mod))
|
|
23
|
-
if (!__hasOwnProp.call(to, key))
|
|
24
|
-
__defProp(to, key, {
|
|
25
|
-
get: __accessProp.bind(mod, key),
|
|
26
|
-
enumerable: true
|
|
27
|
-
});
|
|
28
|
-
if (canCache)
|
|
29
|
-
cache.set(mod, to);
|
|
30
|
-
return to;
|
|
31
|
-
};
|
|
32
|
-
var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
|
|
33
|
-
var __returnValue = (v) => v;
|
|
34
|
-
function __exportSetter(name, newValue) {
|
|
35
|
-
this[name] = __returnValue.bind(null, newValue);
|
|
36
|
-
}
|
|
37
|
-
var __export = (target, all) => {
|
|
38
|
-
for (var name in all)
|
|
39
|
-
__defProp(target, name, {
|
|
40
|
-
get: all[name],
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
set: __exportSetter.bind(all, name)
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
47
|
-
|
|
48
1
|
// src/contracts/index.ts
|
|
49
2
|
function defineAdapter(adapter) {
|
|
50
3
|
if (!adapter.provider) {
|
|
@@ -92,8 +45,8 @@ function definePlugin(plugin) {
|
|
|
92
45
|
return plugin;
|
|
93
46
|
}
|
|
94
47
|
export {
|
|
95
|
-
|
|
96
|
-
definePlugin,
|
|
48
|
+
defineAdapter,
|
|
97
49
|
defineEvaluator,
|
|
98
|
-
|
|
50
|
+
definePlugin,
|
|
51
|
+
defineStorage
|
|
99
52
|
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File-system boundary for experiment surfaces: bounded reads, result directories and
|
|
3
|
+
* write-once evidence files. Shared by the SDK functions and the `akit experiment` CLI.
|
|
4
|
+
*/
|
|
5
|
+
import type { ExperimentRunResult } from '@artemiskit/core';
|
|
6
|
+
/** Largest experiment definition the surfaces will read. */
|
|
7
|
+
export declare const MAX_EXPERIMENT_FILE_BYTES = 1048576;
|
|
8
|
+
/** Largest saved experiment result the report surface will read. */
|
|
9
|
+
export declare const MAX_EXPERIMENT_RESULT_BYTES: number;
|
|
10
|
+
/** Default output root, relative to the working directory; ignored by git. */
|
|
11
|
+
export declare const DEFAULT_EXPERIMENT_OUTPUT_ROOT: string;
|
|
12
|
+
export declare const EXPERIMENT_RESULT_FILE = "result.json";
|
|
13
|
+
export declare const EXPERIMENT_EVIDENCE_DIR = "evidence";
|
|
14
|
+
/**
|
|
15
|
+
* Exit-code-bearing failure for the experiment surfaces. `exitCode` is the CLI exit code the
|
|
16
|
+
* same failure maps to, so SDK callers and the CLI agree.
|
|
17
|
+
*/
|
|
18
|
+
export declare class ExperimentSurfaceError extends Error {
|
|
19
|
+
readonly code: string;
|
|
20
|
+
readonly exitCode: number;
|
|
21
|
+
constructor(code: string, message: string, exitCode: number);
|
|
22
|
+
}
|
|
23
|
+
/** Read a regular file of at most `limit` bytes as strict UTF-8. */
|
|
24
|
+
export declare function readBoundedText(file: string, limit: number): Promise<string>;
|
|
25
|
+
/** Lowercase hex SHA-256 of exact bytes. */
|
|
26
|
+
export declare function sha256Hex(bytes: Uint8Array | string): string;
|
|
27
|
+
/**
|
|
28
|
+
* Canonical JSON with recursively sorted object keys. Matches the serialization of core's
|
|
29
|
+
* `runManifestIdentity` and `workflowRecordIdentity`, so a saved evidence file's SHA-256 equals
|
|
30
|
+
* its recorded artifact digest.
|
|
31
|
+
*/
|
|
32
|
+
export declare function canonicalJson(value: unknown): string;
|
|
33
|
+
/** `artemis-output/experiments/<id>/<timestamp>` under `cwd`, with a filename-safe timestamp. */
|
|
34
|
+
export declare function defaultExperimentOutputDir(experimentId: string, cwd: string, now: Date): string;
|
|
35
|
+
/** Create `directory` (and parents) and refuse to reuse one that already holds files. */
|
|
36
|
+
export declare function prepareEmptyDirectory(directory: string): Promise<void>;
|
|
37
|
+
/** Write a new file; never overwrite. */
|
|
38
|
+
export declare function writeNewFile(file: string, content: string): Promise<void>;
|
|
39
|
+
/**
|
|
40
|
+
* Save one attempt's evidence (a scenario run manifest or a workflow record) as canonical JSON
|
|
41
|
+
* named by its digest, and prove the written bytes match the identity recorded in the
|
|
42
|
+
* experiment result. A throw marks the attempt `infrastructure_failed` in the executor.
|
|
43
|
+
*/
|
|
44
|
+
export declare function saveAttemptEvidence(directory: string, evidence: unknown, digest: string): Promise<string>;
|
|
45
|
+
/** Persist the V1 experiment result as pretty JSON. */
|
|
46
|
+
export declare function saveExperimentResult(directory: string, result: ExperimentRunResult): Promise<string>;
|
|
47
|
+
/**
|
|
48
|
+
* Read the saved evidence files named by `digests` from an `evidence/` directory, as exact bytes
|
|
49
|
+
* keyed by digest. A missing, unreadable or oversized file maps to `null` so the case-level
|
|
50
|
+
* summary reports it; the bytes are verified against the digest by the report model, not here.
|
|
51
|
+
* Returns `undefined` when the directory does not exist (results saved without evidence).
|
|
52
|
+
*/
|
|
53
|
+
export declare function readExperimentCaseEvidence(evidenceDir: string, digests: readonly string[]): Promise<Map<string, Uint8Array | null> | undefined>;
|
|
54
|
+
//# sourceMappingURL=experiment-files.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment-files.d.ts","sourceRoot":"","sources":["../src/experiment-files.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAG5D,4DAA4D;AAC5D,eAAO,MAAM,yBAAyB,UAAY,CAAC;AACnD,oEAAoE;AACpE,eAAO,MAAM,2BAA2B,QAAiB,CAAC;AAC1D,8EAA8E;AAC9E,eAAO,MAAM,8BAA8B,QAA6C,CAAC;AACzF,eAAO,MAAM,sBAAsB,gBAAgB,CAAC;AACpD,eAAO,MAAM,uBAAuB,aAAa,CAAC;AAKlD;;;GAGG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;CAM5D;AAED,oEAAoE;AACpE,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAWlF;AAED,4CAA4C;AAC5C,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM,CAE5D;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEpD;AAaD,iGAAiG;AACjG,wBAAgB,0BAA0B,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,MAAM,CAG/F;AAED,yFAAyF;AACzF,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAW5E;AAED,yCAAyC;AACzC,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ/E;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,CAAC,CAcjB;AAED,uDAAuD;AACvD,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,mBAAmB,GAC1B,OAAO,CAAC,MAAM,CAAC,CAYjB;AAID;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAC9C,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC,GAAG,SAAS,CAAC,CAcrD"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic in-memory model client for fixture-mode experiments.
|
|
3
|
+
*
|
|
4
|
+
* Fixture runs exercise the full experiment pipeline (matrix, executor, evaluators, evidence,
|
|
5
|
+
* reports) without any network access. Responses are fixed text, so fixture outcomes describe
|
|
6
|
+
* the pipeline, never a model.
|
|
7
|
+
*/
|
|
8
|
+
import type { ExperimentTarget, ModelClient } from '@artemiskit/core';
|
|
9
|
+
export declare const FIXTURE_PROVIDER = "artemiskit-fixture";
|
|
10
|
+
export interface ExperimentFixtureClientOptions {
|
|
11
|
+
/**
|
|
12
|
+
* Advertise structured tool use. The client accepts tool definitions but never calls a
|
|
13
|
+
* tool, so a fixture workflow can run (the workflow engine requires tool support) and its
|
|
14
|
+
* outcome depends only on the declared state and outcomes.
|
|
15
|
+
*/
|
|
16
|
+
toolUse?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** Create the offline client bound to one fixture-mode target. */
|
|
19
|
+
export declare function createExperimentFixtureClient(target: ExperimentTarget, options?: ExperimentFixtureClientOptions): ModelClient;
|
|
20
|
+
//# sourceMappingURL=experiment-fixture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment-fixture.d.ts","sourceRoot":"","sources":["../src/experiment-fixture.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,gBAAgB,EAGhB,WAAW,EACZ,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,gBAAgB,uBAAuB,CAAC;AAKrD,MAAM,WAAW,8BAA8B;IAC7C;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,kEAAkE;AAClE,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,gBAAgB,EACxB,OAAO,GAAE,8BAAmC,GAC3C,WAAW,CA2Bb"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exit codes and the plain-text run summary shared by the SDK and `akit experiment run`.
|
|
3
|
+
* The summary is descriptive: it lists targets in declared order and never ranks them.
|
|
4
|
+
*/
|
|
5
|
+
import type { ExperimentRunResult, ExperimentTarget } from '@artemiskit/core';
|
|
6
|
+
import { type ExperimentCaseLevelSummary } from '@artemiskit/reports';
|
|
7
|
+
/** Exit codes for `akit experiment run`; SDK outcomes carry the same value. */
|
|
8
|
+
export declare const EXPERIMENT_EXIT_CODES: {
|
|
9
|
+
/** Completed with a valid passed or failed measurement for every declared coordinate. */
|
|
10
|
+
readonly complete: 0;
|
|
11
|
+
/** Unexpected I/O or internal failure, including an unusable output directory. */
|
|
12
|
+
readonly error: 1;
|
|
13
|
+
/** The experiment file, a task source or a target declaration failed validation. */
|
|
14
|
+
readonly invalid: 2;
|
|
15
|
+
/** Live experiment without a valid explicit approval; nothing was executed. */
|
|
16
|
+
readonly unauthorized: 3;
|
|
17
|
+
/** Execution stopped early (budget or usage stop condition); see the stop reason. */
|
|
18
|
+
readonly stopped: 4;
|
|
19
|
+
/** Finished, but some coordinates are invalid, unsupported, excluded, errored or incomplete. */
|
|
20
|
+
readonly coverageIncomplete: 5;
|
|
21
|
+
};
|
|
22
|
+
export type ExperimentRunStatus = keyof typeof EXPERIMENT_EXIT_CODES;
|
|
23
|
+
export type ExperimentExitCode = (typeof EXPERIMENT_EXIT_CODES)[ExperimentRunStatus];
|
|
24
|
+
/** Classify a finished run. Early stops take precedence over incomplete coverage. */
|
|
25
|
+
export declare function experimentRunStatus(result: ExperimentRunResult): 'complete' | 'stopped' | 'coverageIncomplete';
|
|
26
|
+
/** Deployment label for a target, e.g. "open-weight, run locally via ollama; cost not applicable". */
|
|
27
|
+
export declare function describeTargetDeployment(target: ExperimentTarget): string;
|
|
28
|
+
/**
|
|
29
|
+
* Declared concurrency limit and, for concurrent runs, the recorded peak and caveat, e.g.
|
|
30
|
+
* "up to 4 coordinates in flight (declared limit); recorded peak 3; completion order and
|
|
31
|
+
* sampled outputs may differ from a serial run".
|
|
32
|
+
*/
|
|
33
|
+
export declare function describeConcurrency(result: ExperimentRunResult): string;
|
|
34
|
+
/** Human-readable summary of a saved run. Makes no ranking or winner claim. */
|
|
35
|
+
export declare function formatExperimentRunSummary(result: ExperimentRunResult, options?: {
|
|
36
|
+
resultPath?: string;
|
|
37
|
+
caseLevel?: ExperimentCaseLevelSummary;
|
|
38
|
+
}): string;
|
|
39
|
+
/**
|
|
40
|
+
* Plain-text case-level summary: individual scenario cases from verified per-attempt evidence,
|
|
41
|
+
* per target (in `targetOrder` when given) and overall, or why it is unavailable. No ranking.
|
|
42
|
+
*/
|
|
43
|
+
export declare function formatExperimentCaseSummary(caseLevel: ExperimentCaseLevelSummary, targetOrder?: readonly string[]): string;
|
|
44
|
+
//# sourceMappingURL=experiment-summary.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment-summary.d.ts","sourceRoot":"","sources":["../src/experiment-summary.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAqB,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAEL,KAAK,0BAA0B,EAEhC,MAAM,qBAAqB,CAAC;AAE7B,+EAA+E;AAC/E,eAAO,MAAM,qBAAqB;IAChC,yFAAyF;;IAEzF,kFAAkF;;IAElF,oFAAoF;;IAEpF,+EAA+E;;IAE/E,qFAAqF;;IAErF,gGAAgG;;CAExF,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,qBAAqB,CAAC;AACrE,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,mBAAmB,CAAC,CAAC;AAErF,qFAAqF;AACrF,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,mBAAmB,GAC1B,UAAU,GAAG,SAAS,GAAG,oBAAoB,CAI/C;AAED,sGAAsG;AACtG,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAOzE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAKvE;AAED,+EAA+E;AAC/E,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,0BAA0B,CAAA;CAAO,GAC5E,MAAM,CAyCR;AAKD;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,SAAS,EAAE,0BAA0B,EACrC,WAAW,GAAE,SAAS,MAAM,EAAO,GAClC,MAAM,CAeR"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Offline validation of an experiment file: manifest contract, task source identities and the
|
|
3
|
+
* host-side locality check for targets that declare no metered cost. Makes no model calls.
|
|
4
|
+
*/
|
|
5
|
+
import { type ExperimentManifest, type ExperimentMode } from '@artemiskit/core';
|
|
6
|
+
export type ExperimentCheckSubject = 'file' | 'manifest' | 'task' | 'target';
|
|
7
|
+
export type ExperimentCheckStatus = 'ok' | 'warning' | 'error';
|
|
8
|
+
/** One validation finding. `code` is stable and machine-readable. */
|
|
9
|
+
export interface ExperimentCheck {
|
|
10
|
+
subject: ExperimentCheckSubject;
|
|
11
|
+
/** Task or target ID for task and target checks. */
|
|
12
|
+
id?: string;
|
|
13
|
+
status: ExperimentCheckStatus;
|
|
14
|
+
code: string;
|
|
15
|
+
message: string;
|
|
16
|
+
}
|
|
17
|
+
export interface ExperimentValidation {
|
|
18
|
+
/** True when no check has status `error`. Warnings do not block a run. */
|
|
19
|
+
valid: boolean;
|
|
20
|
+
/** Absolute path of the experiment file. */
|
|
21
|
+
file: string;
|
|
22
|
+
/** Directory that task source paths resolve under (the experiment file's directory). */
|
|
23
|
+
root: string;
|
|
24
|
+
experimentId?: string;
|
|
25
|
+
mode?: ExperimentMode;
|
|
26
|
+
/** Declared concurrency limit (coordinates in flight), when the manifest is valid. */
|
|
27
|
+
concurrency?: number;
|
|
28
|
+
/** Parsed manifest, present when the manifest contract itself is valid. */
|
|
29
|
+
manifest?: ExperimentManifest;
|
|
30
|
+
checks: ExperimentCheck[];
|
|
31
|
+
/** CLI exit code for `akit experiment validate`: 0 valid, 1 unreadable file, 2 invalid. */
|
|
32
|
+
exitCode: 0 | 1 | 2;
|
|
33
|
+
}
|
|
34
|
+
export declare const LOOPBACK_REQUIRED_MESSAGE = "cost not applicable is only accepted for loopback endpoints";
|
|
35
|
+
/**
|
|
36
|
+
* Validate an experiment file without contacting any model or network endpoint.
|
|
37
|
+
*
|
|
38
|
+
* Checks the manifest contract; that each task source exists under the experiment directory and
|
|
39
|
+
* its exact bytes match the declared SHA-256; that scenario and workflow sources parse; that
|
|
40
|
+
* workflow tasks can run as declared (warnings otherwise); that every target
|
|
41
|
+
* declaring `cost_basis: not_applicable` names a loopback `settings.base_url`; and that target
|
|
42
|
+
* generation settings are in range and sent by the target's provider adapter.
|
|
43
|
+
*/
|
|
44
|
+
export declare function validateExperimentFile(experimentFile: string): Promise<ExperimentValidation>;
|
|
45
|
+
/** Values accepted for a target's `settings.reasoning_effort`. */
|
|
46
|
+
export declare const REASONING_EFFORTS: readonly ["none", "minimal", "low", "medium", "high"];
|
|
47
|
+
export declare function isReasoningEffort(value: unknown): value is (typeof REASONING_EFFORTS)[number];
|
|
48
|
+
//# sourceMappingURL=experiment-validation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment-validation.d.ts","sourceRoot":"","sources":["../src/experiment-validation.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,cAAc,EAWpB,MAAM,kBAAkB,CAAC;AAI1B,MAAM,MAAM,sBAAsB,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAC7E,MAAM,MAAM,qBAAqB,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC;AAE/D,qEAAqE;AACrE,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,sBAAsB,CAAC;IAChC,oDAAoD;IACpD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,qBAAqB,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,0EAA0E;IAC1E,KAAK,EAAE,OAAO,CAAC;IACf,4CAA4C;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB,sFAAsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,2FAA2F;IAC3F,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;CACrB;AAED,eAAO,MAAM,yBAAyB,gEACyB,CAAC;AAEhE;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,oBAAoB,CAAC,CA4D/B;AAwGD,kEAAkE;AAClE,eAAO,MAAM,iBAAiB,uDAAwD,CAAC;AAEvF,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAE7F"}
|