@artemiskit/core 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/artifacts/manifest.d.ts +2 -1
- package/dist/artifacts/manifest.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +29 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/index.js +106 -11
- package/dist/provenance/execution-provenance.d.ts +11 -0
- package/dist/provenance/execution-provenance.d.ts.map +1 -0
- package/dist/provenance/index.d.ts +1 -0
- package/dist/provenance/index.d.ts.map +1 -1
- package/dist/runner/executor.d.ts.map +1 -1
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/types.d.ts +2 -0
- package/dist/runner/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/artifacts/manifest.test.ts +37 -1
- package/src/artifacts/manifest.ts +5 -1
- package/src/artifacts/types.ts +89 -0
- package/src/provenance/execution-provenance.test.ts +59 -0
- package/src/provenance/execution-provenance.ts +53 -0
- package/src/provenance/index.ts +1 -0
- package/src/runner/executor.test.ts +10 -0
- package/src/runner/executor.ts +71 -9
- package/src/runner/release-validation.test.ts +36 -0
- package/src/runner/runner.ts +11 -1
- package/src/runner/types.ts +2 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# @artemiskit/core
|
|
2
2
|
|
|
3
|
+
## 0.5.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Preserve bounded requested and observed target-model evidence per case and per run, separately from
|
|
8
|
+
evaluator identities. Normal manifests now use v1.3 execution provenance while retaining readable
|
|
9
|
+
historical artifacts. CI JSON output is machine-readable without progress text.
|
|
10
|
+
|
|
3
11
|
## 0.5.0
|
|
4
12
|
|
|
5
13
|
### Minor Changes
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run manifest generation utilities
|
|
3
3
|
*/
|
|
4
|
-
import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
|
|
4
|
+
import type { CaseResult, ExecutionProvenance, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
|
|
5
5
|
/**
|
|
6
6
|
* Create a new run manifest
|
|
7
7
|
*/
|
|
@@ -10,6 +10,7 @@ export declare function createRunManifest(options: {
|
|
|
10
10
|
config: RunConfig;
|
|
11
11
|
resolvedConfig?: ResolvedConfig;
|
|
12
12
|
workloadIdentity?: WorkloadIdentity;
|
|
13
|
+
executionProvenance?: ExecutionProvenance;
|
|
13
14
|
cases: CaseResult[];
|
|
14
15
|
startTime: Date;
|
|
15
16
|
endTime: Date;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CA2Cd"}
|
|
@@ -52,6 +52,31 @@ export interface WorkloadIdentity {
|
|
|
52
52
|
workload: ContentIdentity;
|
|
53
53
|
rubric: ContentIdentity;
|
|
54
54
|
}
|
|
55
|
+
/** Bounded target identity captured from a case execution. */
|
|
56
|
+
export interface CaseTargetEvidence {
|
|
57
|
+
provider: string;
|
|
58
|
+
requested_model?: string;
|
|
59
|
+
/** Model identifiers returned by the target provider during this case. */
|
|
60
|
+
observed_models?: string[];
|
|
61
|
+
}
|
|
62
|
+
/** Requested and observed execution configuration for a complete run. */
|
|
63
|
+
export interface ExecutionProvenance {
|
|
64
|
+
schema_version: '1';
|
|
65
|
+
target: {
|
|
66
|
+
provider: string;
|
|
67
|
+
requested_models?: string[];
|
|
68
|
+
observed_models?: string[];
|
|
69
|
+
generation?: {
|
|
70
|
+
temperature?: number;
|
|
71
|
+
max_tokens?: number;
|
|
72
|
+
seed?: number;
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
/** Judge/evaluator model identities, never combined with target identity. */
|
|
76
|
+
evaluator?: {
|
|
77
|
+
models?: string[];
|
|
78
|
+
};
|
|
79
|
+
}
|
|
55
80
|
/**
|
|
56
81
|
* Terminal status of a case measurement.
|
|
57
82
|
*
|
|
@@ -99,6 +124,8 @@ export interface CaseResult {
|
|
|
99
124
|
error?: string;
|
|
100
125
|
/** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
|
|
101
126
|
evidence?: CaseEvaluationEvidence;
|
|
127
|
+
/** Requested and observed target identity for this case. */
|
|
128
|
+
target?: CaseTargetEvidence;
|
|
102
129
|
/** Redaction information for this case */
|
|
103
130
|
redaction?: CaseRedactionInfo;
|
|
104
131
|
/** Ordered tool activity captured for an enabled tool loop. */
|
|
@@ -256,6 +283,8 @@ export interface RunManifest {
|
|
|
256
283
|
resolved_config?: ResolvedConfig;
|
|
257
284
|
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
258
285
|
workload_identity?: WorkloadIdentity;
|
|
286
|
+
/** Requested and observed target/evaluator configuration for this run. */
|
|
287
|
+
execution_provenance?: ExecutionProvenance;
|
|
259
288
|
metrics: RunMetrics;
|
|
260
289
|
git: GitInfo;
|
|
261
290
|
provenance: ProvenanceInfo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CA4B7F;AAqED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,8DAA8D;AAC9D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0EAA0E;IAC1E,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,yEAAyE;AACzE,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,GAAG,CAAC;IACpB,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,UAAU,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,IAAI,CAAC,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC;IACF,6EAA6E;IAC7E,SAAS,CAAC,EAAE;QACV,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,4DAA4D;IAC5D,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,0EAA0E;IAC1E,oBAAoB,CAAC,EAAE,mBAAmB,CAAC;IAC3C,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAkC7F;AAyHD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
package/dist/index.js
CHANGED
|
@@ -20525,6 +20525,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
20525
20525
|
async function executeCase(testCase, context) {
|
|
20526
20526
|
const { timeout, retries = 0 } = context;
|
|
20527
20527
|
const caseStartTime = Date.now();
|
|
20528
|
+
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
20528
20529
|
let lastError = null;
|
|
20529
20530
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
20530
20531
|
try {
|
|
@@ -20555,11 +20556,12 @@ async function executeCase(testCase, context) {
|
|
|
20555
20556
|
response: "",
|
|
20556
20557
|
expected: testCase.expected,
|
|
20557
20558
|
tags: testCase.tags,
|
|
20558
|
-
error: lastError?.message
|
|
20559
|
+
error: lastError?.message,
|
|
20560
|
+
target: targetEvidence(context.client.provider, requestedModel)
|
|
20559
20561
|
};
|
|
20560
20562
|
}
|
|
20561
20563
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
20562
|
-
const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
|
|
20564
|
+
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
20563
20565
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
20564
20566
|
let prompt = substituteVariables(testCase.prompt, variables);
|
|
20565
20567
|
if (scenario.setup?.systemPrompt && typeof prompt === "string") {
|
|
@@ -20577,7 +20579,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20577
20579
|
const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
|
20578
20580
|
const generate = () => client.generate({
|
|
20579
20581
|
prompt: loopPrompt,
|
|
20580
|
-
model: testCase.model || scenario.model,
|
|
20582
|
+
model: testCase.model || requestedModel || scenario.model,
|
|
20581
20583
|
temperature: scenario.temperature,
|
|
20582
20584
|
maxTokens: scenario.maxTokens,
|
|
20583
20585
|
seed: scenario.seed,
|
|
@@ -20585,6 +20587,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20585
20587
|
});
|
|
20586
20588
|
const generatePromise = generate();
|
|
20587
20589
|
let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
|
|
20590
|
+
const observedModels = [result.model];
|
|
20588
20591
|
const generationMetrics = {
|
|
20589
20592
|
latencyMs: result.latencyMs,
|
|
20590
20593
|
tokens: { ...result.tokens }
|
|
@@ -20595,7 +20598,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20595
20598
|
status: "error",
|
|
20596
20599
|
steps: 0,
|
|
20597
20600
|
terminationReason: "tool_error"
|
|
20598
|
-
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
|
|
20601
|
+
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20599
20602
|
}
|
|
20600
20603
|
const executor = toolExecutor ?? new FixtureToolExecutor({
|
|
20601
20604
|
tools,
|
|
@@ -20614,7 +20617,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20614
20617
|
status: "error",
|
|
20615
20618
|
steps: step,
|
|
20616
20619
|
terminationReason: "duplicate_call"
|
|
20617
|
-
}, generationMetrics, "TOOL_DUPLICATE_CALL");
|
|
20620
|
+
}, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20618
20621
|
}
|
|
20619
20622
|
seenCalls.add(fingerprint);
|
|
20620
20623
|
const toolStart = Date.now();
|
|
@@ -20641,7 +20644,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20641
20644
|
status: "error",
|
|
20642
20645
|
steps: step + 1,
|
|
20643
20646
|
terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
|
|
20644
|
-
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
|
|
20647
|
+
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20645
20648
|
}
|
|
20646
20649
|
const content = JSON.stringify(execution.result ?? {});
|
|
20647
20650
|
loopPrompt.push({
|
|
@@ -20653,7 +20656,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20653
20656
|
}
|
|
20654
20657
|
const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
|
|
20655
20658
|
if (remainingLoopTime <= 0) {
|
|
20656
|
-
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
|
|
20659
|
+
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20657
20660
|
}
|
|
20658
20661
|
const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
|
|
20659
20662
|
try {
|
|
@@ -20664,8 +20667,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20664
20667
|
status: "error",
|
|
20665
20668
|
steps: step + 1,
|
|
20666
20669
|
terminationReason: timedOut ? "timeout" : "tool_error"
|
|
20667
|
-
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
|
|
20670
|
+
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20668
20671
|
}
|
|
20672
|
+
observedModels.push(result.model);
|
|
20669
20673
|
generationMetrics.latencyMs += result.latencyMs;
|
|
20670
20674
|
generationMetrics.tokens.prompt += result.tokens.prompt;
|
|
20671
20675
|
generationMetrics.tokens.completion += result.tokens.completion;
|
|
@@ -20676,7 +20680,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20676
20680
|
status: "error",
|
|
20677
20681
|
steps: policy.maxSteps,
|
|
20678
20682
|
terminationReason: "max_steps"
|
|
20679
|
-
}, generationMetrics, "TOOL_LOOP_MAX_STEPS");
|
|
20683
|
+
}, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20680
20684
|
}
|
|
20681
20685
|
toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
|
|
20682
20686
|
}
|
|
@@ -20772,11 +20776,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20772
20776
|
tags: testCase.tags,
|
|
20773
20777
|
redaction: redactionInfo,
|
|
20774
20778
|
evidence: finalEvidence,
|
|
20779
|
+
target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
|
|
20775
20780
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
20776
20781
|
toolLoop
|
|
20777
20782
|
};
|
|
20778
20783
|
}
|
|
20779
|
-
function
|
|
20784
|
+
function targetEvidence(provider, requestedModel, observedModels) {
|
|
20785
|
+
const observed = [
|
|
20786
|
+
...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
|
|
20787
|
+
].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
|
|
20788
|
+
return {
|
|
20789
|
+
provider: sanitizeArtifactText(provider, 100) ?? "unknown",
|
|
20790
|
+
...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
|
|
20791
|
+
...observed.length ? { observed_models: observed } : {}
|
|
20792
|
+
};
|
|
20793
|
+
}
|
|
20794
|
+
function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
|
|
20780
20795
|
return new ToolLoopError(code, {
|
|
20781
20796
|
id: testCase.id,
|
|
20782
20797
|
name: testCase.name,
|
|
@@ -20792,6 +20807,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
|
|
|
20792
20807
|
expected: testCase.expected,
|
|
20793
20808
|
tags: testCase.tags,
|
|
20794
20809
|
error: code,
|
|
20810
|
+
target,
|
|
20795
20811
|
toolTrace,
|
|
20796
20812
|
toolLoop
|
|
20797
20813
|
});
|
|
@@ -21295,6 +21311,9 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21295
21311
|
if (manifest.workload_identity !== undefined) {
|
|
21296
21312
|
assertWorkloadIdentity(manifest.workload_identity);
|
|
21297
21313
|
}
|
|
21314
|
+
if (manifest.execution_provenance !== undefined) {
|
|
21315
|
+
assertExecutionProvenance(manifest.execution_provenance);
|
|
21316
|
+
}
|
|
21298
21317
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
21299
21318
|
if (!isRecord2(caseResult)) {
|
|
21300
21319
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -21305,8 +21324,38 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21305
21324
|
if (caseResult.evidence !== undefined) {
|
|
21306
21325
|
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
21307
21326
|
}
|
|
21327
|
+
if (caseResult.target !== undefined) {
|
|
21328
|
+
assertCaseTargetEvidence(caseResult.target);
|
|
21329
|
+
}
|
|
21330
|
+
}
|
|
21331
|
+
}
|
|
21332
|
+
function assertCaseTargetEvidence(target) {
|
|
21333
|
+
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
21334
|
+
throw new Error("Invalid run manifest: malformed target evidence");
|
|
21335
|
+
}
|
|
21336
|
+
}
|
|
21337
|
+
function assertExecutionProvenance(provenance) {
|
|
21338
|
+
if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
|
|
21339
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
21340
|
+
}
|
|
21341
|
+
const target = provenance.target;
|
|
21342
|
+
if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
|
|
21343
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
21344
|
+
}
|
|
21345
|
+
if (provenance.evaluator !== undefined) {
|
|
21346
|
+
if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
|
|
21347
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
21348
|
+
}
|
|
21308
21349
|
}
|
|
21309
21350
|
}
|
|
21351
|
+
function isBoundedStringList(value) {
|
|
21352
|
+
return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
|
|
21353
|
+
}
|
|
21354
|
+
function isGenerationConfig(value) {
|
|
21355
|
+
if (!isRecord2(value))
|
|
21356
|
+
return false;
|
|
21357
|
+
return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
|
|
21358
|
+
}
|
|
21310
21359
|
function assertWorkloadIdentity(identity) {
|
|
21311
21360
|
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
21312
21361
|
throw new Error("Invalid run manifest: malformed workload identity");
|
|
@@ -21357,6 +21406,7 @@ function createRunManifest(options) {
|
|
|
21357
21406
|
config,
|
|
21358
21407
|
resolvedConfig,
|
|
21359
21408
|
workloadIdentity,
|
|
21409
|
+
executionProvenance,
|
|
21360
21410
|
cases,
|
|
21361
21411
|
startTime,
|
|
21362
21412
|
endTime,
|
|
@@ -21369,7 +21419,7 @@ function createRunManifest(options) {
|
|
|
21369
21419
|
const git = getGitInfo();
|
|
21370
21420
|
const environment = getEnvironmentInfo();
|
|
21371
21421
|
return {
|
|
21372
|
-
version: "1.
|
|
21422
|
+
version: "1.3",
|
|
21373
21423
|
run_id: nanoid(12),
|
|
21374
21424
|
project,
|
|
21375
21425
|
start_time: startTime.toISOString(),
|
|
@@ -21378,6 +21428,7 @@ function createRunManifest(options) {
|
|
|
21378
21428
|
config,
|
|
21379
21429
|
resolved_config: resolvedConfig,
|
|
21380
21430
|
workload_identity: workloadIdentity,
|
|
21431
|
+
execution_provenance: executionProvenance,
|
|
21381
21432
|
metrics,
|
|
21382
21433
|
git,
|
|
21383
21434
|
provenance: {
|
|
@@ -21530,6 +21581,39 @@ function canonicalize(value, patterns, key) {
|
|
|
21530
21581
|
}
|
|
21531
21582
|
return value;
|
|
21532
21583
|
}
|
|
21584
|
+
// src/provenance/execution-provenance.ts
|
|
21585
|
+
function createExecutionProvenance(options) {
|
|
21586
|
+
const requestedModels = uniqueStrings([
|
|
21587
|
+
options.requestedModel,
|
|
21588
|
+
...options.cases.map((caseResult) => caseResult.target?.requested_model)
|
|
21589
|
+
]);
|
|
21590
|
+
const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
|
|
21591
|
+
const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
|
|
21592
|
+
const generation = omitUndefined({
|
|
21593
|
+
temperature: options.temperature,
|
|
21594
|
+
max_tokens: options.maxTokens,
|
|
21595
|
+
seed: options.seed
|
|
21596
|
+
});
|
|
21597
|
+
return {
|
|
21598
|
+
schema_version: "1",
|
|
21599
|
+
target: {
|
|
21600
|
+
provider: boundedString(options.provider, 100) ?? "unknown",
|
|
21601
|
+
...requestedModels.length ? { requested_models: requestedModels } : {},
|
|
21602
|
+
...observedModels.length ? { observed_models: observedModels } : {},
|
|
21603
|
+
...Object.keys(generation).length ? { generation } : {}
|
|
21604
|
+
},
|
|
21605
|
+
...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
|
|
21606
|
+
};
|
|
21607
|
+
}
|
|
21608
|
+
function uniqueStrings(values) {
|
|
21609
|
+
return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
|
|
21610
|
+
}
|
|
21611
|
+
function boundedString(value, maxLength) {
|
|
21612
|
+
return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
|
|
21613
|
+
}
|
|
21614
|
+
function omitUndefined(value) {
|
|
21615
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
|
21616
|
+
}
|
|
21533
21617
|
// src/runner/runner.ts
|
|
21534
21618
|
async function runScenario(options) {
|
|
21535
21619
|
const {
|
|
@@ -21563,6 +21647,7 @@ async function runScenario(options) {
|
|
|
21563
21647
|
const result = await executeCase(testCase, {
|
|
21564
21648
|
client,
|
|
21565
21649
|
scenario,
|
|
21650
|
+
requestedModel: resolvedConfig?.model,
|
|
21566
21651
|
timeout: testCase.timeout || timeout,
|
|
21567
21652
|
retries: testCase.retries ?? retries,
|
|
21568
21653
|
redaction,
|
|
@@ -21579,6 +21664,7 @@ async function runScenario(options) {
|
|
|
21579
21664
|
const result = await executeCase(testCase, {
|
|
21580
21665
|
client,
|
|
21581
21666
|
scenario,
|
|
21667
|
+
requestedModel: resolvedConfig?.model,
|
|
21582
21668
|
timeout: testCase.timeout || timeout,
|
|
21583
21669
|
retries: testCase.retries ?? retries,
|
|
21584
21670
|
redaction,
|
|
@@ -21623,6 +21709,14 @@ async function runScenario(options) {
|
|
|
21623
21709
|
},
|
|
21624
21710
|
resolvedConfig,
|
|
21625
21711
|
workloadIdentity: createWorkloadIdentity(scenario),
|
|
21712
|
+
executionProvenance: createExecutionProvenance({
|
|
21713
|
+
provider: client.provider,
|
|
21714
|
+
requestedModel: resolvedConfig?.model || scenario.model,
|
|
21715
|
+
temperature: resolvedConfig?.temperature ?? scenario.temperature,
|
|
21716
|
+
maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
|
|
21717
|
+
seed: scenario.seed,
|
|
21718
|
+
cases: results
|
|
21719
|
+
}),
|
|
21626
21720
|
cases: results,
|
|
21627
21721
|
startTime,
|
|
21628
21722
|
endTime,
|
|
@@ -32705,6 +32799,7 @@ export {
|
|
|
32705
32799
|
assertRunManifestIntegrity,
|
|
32706
32800
|
createAdapter,
|
|
32707
32801
|
createDefaultRedactor,
|
|
32802
|
+
createExecutionProvenance,
|
|
32708
32803
|
createNoOpRedactor,
|
|
32709
32804
|
createRedactionOptions,
|
|
32710
32805
|
createRunManifest,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Bounded execution provenance derived from declared and observed run evidence. */
|
|
2
|
+
import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
|
|
3
|
+
export declare function createExecutionProvenance(options: {
|
|
4
|
+
provider: string;
|
|
5
|
+
requestedModel?: string;
|
|
6
|
+
temperature?: number;
|
|
7
|
+
maxTokens?: number;
|
|
8
|
+
seed?: number;
|
|
9
|
+
cases: CaseResult[];
|
|
10
|
+
}): ExecutionProvenance;
|
|
11
|
+
//# sourceMappingURL=execution-provenance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-provenance.d.ts","sourceRoot":"","sources":["../../src/provenance/execution-provenance.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAEpF,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,GAAG,mBAAmB,CA2BtB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAyCrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAsIzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
|
package/dist/runner/types.d.ts
CHANGED
|
@@ -52,6 +52,8 @@ export interface RunResult {
|
|
|
52
52
|
export interface ExecutorContext {
|
|
53
53
|
client: ModelClient;
|
|
54
54
|
scenario: Scenario;
|
|
55
|
+
/** Effective model selected outside the scenario, such as a CLI override. */
|
|
56
|
+
requestedModel?: string;
|
|
55
57
|
timeout?: number;
|
|
56
58
|
retries?: number;
|
|
57
59
|
/** Redaction configuration for this execution */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
|
package/package.json
CHANGED
|
@@ -55,7 +55,7 @@ describe('createRunManifest', () => {
|
|
|
55
55
|
endTime,
|
|
56
56
|
});
|
|
57
57
|
|
|
58
|
-
expect(manifest.version).toBe('1.
|
|
58
|
+
expect(manifest.version).toBe('1.3');
|
|
59
59
|
expect(manifest.project).toBe('test-project');
|
|
60
60
|
expect(manifest.run_id).toBeTruthy();
|
|
61
61
|
expect(manifest.run_id.length).toBe(12);
|
|
@@ -189,6 +189,30 @@ describe('createRunManifest', () => {
|
|
|
189
189
|
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
190
190
|
});
|
|
191
191
|
|
|
192
|
+
test('retains target and evaluator execution provenance separately', () => {
|
|
193
|
+
const manifest = createRunManifest({
|
|
194
|
+
project: 'test-project',
|
|
195
|
+
config: { scenario: 'test-scenario', provider: 'openai' },
|
|
196
|
+
executionProvenance: {
|
|
197
|
+
schema_version: '1',
|
|
198
|
+
target: {
|
|
199
|
+
provider: 'openai',
|
|
200
|
+
requested_models: ['gpt-requested'],
|
|
201
|
+
observed_models: ['gpt-observed'],
|
|
202
|
+
generation: { temperature: 0, max_tokens: 100, seed: 42 },
|
|
203
|
+
},
|
|
204
|
+
evaluator: { models: ['judge-model'] },
|
|
205
|
+
},
|
|
206
|
+
cases: mockCases,
|
|
207
|
+
startTime: new Date(),
|
|
208
|
+
endTime: new Date(),
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
expect(manifest.execution_provenance?.target.observed_models).toEqual(['gpt-observed']);
|
|
212
|
+
expect(manifest.execution_provenance?.evaluator?.models).toEqual(['judge-model']);
|
|
213
|
+
expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
|
|
214
|
+
});
|
|
215
|
+
|
|
192
216
|
test('includes provenance information', () => {
|
|
193
217
|
const manifest = createRunManifest({
|
|
194
218
|
project: 'test-project',
|
|
@@ -303,5 +327,17 @@ describe('createRunManifest', () => {
|
|
|
303
327
|
expect(() =>
|
|
304
328
|
assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
|
|
305
329
|
).toThrow('malformed workload identity');
|
|
330
|
+
expect(() =>
|
|
331
|
+
assertRunManifestIntegrity({
|
|
332
|
+
...historical,
|
|
333
|
+
execution_provenance: { schema_version: '1', target: { provider: '' } },
|
|
334
|
+
})
|
|
335
|
+
).toThrow('malformed execution provenance');
|
|
336
|
+
expect(() =>
|
|
337
|
+
assertRunManifestIntegrity({
|
|
338
|
+
...historical,
|
|
339
|
+
cases: [{ ...historical.cases[0], target: { provider: '', observed_models: ['x'] } }],
|
|
340
|
+
})
|
|
341
|
+
).toThrow('malformed target evidence');
|
|
306
342
|
});
|
|
307
343
|
});
|
|
@@ -9,6 +9,7 @@ import { getGitInfo } from '../provenance/git';
|
|
|
9
9
|
import type {
|
|
10
10
|
CaseResult,
|
|
11
11
|
CostEstimateInfo,
|
|
12
|
+
ExecutionProvenance,
|
|
12
13
|
ManifestRedactionInfo,
|
|
13
14
|
ResolvedConfig,
|
|
14
15
|
RunConfig,
|
|
@@ -26,6 +27,7 @@ export function createRunManifest(options: {
|
|
|
26
27
|
config: RunConfig;
|
|
27
28
|
resolvedConfig?: ResolvedConfig;
|
|
28
29
|
workloadIdentity?: WorkloadIdentity;
|
|
30
|
+
executionProvenance?: ExecutionProvenance;
|
|
29
31
|
cases: CaseResult[];
|
|
30
32
|
startTime: Date;
|
|
31
33
|
endTime: Date;
|
|
@@ -38,6 +40,7 @@ export function createRunManifest(options: {
|
|
|
38
40
|
config,
|
|
39
41
|
resolvedConfig,
|
|
40
42
|
workloadIdentity,
|
|
43
|
+
executionProvenance,
|
|
41
44
|
cases,
|
|
42
45
|
startTime,
|
|
43
46
|
endTime,
|
|
@@ -53,7 +56,7 @@ export function createRunManifest(options: {
|
|
|
53
56
|
const environment = getEnvironmentInfo();
|
|
54
57
|
|
|
55
58
|
return {
|
|
56
|
-
version: '1.
|
|
59
|
+
version: '1.3',
|
|
57
60
|
run_id: nanoid(12),
|
|
58
61
|
project,
|
|
59
62
|
start_time: startTime.toISOString(),
|
|
@@ -62,6 +65,7 @@ export function createRunManifest(options: {
|
|
|
62
65
|
config,
|
|
63
66
|
resolved_config: resolvedConfig,
|
|
64
67
|
workload_identity: workloadIdentity,
|
|
68
|
+
execution_provenance: executionProvenance,
|
|
65
69
|
metrics,
|
|
66
70
|
git,
|
|
67
71
|
provenance: {
|
package/src/artifacts/types.ts
CHANGED
|
@@ -65,6 +65,33 @@ export interface WorkloadIdentity {
|
|
|
65
65
|
rubric: ContentIdentity;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
+
/** Bounded target identity captured from a case execution. */
|
|
69
|
+
export interface CaseTargetEvidence {
|
|
70
|
+
provider: string;
|
|
71
|
+
requested_model?: string;
|
|
72
|
+
/** Model identifiers returned by the target provider during this case. */
|
|
73
|
+
observed_models?: string[];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Requested and observed execution configuration for a complete run. */
|
|
77
|
+
export interface ExecutionProvenance {
|
|
78
|
+
schema_version: '1';
|
|
79
|
+
target: {
|
|
80
|
+
provider: string;
|
|
81
|
+
requested_models?: string[];
|
|
82
|
+
observed_models?: string[];
|
|
83
|
+
generation?: {
|
|
84
|
+
temperature?: number;
|
|
85
|
+
max_tokens?: number;
|
|
86
|
+
seed?: number;
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
/** Judge/evaluator model identities, never combined with target identity. */
|
|
90
|
+
evaluator?: {
|
|
91
|
+
models?: string[];
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
68
95
|
// ============================================================================
|
|
69
96
|
// Case Result Types
|
|
70
97
|
// ============================================================================
|
|
@@ -124,6 +151,8 @@ export interface CaseResult {
|
|
|
124
151
|
error?: string;
|
|
125
152
|
/** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
|
|
126
153
|
evidence?: CaseEvaluationEvidence;
|
|
154
|
+
/** Requested and observed target identity for this case. */
|
|
155
|
+
target?: CaseTargetEvidence;
|
|
127
156
|
/** Redaction information for this case */
|
|
128
157
|
redaction?: CaseRedactionInfo;
|
|
129
158
|
/** Ordered tool activity captured for an enabled tool loop. */
|
|
@@ -302,6 +331,8 @@ export interface RunManifest {
|
|
|
302
331
|
resolved_config?: ResolvedConfig;
|
|
303
332
|
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
304
333
|
workload_identity?: WorkloadIdentity;
|
|
334
|
+
/** Requested and observed target/evaluator configuration for this run. */
|
|
335
|
+
execution_provenance?: ExecutionProvenance;
|
|
305
336
|
metrics: RunMetrics;
|
|
306
337
|
git: GitInfo;
|
|
307
338
|
provenance: ProvenanceInfo;
|
|
@@ -351,6 +382,9 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
|
|
|
351
382
|
if (manifest.workload_identity !== undefined) {
|
|
352
383
|
assertWorkloadIdentity(manifest.workload_identity);
|
|
353
384
|
}
|
|
385
|
+
if (manifest.execution_provenance !== undefined) {
|
|
386
|
+
assertExecutionProvenance(manifest.execution_provenance);
|
|
387
|
+
}
|
|
354
388
|
|
|
355
389
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
356
390
|
if (!isRecord(caseResult)) {
|
|
@@ -370,9 +404,64 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
|
|
|
370
404
|
if (caseResult.evidence !== undefined) {
|
|
371
405
|
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
372
406
|
}
|
|
407
|
+
if (caseResult.target !== undefined) {
|
|
408
|
+
assertCaseTargetEvidence(caseResult.target);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function assertCaseTargetEvidence(target: unknown): void {
|
|
414
|
+
if (
|
|
415
|
+
!isRecord(target) ||
|
|
416
|
+
typeof target.provider !== 'string' ||
|
|
417
|
+
target.provider.length === 0 ||
|
|
418
|
+
target.provider.length > 100 ||
|
|
419
|
+
(target.requested_model !== undefined &&
|
|
420
|
+
(typeof target.requested_model !== 'string' || target.requested_model.length > 200)) ||
|
|
421
|
+
!isBoundedStringList(target.observed_models)
|
|
422
|
+
) {
|
|
423
|
+
throw new Error('Invalid run manifest: malformed target evidence');
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
function assertExecutionProvenance(provenance: unknown): void {
|
|
428
|
+
if (!isRecord(provenance) || provenance.schema_version !== '1' || !isRecord(provenance.target)) {
|
|
429
|
+
throw new Error('Invalid run manifest: malformed execution provenance');
|
|
430
|
+
}
|
|
431
|
+
const target = provenance.target;
|
|
432
|
+
if (
|
|
433
|
+
typeof target.provider !== 'string' ||
|
|
434
|
+
target.provider.length === 0 ||
|
|
435
|
+
target.provider.length > 100 ||
|
|
436
|
+
!isBoundedStringList(target.requested_models) ||
|
|
437
|
+
!isBoundedStringList(target.observed_models) ||
|
|
438
|
+
(target.generation !== undefined && !isGenerationConfig(target.generation))
|
|
439
|
+
) {
|
|
440
|
+
throw new Error('Invalid run manifest: malformed execution provenance');
|
|
441
|
+
}
|
|
442
|
+
if (provenance.evaluator !== undefined) {
|
|
443
|
+
if (!isRecord(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
|
|
444
|
+
throw new Error('Invalid run manifest: malformed execution provenance');
|
|
445
|
+
}
|
|
373
446
|
}
|
|
374
447
|
}
|
|
375
448
|
|
|
449
|
+
function isBoundedStringList(value: unknown): boolean {
|
|
450
|
+
return (
|
|
451
|
+
value === undefined ||
|
|
452
|
+
(Array.isArray(value) &&
|
|
453
|
+
value.length <= 100 &&
|
|
454
|
+
value.every((item) => typeof item === 'string' && item.length > 0 && item.length <= 200))
|
|
455
|
+
);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
function isGenerationConfig(value: unknown): boolean {
|
|
459
|
+
if (!isRecord(value)) return false;
|
|
460
|
+
return [value.temperature, value.max_tokens, value.seed].every(
|
|
461
|
+
(item) => item === undefined || (typeof item === 'number' && Number.isFinite(item))
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
|
|
376
465
|
function assertWorkloadIdentity(identity: unknown): void {
|
|
377
466
|
if (
|
|
378
467
|
!isRecord(identity) ||
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { createExecutionProvenance } from './execution-provenance';
|
|
3
|
+
|
|
4
|
+
describe('createExecutionProvenance', () => {
|
|
5
|
+
test('keeps requested, observed target, and evaluator model identities separate', () => {
|
|
6
|
+
const provenance = createExecutionProvenance({
|
|
7
|
+
provider: 'openai',
|
|
8
|
+
requestedModel: 'gpt-requested',
|
|
9
|
+
temperature: 0.2,
|
|
10
|
+
maxTokens: 100,
|
|
11
|
+
seed: 7,
|
|
12
|
+
cases: [
|
|
13
|
+
{
|
|
14
|
+
id: 'one',
|
|
15
|
+
ok: true,
|
|
16
|
+
score: 1,
|
|
17
|
+
matcherType: 'exact',
|
|
18
|
+
latencyMs: 1,
|
|
19
|
+
tokens: { prompt: 1, completion: 1, total: 2 },
|
|
20
|
+
prompt: 'prompt',
|
|
21
|
+
response: 'response',
|
|
22
|
+
expected: { type: 'exact', value: 'response', caseSensitive: true },
|
|
23
|
+
tags: [],
|
|
24
|
+
target: {
|
|
25
|
+
provider: 'openai',
|
|
26
|
+
requested_model: 'gpt-requested',
|
|
27
|
+
observed_models: ['gpt-observed'],
|
|
28
|
+
},
|
|
29
|
+
evidence: { evaluator: 'llm_grader', model: 'judge-model' },
|
|
30
|
+
},
|
|
31
|
+
],
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
expect(provenance).toEqual({
|
|
35
|
+
schema_version: '1',
|
|
36
|
+
target: {
|
|
37
|
+
provider: 'openai',
|
|
38
|
+
requested_models: ['gpt-requested'],
|
|
39
|
+
observed_models: ['gpt-observed'],
|
|
40
|
+
generation: { temperature: 0.2, max_tokens: 100, seed: 7 },
|
|
41
|
+
},
|
|
42
|
+
evaluator: { models: ['judge-model'] },
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test('records an unavailable observed identity without inventing one', () => {
|
|
47
|
+
const provenance = createExecutionProvenance({
|
|
48
|
+
provider: 'custom',
|
|
49
|
+
requestedModel: 'requested-model',
|
|
50
|
+
cases: [],
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
expect(provenance.target).toEqual({
|
|
54
|
+
provider: 'custom',
|
|
55
|
+
requested_models: ['requested-model'],
|
|
56
|
+
});
|
|
57
|
+
expect(provenance.evaluator).toBeUndefined();
|
|
58
|
+
});
|
|
59
|
+
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/** Bounded execution provenance derived from declared and observed run evidence. */
|
|
2
|
+
|
|
3
|
+
import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
|
|
4
|
+
|
|
5
|
+
export function createExecutionProvenance(options: {
|
|
6
|
+
provider: string;
|
|
7
|
+
requestedModel?: string;
|
|
8
|
+
temperature?: number;
|
|
9
|
+
maxTokens?: number;
|
|
10
|
+
seed?: number;
|
|
11
|
+
cases: CaseResult[];
|
|
12
|
+
}): ExecutionProvenance {
|
|
13
|
+
const requestedModels = uniqueStrings([
|
|
14
|
+
options.requestedModel,
|
|
15
|
+
...options.cases.map((caseResult) => caseResult.target?.requested_model),
|
|
16
|
+
]);
|
|
17
|
+
const observedModels = uniqueStrings(
|
|
18
|
+
options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? [])
|
|
19
|
+
);
|
|
20
|
+
const evaluatorModels = uniqueStrings(
|
|
21
|
+
options.cases.map((caseResult) => caseResult.evidence?.model)
|
|
22
|
+
);
|
|
23
|
+
const generation = omitUndefined({
|
|
24
|
+
temperature: options.temperature,
|
|
25
|
+
max_tokens: options.maxTokens,
|
|
26
|
+
seed: options.seed,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
schema_version: '1',
|
|
31
|
+
target: {
|
|
32
|
+
provider: boundedString(options.provider, 100) ?? 'unknown',
|
|
33
|
+
...(requestedModels.length ? { requested_models: requestedModels } : {}),
|
|
34
|
+
...(observedModels.length ? { observed_models: observedModels } : {}),
|
|
35
|
+
...(Object.keys(generation).length ? { generation } : {}),
|
|
36
|
+
},
|
|
37
|
+
...(evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function uniqueStrings(values: unknown[]): string[] {
|
|
42
|
+
return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))] as string[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function boundedString(value: unknown, maxLength: number): string | undefined {
|
|
46
|
+
return typeof value === 'string' && value.length > 0 ? value.slice(0, maxLength) : undefined;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function omitUndefined<T extends Record<string, number | undefined>>(value: T): Partial<T> {
|
|
50
|
+
return Object.fromEntries(
|
|
51
|
+
Object.entries(value).filter(([, item]) => item !== undefined)
|
|
52
|
+
) as Partial<T>;
|
|
53
|
+
}
|
package/src/provenance/index.ts
CHANGED
|
@@ -170,6 +170,11 @@ describe('executeCase tool loop', () => {
|
|
|
170
170
|
expect(result.error).toBe('TOOL_EXECUTOR_REQUIRED');
|
|
171
171
|
expect(result.latencyMs).toBe(4);
|
|
172
172
|
expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
|
|
173
|
+
expect(result.target).toEqual({
|
|
174
|
+
provider: 'ling',
|
|
175
|
+
requested_model: 'Ling-3.0-flash',
|
|
176
|
+
observed_models: ['Ling-3.0-flash'],
|
|
177
|
+
});
|
|
173
178
|
});
|
|
174
179
|
|
|
175
180
|
it('retains prior generation metrics when a later generation rejects', async () => {
|
|
@@ -201,6 +206,7 @@ describe('executeCase tool loop', () => {
|
|
|
201
206
|
steps: 1,
|
|
202
207
|
terminationReason: 'tool_error',
|
|
203
208
|
});
|
|
209
|
+
expect(result.target?.observed_models).toEqual(['Ling-3.0-flash']);
|
|
204
210
|
});
|
|
205
211
|
|
|
206
212
|
it('retains prior generation metrics when a later generation times out', async () => {
|
|
@@ -304,7 +310,11 @@ describe('executeCase measurement integrity', () => {
|
|
|
304
310
|
status: 'error',
|
|
305
311
|
response: '',
|
|
306
312
|
error: 'provider unavailable',
|
|
313
|
+
target: {
|
|
314
|
+
provider: 'test',
|
|
315
|
+
},
|
|
307
316
|
});
|
|
317
|
+
expect(result.target?.observed_models).toBeUndefined();
|
|
308
318
|
});
|
|
309
319
|
|
|
310
320
|
it('retains only the bounded evidence contract rather than evaluator details', async () => {
|
package/src/runner/executor.ts
CHANGED
|
@@ -100,6 +100,7 @@ export async function executeCase(
|
|
|
100
100
|
): Promise<CaseResult> {
|
|
101
101
|
const { timeout, retries = 0 } = context;
|
|
102
102
|
const caseStartTime = Date.now();
|
|
103
|
+
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
103
104
|
|
|
104
105
|
let lastError: Error | null = null;
|
|
105
106
|
|
|
@@ -135,6 +136,7 @@ export async function executeCase(
|
|
|
135
136
|
expected: testCase.expected,
|
|
136
137
|
tags: testCase.tags,
|
|
137
138
|
error: lastError?.message,
|
|
139
|
+
target: targetEvidence(context.client.provider, requestedModel),
|
|
138
140
|
};
|
|
139
141
|
}
|
|
140
142
|
|
|
@@ -143,7 +145,7 @@ async function executeCaseAttempt(
|
|
|
143
145
|
context: ExecutorContext,
|
|
144
146
|
timeout?: number
|
|
145
147
|
): Promise<CaseResult> {
|
|
146
|
-
const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
|
|
148
|
+
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
147
149
|
|
|
148
150
|
// Merge scenario-level and case-level variables (case overrides scenario)
|
|
149
151
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
@@ -171,7 +173,7 @@ async function executeCaseAttempt(
|
|
|
171
173
|
const generate = () =>
|
|
172
174
|
client.generate({
|
|
173
175
|
prompt: loopPrompt,
|
|
174
|
-
model: testCase.model || scenario.model,
|
|
176
|
+
model: testCase.model || requestedModel || scenario.model,
|
|
175
177
|
temperature: scenario.temperature,
|
|
176
178
|
maxTokens: scenario.maxTokens,
|
|
177
179
|
seed: scenario.seed,
|
|
@@ -182,6 +184,7 @@ async function executeCaseAttempt(
|
|
|
182
184
|
let result = timeout
|
|
183
185
|
? await Promise.race([generatePromise, createTimeout(timeout)])
|
|
184
186
|
: await generatePromise;
|
|
187
|
+
const observedModels = [result.model];
|
|
185
188
|
const generationMetrics = {
|
|
186
189
|
latencyMs: result.latencyMs,
|
|
187
190
|
tokens: { ...result.tokens },
|
|
@@ -198,7 +201,12 @@ async function executeCaseAttempt(
|
|
|
198
201
|
terminationReason: 'tool_error',
|
|
199
202
|
},
|
|
200
203
|
generationMetrics,
|
|
201
|
-
'TOOL_EXECUTOR_REQUIRED'
|
|
204
|
+
'TOOL_EXECUTOR_REQUIRED',
|
|
205
|
+
targetEvidence(
|
|
206
|
+
client.provider,
|
|
207
|
+
testCase.model || requestedModel || scenario.model,
|
|
208
|
+
observedModels
|
|
209
|
+
)
|
|
202
210
|
);
|
|
203
211
|
}
|
|
204
212
|
const executor =
|
|
@@ -225,7 +233,12 @@ async function executeCaseAttempt(
|
|
|
225
233
|
terminationReason: 'duplicate_call',
|
|
226
234
|
},
|
|
227
235
|
generationMetrics,
|
|
228
|
-
'TOOL_DUPLICATE_CALL'
|
|
236
|
+
'TOOL_DUPLICATE_CALL',
|
|
237
|
+
targetEvidence(
|
|
238
|
+
client.provider,
|
|
239
|
+
testCase.model || requestedModel || scenario.model,
|
|
240
|
+
observedModels
|
|
241
|
+
)
|
|
229
242
|
);
|
|
230
243
|
}
|
|
231
244
|
seenCalls.add(fingerprint);
|
|
@@ -263,7 +276,12 @@ async function executeCaseAttempt(
|
|
|
263
276
|
: 'tool_error',
|
|
264
277
|
},
|
|
265
278
|
generationMetrics,
|
|
266
|
-
execution.error?.code ?? 'TOOL_EXECUTION_FAILED'
|
|
279
|
+
execution.error?.code ?? 'TOOL_EXECUTION_FAILED',
|
|
280
|
+
targetEvidence(
|
|
281
|
+
client.provider,
|
|
282
|
+
testCase.model || requestedModel || scenario.model,
|
|
283
|
+
observedModels
|
|
284
|
+
)
|
|
267
285
|
);
|
|
268
286
|
}
|
|
269
287
|
const content = JSON.stringify(execution.result ?? {});
|
|
@@ -281,7 +299,12 @@ async function executeCaseAttempt(
|
|
|
281
299
|
toolTrace,
|
|
282
300
|
{ status: 'error', steps: step + 1, terminationReason: 'timeout' },
|
|
283
301
|
generationMetrics,
|
|
284
|
-
'TOOL_LOOP_TIMEOUT'
|
|
302
|
+
'TOOL_LOOP_TIMEOUT',
|
|
303
|
+
targetEvidence(
|
|
304
|
+
client.provider,
|
|
305
|
+
testCase.model || requestedModel || scenario.model,
|
|
306
|
+
observedModels
|
|
307
|
+
)
|
|
285
308
|
);
|
|
286
309
|
}
|
|
287
310
|
const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
|
|
@@ -298,9 +321,15 @@ async function executeCaseAttempt(
|
|
|
298
321
|
terminationReason: timedOut ? 'timeout' : 'tool_error',
|
|
299
322
|
},
|
|
300
323
|
generationMetrics,
|
|
301
|
-
timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED'
|
|
324
|
+
timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED',
|
|
325
|
+
targetEvidence(
|
|
326
|
+
client.provider,
|
|
327
|
+
testCase.model || requestedModel || scenario.model,
|
|
328
|
+
observedModels
|
|
329
|
+
)
|
|
302
330
|
);
|
|
303
331
|
}
|
|
332
|
+
observedModels.push(result.model);
|
|
304
333
|
generationMetrics.latencyMs += result.latencyMs;
|
|
305
334
|
generationMetrics.tokens.prompt += result.tokens.prompt;
|
|
306
335
|
generationMetrics.tokens.completion += result.tokens.completion;
|
|
@@ -316,7 +345,12 @@ async function executeCaseAttempt(
|
|
|
316
345
|
terminationReason: 'max_steps',
|
|
317
346
|
},
|
|
318
347
|
generationMetrics,
|
|
319
|
-
'TOOL_LOOP_MAX_STEPS'
|
|
348
|
+
'TOOL_LOOP_MAX_STEPS',
|
|
349
|
+
targetEvidence(
|
|
350
|
+
client.provider,
|
|
351
|
+
testCase.model || requestedModel || scenario.model,
|
|
352
|
+
observedModels
|
|
353
|
+
)
|
|
320
354
|
);
|
|
321
355
|
}
|
|
322
356
|
toolLoop = { status: 'completed', steps: toolTrace.length, terminationReason: 'completed' };
|
|
@@ -435,17 +469,44 @@ async function executeCaseAttempt(
|
|
|
435
469
|
tags: testCase.tags,
|
|
436
470
|
redaction: redactionInfo,
|
|
437
471
|
evidence: finalEvidence,
|
|
472
|
+
target: targetEvidence(
|
|
473
|
+
client.provider,
|
|
474
|
+
testCase.model || requestedModel || scenario.model,
|
|
475
|
+
observedModels
|
|
476
|
+
),
|
|
438
477
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
439
478
|
toolLoop,
|
|
440
479
|
};
|
|
441
480
|
}
|
|
442
481
|
|
|
482
|
+
function targetEvidence(
|
|
483
|
+
provider: string,
|
|
484
|
+
requestedModel?: string,
|
|
485
|
+
observedModels?: unknown[]
|
|
486
|
+
): NonNullable<CaseResult['target']> {
|
|
487
|
+
const observed = [
|
|
488
|
+
...new Set(
|
|
489
|
+
(observedModels ?? []).filter(
|
|
490
|
+
(model): model is string => typeof model === 'string' && model.length > 0
|
|
491
|
+
)
|
|
492
|
+
),
|
|
493
|
+
]
|
|
494
|
+
.map((model) => sanitizeArtifactText(model, 200))
|
|
495
|
+
.filter((model): model is string => Boolean(model));
|
|
496
|
+
return {
|
|
497
|
+
provider: sanitizeArtifactText(provider, 100) ?? 'unknown',
|
|
498
|
+
...(requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {}),
|
|
499
|
+
...(observed.length ? { observed_models: observed } : {}),
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
|
|
443
503
|
function createToolLoopError(
|
|
444
504
|
testCase: TestCase,
|
|
445
505
|
toolTrace: ToolTraceEntry[],
|
|
446
506
|
toolLoop: ToolLoopSummary,
|
|
447
507
|
generationMetrics: Pick<CaseResult, 'latencyMs' | 'tokens'>,
|
|
448
|
-
code: string
|
|
508
|
+
code: string,
|
|
509
|
+
target?: CaseResult['target']
|
|
449
510
|
): ToolLoopError {
|
|
450
511
|
return new ToolLoopError(code, {
|
|
451
512
|
id: testCase.id,
|
|
@@ -462,6 +523,7 @@ function createToolLoopError(
|
|
|
462
523
|
expected: testCase.expected,
|
|
463
524
|
tags: testCase.tags,
|
|
464
525
|
error: code,
|
|
526
|
+
target,
|
|
465
527
|
toolTrace,
|
|
466
528
|
toolLoop,
|
|
467
529
|
});
|
|
@@ -96,6 +96,13 @@ describe('release validation: fixture-backed workflow cases', () => {
|
|
|
96
96
|
workload: { algorithm: 'sha256' },
|
|
97
97
|
rubric: { algorithm: 'sha256' },
|
|
98
98
|
});
|
|
99
|
+
expect(result.manifest.execution_provenance).toMatchObject({
|
|
100
|
+
schema_version: '1',
|
|
101
|
+
target: {
|
|
102
|
+
provider: 'fixture',
|
|
103
|
+
observed_models: ['fixture-model'],
|
|
104
|
+
},
|
|
105
|
+
});
|
|
99
106
|
});
|
|
100
107
|
|
|
101
108
|
test('RV-02 retains independent logistics tool evidence through a fixture-backed workflow', async () => {
|
|
@@ -171,4 +178,33 @@ describe('release validation: fixture-backed workflow cases', () => {
|
|
|
171
178
|
toolTrace: [{ toolCall: { id: 'capacity-1' }, result: { available: true } }],
|
|
172
179
|
});
|
|
173
180
|
});
|
|
181
|
+
|
|
182
|
+
test('records a CLI-resolved model for each case when the scenario does not declare one', async () => {
|
|
183
|
+
const scenario = ScenarioSchema.parse({
|
|
184
|
+
name: 'resolved model evidence',
|
|
185
|
+
cases: [
|
|
186
|
+
{
|
|
187
|
+
id: 'cli-model-override',
|
|
188
|
+
prompt: 'Respond with assured',
|
|
189
|
+
expected: { type: 'exact', value: 'assured' },
|
|
190
|
+
},
|
|
191
|
+
],
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
const result = await runScenario({
|
|
195
|
+
scenario,
|
|
196
|
+
client: fixtureClient(['assured']),
|
|
197
|
+
resolvedConfig: {
|
|
198
|
+
provider: 'fixture',
|
|
199
|
+
model: 'fixture-cli-model',
|
|
200
|
+
source: { provider: 'cli', model: 'cli' },
|
|
201
|
+
},
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
expect(result.cases[0].target).toEqual({
|
|
205
|
+
provider: 'fixture',
|
|
206
|
+
requested_model: 'fixture-cli-model',
|
|
207
|
+
observed_models: ['fixture-model'],
|
|
208
|
+
});
|
|
209
|
+
});
|
|
174
210
|
});
|
package/src/runner/runner.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { createRunManifest } from '../artifacts/manifest';
|
|
6
6
|
import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
|
|
7
|
-
import { createWorkloadIdentity } from '../provenance';
|
|
7
|
+
import { createExecutionProvenance, createWorkloadIdentity } from '../provenance';
|
|
8
8
|
import { Redactor } from '../redaction';
|
|
9
9
|
import { executeCase } from './executor';
|
|
10
10
|
import type { RunOptions, RunResult } from './types';
|
|
@@ -51,6 +51,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
|
|
|
51
51
|
const result = await executeCase(testCase, {
|
|
52
52
|
client,
|
|
53
53
|
scenario,
|
|
54
|
+
requestedModel: resolvedConfig?.model,
|
|
54
55
|
timeout: testCase.timeout || timeout,
|
|
55
56
|
retries: testCase.retries ?? retries,
|
|
56
57
|
redaction,
|
|
@@ -70,6 +71,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
|
|
|
70
71
|
const result = await executeCase(testCase, {
|
|
71
72
|
client,
|
|
72
73
|
scenario,
|
|
74
|
+
requestedModel: resolvedConfig?.model,
|
|
73
75
|
timeout: testCase.timeout || timeout,
|
|
74
76
|
retries: testCase.retries ?? retries,
|
|
75
77
|
redaction,
|
|
@@ -122,6 +124,14 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
|
|
|
122
124
|
},
|
|
123
125
|
resolvedConfig,
|
|
124
126
|
workloadIdentity: createWorkloadIdentity(scenario),
|
|
127
|
+
executionProvenance: createExecutionProvenance({
|
|
128
|
+
provider: client.provider,
|
|
129
|
+
requestedModel: resolvedConfig?.model || scenario.model,
|
|
130
|
+
temperature: resolvedConfig?.temperature ?? scenario.temperature,
|
|
131
|
+
maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
|
|
132
|
+
seed: scenario.seed,
|
|
133
|
+
cases: results,
|
|
134
|
+
}),
|
|
125
135
|
cases: results,
|
|
126
136
|
startTime,
|
|
127
137
|
endTime,
|
package/src/runner/types.ts
CHANGED
|
@@ -56,6 +56,8 @@ export interface RunResult {
|
|
|
56
56
|
export interface ExecutorContext {
|
|
57
57
|
client: ModelClient;
|
|
58
58
|
scenario: Scenario;
|
|
59
|
+
/** Effective model selected outside the scenario, such as a CLI override. */
|
|
60
|
+
requestedModel?: string;
|
|
59
61
|
timeout?: number;
|
|
60
62
|
retries?: number;
|
|
61
63
|
/** Redaction configuration for this execution */
|