@artemiskit/core 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # @artemiskit/core
2
2
 
3
+ ## 0.5.1
4
+
5
+ ### Patch Changes
6
+
7
+ - Preserve bounded requested and observed target-model evidence per case and per run, separately from
8
+ evaluator identities. Normal manifests now use v1.3 execution provenance while retaining readable
9
+ historical artifacts. CI JSON output is machine-readable without progress text.
10
+
3
11
  ## 0.5.0
4
12
 
5
13
  ### Minor Changes
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Run manifest generation utilities
3
3
  */
4
- import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
4
+ import type { CaseResult, ExecutionProvenance, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
5
5
  /**
6
6
  * Create a new run manifest
7
7
  */
@@ -10,6 +10,7 @@ export declare function createRunManifest(options: {
10
10
  config: RunConfig;
11
11
  resolvedConfig?: ResolvedConfig;
12
12
  workloadIdentity?: WorkloadIdentity;
13
+ executionProvenance?: ExecutionProvenance;
13
14
  cases: CaseResult[];
14
15
  startTime: Date;
15
16
  endTime: Date;
@@ -1 +1 @@
1
- {"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CAyCd"}
1
+ {"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CA2Cd"}
@@ -52,6 +52,31 @@ export interface WorkloadIdentity {
52
52
  workload: ContentIdentity;
53
53
  rubric: ContentIdentity;
54
54
  }
55
+ /** Bounded target identity captured from a case execution. */
56
+ export interface CaseTargetEvidence {
57
+ provider: string;
58
+ requested_model?: string;
59
+ /** Model identifiers returned by the target provider during this case. */
60
+ observed_models?: string[];
61
+ }
62
+ /** Requested and observed execution configuration for a complete run. */
63
+ export interface ExecutionProvenance {
64
+ schema_version: '1';
65
+ target: {
66
+ provider: string;
67
+ requested_models?: string[];
68
+ observed_models?: string[];
69
+ generation?: {
70
+ temperature?: number;
71
+ max_tokens?: number;
72
+ seed?: number;
73
+ };
74
+ };
75
+ /** Judge/evaluator model identities, never combined with target identity. */
76
+ evaluator?: {
77
+ models?: string[];
78
+ };
79
+ }
55
80
  /**
56
81
  * Terminal status of a case measurement.
57
82
  *
@@ -99,6 +124,8 @@ export interface CaseResult {
99
124
  error?: string;
100
125
  /** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
101
126
  evidence?: CaseEvaluationEvidence;
127
+ /** Requested and observed target identity for this case. */
128
+ target?: CaseTargetEvidence;
102
129
  /** Redaction information for this case */
103
130
  redaction?: CaseRedactionInfo;
104
131
  /** Ordered tool activity captured for an enabled tool loop. */
@@ -256,6 +283,8 @@ export interface RunManifest {
256
283
  resolved_config?: ResolvedConfig;
257
284
  /** Versioned identities for the declared workload and evaluation rubric. */
258
285
  workload_identity?: WorkloadIdentity;
286
+ /** Requested and observed target/evaluator configuration for this run. */
287
+ execution_provenance?: ExecutionProvenance;
259
288
  metrics: RunMetrics;
260
289
  git: GitInfo;
261
290
  provenance: ProvenanceInfo;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CA4B7F;AAqED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,8DAA8D;AAC9D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0EAA0E;IAC1E,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,yEAAyE;AACzE,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,GAAG,CAAC;IACpB,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,UAAU,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,IAAI,CAAC,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC;IACF,6EAA6E;IAC7E,SAAS,CAAC,EAAE;QACV,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,4DAA4D;IAC5D,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,0EAA0E;IAC1E,oBAAoB,CAAC,EAAE,mBAAmB,CAAC;IAC3C,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAkC7F;AAyHD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
package/dist/index.js CHANGED
@@ -20525,6 +20525,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
20525
20525
  async function executeCase(testCase, context) {
20526
20526
  const { timeout, retries = 0 } = context;
20527
20527
  const caseStartTime = Date.now();
20528
+ const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
20528
20529
  let lastError = null;
20529
20530
  for (let attempt = 0;attempt <= retries; attempt++) {
20530
20531
  try {
@@ -20555,11 +20556,12 @@ async function executeCase(testCase, context) {
20555
20556
  response: "",
20556
20557
  expected: testCase.expected,
20557
20558
  tags: testCase.tags,
20558
- error: lastError?.message
20559
+ error: lastError?.message,
20560
+ target: targetEvidence(context.client.provider, requestedModel)
20559
20561
  };
20560
20562
  }
20561
20563
  async function executeCaseAttempt(testCase, context, timeout) {
20562
- const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
20564
+ const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
20563
20565
  const variables = mergeVariables(scenario.variables, testCase.variables);
20564
20566
  let prompt = substituteVariables(testCase.prompt, variables);
20565
20567
  if (scenario.setup?.systemPrompt && typeof prompt === "string") {
@@ -20577,7 +20579,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20577
20579
  const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
20578
20580
  const generate = () => client.generate({
20579
20581
  prompt: loopPrompt,
20580
- model: testCase.model || scenario.model,
20582
+ model: testCase.model || requestedModel || scenario.model,
20581
20583
  temperature: scenario.temperature,
20582
20584
  maxTokens: scenario.maxTokens,
20583
20585
  seed: scenario.seed,
@@ -20585,6 +20587,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20585
20587
  });
20586
20588
  const generatePromise = generate();
20587
20589
  let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
20590
+ const observedModels = [result.model];
20588
20591
  const generationMetrics = {
20589
20592
  latencyMs: result.latencyMs,
20590
20593
  tokens: { ...result.tokens }
@@ -20595,7 +20598,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20595
20598
  status: "error",
20596
20599
  steps: 0,
20597
20600
  terminationReason: "tool_error"
20598
- }, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
20601
+ }, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
20599
20602
  }
20600
20603
  const executor = toolExecutor ?? new FixtureToolExecutor({
20601
20604
  tools,
@@ -20614,7 +20617,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20614
20617
  status: "error",
20615
20618
  steps: step,
20616
20619
  terminationReason: "duplicate_call"
20617
- }, generationMetrics, "TOOL_DUPLICATE_CALL");
20620
+ }, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
20618
20621
  }
20619
20622
  seenCalls.add(fingerprint);
20620
20623
  const toolStart = Date.now();
@@ -20641,7 +20644,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20641
20644
  status: "error",
20642
20645
  steps: step + 1,
20643
20646
  terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
20644
- }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
20647
+ }, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
20645
20648
  }
20646
20649
  const content = JSON.stringify(execution.result ?? {});
20647
20650
  loopPrompt.push({
@@ -20653,7 +20656,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20653
20656
  }
20654
20657
  const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
20655
20658
  if (remainingLoopTime <= 0) {
20656
- throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
20659
+ throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
20657
20660
  }
20658
20661
  const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
20659
20662
  try {
@@ -20664,8 +20667,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
20664
20667
  status: "error",
20665
20668
  steps: step + 1,
20666
20669
  terminationReason: timedOut ? "timeout" : "tool_error"
20667
- }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
20670
+ }, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
20668
20671
  }
20672
+ observedModels.push(result.model);
20669
20673
  generationMetrics.latencyMs += result.latencyMs;
20670
20674
  generationMetrics.tokens.prompt += result.tokens.prompt;
20671
20675
  generationMetrics.tokens.completion += result.tokens.completion;
@@ -20676,7 +20680,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
20676
20680
  status: "error",
20677
20681
  steps: policy.maxSteps,
20678
20682
  terminationReason: "max_steps"
20679
- }, generationMetrics, "TOOL_LOOP_MAX_STEPS");
20683
+ }, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
20680
20684
  }
20681
20685
  toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
20682
20686
  }
@@ -20772,11 +20776,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
20772
20776
  tags: testCase.tags,
20773
20777
  redaction: redactionInfo,
20774
20778
  evidence: finalEvidence,
20779
+ target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
20775
20780
  toolTrace: toolTrace.length ? toolTrace : undefined,
20776
20781
  toolLoop
20777
20782
  };
20778
20783
  }
20779
- function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code) {
20784
+ function targetEvidence(provider, requestedModel, observedModels) {
20785
+ const observed = [
20786
+ ...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
20787
+ ].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
20788
+ return {
20789
+ provider: sanitizeArtifactText(provider, 100) ?? "unknown",
20790
+ ...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
20791
+ ...observed.length ? { observed_models: observed } : {}
20792
+ };
20793
+ }
20794
+ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
20780
20795
  return new ToolLoopError(code, {
20781
20796
  id: testCase.id,
20782
20797
  name: testCase.name,
@@ -20792,6 +20807,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
20792
20807
  expected: testCase.expected,
20793
20808
  tags: testCase.tags,
20794
20809
  error: code,
20810
+ target,
20795
20811
  toolTrace,
20796
20812
  toolLoop
20797
20813
  });
@@ -21295,6 +21311,9 @@ function assertRunManifestIntegrity(manifest) {
21295
21311
  if (manifest.workload_identity !== undefined) {
21296
21312
  assertWorkloadIdentity(manifest.workload_identity);
21297
21313
  }
21314
+ if (manifest.execution_provenance !== undefined) {
21315
+ assertExecutionProvenance(manifest.execution_provenance);
21316
+ }
21298
21317
  for (const [index, caseResult] of manifest.cases.entries()) {
21299
21318
  if (!isRecord2(caseResult)) {
21300
21319
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -21305,8 +21324,38 @@ function assertRunManifestIntegrity(manifest) {
21305
21324
  if (caseResult.evidence !== undefined) {
21306
21325
  assertCaseEvaluationEvidence(caseResult.evidence, index);
21307
21326
  }
21327
+ if (caseResult.target !== undefined) {
21328
+ assertCaseTargetEvidence(caseResult.target);
21329
+ }
21330
+ }
21331
+ }
21332
+ function assertCaseTargetEvidence(target) {
21333
+ if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
21334
+ throw new Error("Invalid run manifest: malformed target evidence");
21335
+ }
21336
+ }
21337
+ function assertExecutionProvenance(provenance) {
21338
+ if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
21339
+ throw new Error("Invalid run manifest: malformed execution provenance");
21340
+ }
21341
+ const target = provenance.target;
21342
+ if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
21343
+ throw new Error("Invalid run manifest: malformed execution provenance");
21344
+ }
21345
+ if (provenance.evaluator !== undefined) {
21346
+ if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
21347
+ throw new Error("Invalid run manifest: malformed execution provenance");
21348
+ }
21308
21349
  }
21309
21350
  }
21351
+ function isBoundedStringList(value) {
21352
+ return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
21353
+ }
21354
+ function isGenerationConfig(value) {
21355
+ if (!isRecord2(value))
21356
+ return false;
21357
+ return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
21358
+ }
21310
21359
  function assertWorkloadIdentity(identity) {
21311
21360
  if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
21312
21361
  throw new Error("Invalid run manifest: malformed workload identity");
@@ -21357,6 +21406,7 @@ function createRunManifest(options) {
21357
21406
  config,
21358
21407
  resolvedConfig,
21359
21408
  workloadIdentity,
21409
+ executionProvenance,
21360
21410
  cases,
21361
21411
  startTime,
21362
21412
  endTime,
@@ -21369,7 +21419,7 @@ function createRunManifest(options) {
21369
21419
  const git = getGitInfo();
21370
21420
  const environment = getEnvironmentInfo();
21371
21421
  return {
21372
- version: "1.2",
21422
+ version: "1.3",
21373
21423
  run_id: nanoid(12),
21374
21424
  project,
21375
21425
  start_time: startTime.toISOString(),
@@ -21378,6 +21428,7 @@ function createRunManifest(options) {
21378
21428
  config,
21379
21429
  resolved_config: resolvedConfig,
21380
21430
  workload_identity: workloadIdentity,
21431
+ execution_provenance: executionProvenance,
21381
21432
  metrics,
21382
21433
  git,
21383
21434
  provenance: {
@@ -21530,6 +21581,39 @@ function canonicalize(value, patterns, key) {
21530
21581
  }
21531
21582
  return value;
21532
21583
  }
21584
+ // src/provenance/execution-provenance.ts
21585
+ function createExecutionProvenance(options) {
21586
+ const requestedModels = uniqueStrings([
21587
+ options.requestedModel,
21588
+ ...options.cases.map((caseResult) => caseResult.target?.requested_model)
21589
+ ]);
21590
+ const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
21591
+ const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
21592
+ const generation = omitUndefined({
21593
+ temperature: options.temperature,
21594
+ max_tokens: options.maxTokens,
21595
+ seed: options.seed
21596
+ });
21597
+ return {
21598
+ schema_version: "1",
21599
+ target: {
21600
+ provider: boundedString(options.provider, 100) ?? "unknown",
21601
+ ...requestedModels.length ? { requested_models: requestedModels } : {},
21602
+ ...observedModels.length ? { observed_models: observedModels } : {},
21603
+ ...Object.keys(generation).length ? { generation } : {}
21604
+ },
21605
+ ...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
21606
+ };
21607
+ }
21608
+ function uniqueStrings(values) {
21609
+ return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
21610
+ }
21611
+ function boundedString(value, maxLength) {
21612
+ return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
21613
+ }
21614
+ function omitUndefined(value) {
21615
+ return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
21616
+ }
21533
21617
  // src/runner/runner.ts
21534
21618
  async function runScenario(options) {
21535
21619
  const {
@@ -21563,6 +21647,7 @@ async function runScenario(options) {
21563
21647
  const result = await executeCase(testCase, {
21564
21648
  client,
21565
21649
  scenario,
21650
+ requestedModel: resolvedConfig?.model,
21566
21651
  timeout: testCase.timeout || timeout,
21567
21652
  retries: testCase.retries ?? retries,
21568
21653
  redaction,
@@ -21579,6 +21664,7 @@ async function runScenario(options) {
21579
21664
  const result = await executeCase(testCase, {
21580
21665
  client,
21581
21666
  scenario,
21667
+ requestedModel: resolvedConfig?.model,
21582
21668
  timeout: testCase.timeout || timeout,
21583
21669
  retries: testCase.retries ?? retries,
21584
21670
  redaction,
@@ -21623,6 +21709,14 @@ async function runScenario(options) {
21623
21709
  },
21624
21710
  resolvedConfig,
21625
21711
  workloadIdentity: createWorkloadIdentity(scenario),
21712
+ executionProvenance: createExecutionProvenance({
21713
+ provider: client.provider,
21714
+ requestedModel: resolvedConfig?.model || scenario.model,
21715
+ temperature: resolvedConfig?.temperature ?? scenario.temperature,
21716
+ maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
21717
+ seed: scenario.seed,
21718
+ cases: results
21719
+ }),
21626
21720
  cases: results,
21627
21721
  startTime,
21628
21722
  endTime,
@@ -32705,6 +32799,7 @@ export {
32705
32799
  assertRunManifestIntegrity,
32706
32800
  createAdapter,
32707
32801
  createDefaultRedactor,
32802
+ createExecutionProvenance,
32708
32803
  createNoOpRedactor,
32709
32804
  createRedactionOptions,
32710
32805
  createRunManifest,
@@ -0,0 +1,11 @@
1
+ /** Bounded execution provenance derived from declared and observed run evidence. */
2
+ import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
3
+ export declare function createExecutionProvenance(options: {
4
+ provider: string;
5
+ requestedModel?: string;
6
+ temperature?: number;
7
+ maxTokens?: number;
8
+ seed?: number;
9
+ cases: CaseResult[];
10
+ }): ExecutionProvenance;
11
+ //# sourceMappingURL=execution-provenance.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"execution-provenance.d.ts","sourceRoot":"","sources":["../../src/provenance/execution-provenance.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAEpF,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,GAAG,mBAAmB,CA2BtB"}
@@ -4,4 +4,5 @@
4
4
  export { getGitInfo } from './git';
5
5
  export { getEnvironmentInfo } from './environment';
6
6
  export { createWorkloadIdentity } from './workload-identity';
7
+ export { createExecutionProvenance } from './execution-provenance';
7
8
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAuCrB"}
1
+ {"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAyCrB"}
@@ -1 +1 @@
1
- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CA4HzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
1
+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAsIzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
@@ -52,6 +52,8 @@ export interface RunResult {
52
52
  export interface ExecutorContext {
53
53
  client: ModelClient;
54
54
  scenario: Scenario;
55
+ /** Effective model selected outside the scenario, such as a CLI override. */
56
+ requestedModel?: string;
55
57
  timeout?: number;
56
58
  retries?: number;
57
59
  /** Redaction configuration for this execution */
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/core",
3
- "version": "0.5.0",
3
+ "version": "0.5.1",
4
4
  "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -55,7 +55,7 @@ describe('createRunManifest', () => {
55
55
  endTime,
56
56
  });
57
57
 
58
- expect(manifest.version).toBe('1.2');
58
+ expect(manifest.version).toBe('1.3');
59
59
  expect(manifest.project).toBe('test-project');
60
60
  expect(manifest.run_id).toBeTruthy();
61
61
  expect(manifest.run_id.length).toBe(12);
@@ -189,6 +189,30 @@ describe('createRunManifest', () => {
189
189
  expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
190
190
  });
191
191
 
192
+ test('retains target and evaluator execution provenance separately', () => {
193
+ const manifest = createRunManifest({
194
+ project: 'test-project',
195
+ config: { scenario: 'test-scenario', provider: 'openai' },
196
+ executionProvenance: {
197
+ schema_version: '1',
198
+ target: {
199
+ provider: 'openai',
200
+ requested_models: ['gpt-requested'],
201
+ observed_models: ['gpt-observed'],
202
+ generation: { temperature: 0, max_tokens: 100, seed: 42 },
203
+ },
204
+ evaluator: { models: ['judge-model'] },
205
+ },
206
+ cases: mockCases,
207
+ startTime: new Date(),
208
+ endTime: new Date(),
209
+ });
210
+
211
+ expect(manifest.execution_provenance?.target.observed_models).toEqual(['gpt-observed']);
212
+ expect(manifest.execution_provenance?.evaluator?.models).toEqual(['judge-model']);
213
+ expect(() => assertRunManifestIntegrity(manifest)).not.toThrow();
214
+ });
215
+
192
216
  test('includes provenance information', () => {
193
217
  const manifest = createRunManifest({
194
218
  project: 'test-project',
@@ -303,5 +327,17 @@ describe('createRunManifest', () => {
303
327
  expect(() =>
304
328
  assertRunManifestIntegrity({ ...historical, workload_identity: { schema_version: '1' } })
305
329
  ).toThrow('malformed workload identity');
330
+ expect(() =>
331
+ assertRunManifestIntegrity({
332
+ ...historical,
333
+ execution_provenance: { schema_version: '1', target: { provider: '' } },
334
+ })
335
+ ).toThrow('malformed execution provenance');
336
+ expect(() =>
337
+ assertRunManifestIntegrity({
338
+ ...historical,
339
+ cases: [{ ...historical.cases[0], target: { provider: '', observed_models: ['x'] } }],
340
+ })
341
+ ).toThrow('malformed target evidence');
306
342
  });
307
343
  });
@@ -9,6 +9,7 @@ import { getGitInfo } from '../provenance/git';
9
9
  import type {
10
10
  CaseResult,
11
11
  CostEstimateInfo,
12
+ ExecutionProvenance,
12
13
  ManifestRedactionInfo,
13
14
  ResolvedConfig,
14
15
  RunConfig,
@@ -26,6 +27,7 @@ export function createRunManifest(options: {
26
27
  config: RunConfig;
27
28
  resolvedConfig?: ResolvedConfig;
28
29
  workloadIdentity?: WorkloadIdentity;
30
+ executionProvenance?: ExecutionProvenance;
29
31
  cases: CaseResult[];
30
32
  startTime: Date;
31
33
  endTime: Date;
@@ -38,6 +40,7 @@ export function createRunManifest(options: {
38
40
  config,
39
41
  resolvedConfig,
40
42
  workloadIdentity,
43
+ executionProvenance,
41
44
  cases,
42
45
  startTime,
43
46
  endTime,
@@ -53,7 +56,7 @@ export function createRunManifest(options: {
53
56
  const environment = getEnvironmentInfo();
54
57
 
55
58
  return {
56
- version: '1.2',
59
+ version: '1.3',
57
60
  run_id: nanoid(12),
58
61
  project,
59
62
  start_time: startTime.toISOString(),
@@ -62,6 +65,7 @@ export function createRunManifest(options: {
62
65
  config,
63
66
  resolved_config: resolvedConfig,
64
67
  workload_identity: workloadIdentity,
68
+ execution_provenance: executionProvenance,
65
69
  metrics,
66
70
  git,
67
71
  provenance: {
@@ -65,6 +65,33 @@ export interface WorkloadIdentity {
65
65
  rubric: ContentIdentity;
66
66
  }
67
67
 
68
+ /** Bounded target identity captured from a case execution. */
69
+ export interface CaseTargetEvidence {
70
+ provider: string;
71
+ requested_model?: string;
72
+ /** Model identifiers returned by the target provider during this case. */
73
+ observed_models?: string[];
74
+ }
75
+
76
+ /** Requested and observed execution configuration for a complete run. */
77
+ export interface ExecutionProvenance {
78
+ schema_version: '1';
79
+ target: {
80
+ provider: string;
81
+ requested_models?: string[];
82
+ observed_models?: string[];
83
+ generation?: {
84
+ temperature?: number;
85
+ max_tokens?: number;
86
+ seed?: number;
87
+ };
88
+ };
89
+ /** Judge/evaluator model identities, never combined with target identity. */
90
+ evaluator?: {
91
+ models?: string[];
92
+ };
93
+ }
94
+
68
95
  // ============================================================================
69
96
  // Case Result Types
70
97
  // ============================================================================
@@ -124,6 +151,8 @@ export interface CaseResult {
124
151
  error?: string;
125
152
  /** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
126
153
  evidence?: CaseEvaluationEvidence;
154
+ /** Requested and observed target identity for this case. */
155
+ target?: CaseTargetEvidence;
127
156
  /** Redaction information for this case */
128
157
  redaction?: CaseRedactionInfo;
129
158
  /** Ordered tool activity captured for an enabled tool loop. */
@@ -302,6 +331,8 @@ export interface RunManifest {
302
331
  resolved_config?: ResolvedConfig;
303
332
  /** Versioned identities for the declared workload and evaluation rubric. */
304
333
  workload_identity?: WorkloadIdentity;
334
+ /** Requested and observed target/evaluator configuration for this run. */
335
+ execution_provenance?: ExecutionProvenance;
305
336
  metrics: RunMetrics;
306
337
  git: GitInfo;
307
338
  provenance: ProvenanceInfo;
@@ -351,6 +382,9 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
351
382
  if (manifest.workload_identity !== undefined) {
352
383
  assertWorkloadIdentity(manifest.workload_identity);
353
384
  }
385
+ if (manifest.execution_provenance !== undefined) {
386
+ assertExecutionProvenance(manifest.execution_provenance);
387
+ }
354
388
 
355
389
  for (const [index, caseResult] of manifest.cases.entries()) {
356
390
  if (!isRecord(caseResult)) {
@@ -370,9 +404,64 @@ export function assertRunManifestIntegrity(manifest: unknown): asserts manifest
370
404
  if (caseResult.evidence !== undefined) {
371
405
  assertCaseEvaluationEvidence(caseResult.evidence, index);
372
406
  }
407
+ if (caseResult.target !== undefined) {
408
+ assertCaseTargetEvidence(caseResult.target);
409
+ }
410
+ }
411
+ }
412
+
413
+ function assertCaseTargetEvidence(target: unknown): void {
414
+ if (
415
+ !isRecord(target) ||
416
+ typeof target.provider !== 'string' ||
417
+ target.provider.length === 0 ||
418
+ target.provider.length > 100 ||
419
+ (target.requested_model !== undefined &&
420
+ (typeof target.requested_model !== 'string' || target.requested_model.length > 200)) ||
421
+ !isBoundedStringList(target.observed_models)
422
+ ) {
423
+ throw new Error('Invalid run manifest: malformed target evidence');
424
+ }
425
+ }
426
+
427
+ function assertExecutionProvenance(provenance: unknown): void {
428
+ if (!isRecord(provenance) || provenance.schema_version !== '1' || !isRecord(provenance.target)) {
429
+ throw new Error('Invalid run manifest: malformed execution provenance');
430
+ }
431
+ const target = provenance.target;
432
+ if (
433
+ typeof target.provider !== 'string' ||
434
+ target.provider.length === 0 ||
435
+ target.provider.length > 100 ||
436
+ !isBoundedStringList(target.requested_models) ||
437
+ !isBoundedStringList(target.observed_models) ||
438
+ (target.generation !== undefined && !isGenerationConfig(target.generation))
439
+ ) {
440
+ throw new Error('Invalid run manifest: malformed execution provenance');
441
+ }
442
+ if (provenance.evaluator !== undefined) {
443
+ if (!isRecord(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
444
+ throw new Error('Invalid run manifest: malformed execution provenance');
445
+ }
373
446
  }
374
447
  }
375
448
 
449
+ function isBoundedStringList(value: unknown): boolean {
450
+ return (
451
+ value === undefined ||
452
+ (Array.isArray(value) &&
453
+ value.length <= 100 &&
454
+ value.every((item) => typeof item === 'string' && item.length > 0 && item.length <= 200))
455
+ );
456
+ }
457
+
458
+ function isGenerationConfig(value: unknown): boolean {
459
+ if (!isRecord(value)) return false;
460
+ return [value.temperature, value.max_tokens, value.seed].every(
461
+ (item) => item === undefined || (typeof item === 'number' && Number.isFinite(item))
462
+ );
463
+ }
464
+
376
465
  function assertWorkloadIdentity(identity: unknown): void {
377
466
  if (
378
467
  !isRecord(identity) ||
@@ -0,0 +1,59 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import { createExecutionProvenance } from './execution-provenance';
3
+
4
+ describe('createExecutionProvenance', () => {
5
+ test('keeps requested, observed target, and evaluator model identities separate', () => {
6
+ const provenance = createExecutionProvenance({
7
+ provider: 'openai',
8
+ requestedModel: 'gpt-requested',
9
+ temperature: 0.2,
10
+ maxTokens: 100,
11
+ seed: 7,
12
+ cases: [
13
+ {
14
+ id: 'one',
15
+ ok: true,
16
+ score: 1,
17
+ matcherType: 'exact',
18
+ latencyMs: 1,
19
+ tokens: { prompt: 1, completion: 1, total: 2 },
20
+ prompt: 'prompt',
21
+ response: 'response',
22
+ expected: { type: 'exact', value: 'response', caseSensitive: true },
23
+ tags: [],
24
+ target: {
25
+ provider: 'openai',
26
+ requested_model: 'gpt-requested',
27
+ observed_models: ['gpt-observed'],
28
+ },
29
+ evidence: { evaluator: 'llm_grader', model: 'judge-model' },
30
+ },
31
+ ],
32
+ });
33
+
34
+ expect(provenance).toEqual({
35
+ schema_version: '1',
36
+ target: {
37
+ provider: 'openai',
38
+ requested_models: ['gpt-requested'],
39
+ observed_models: ['gpt-observed'],
40
+ generation: { temperature: 0.2, max_tokens: 100, seed: 7 },
41
+ },
42
+ evaluator: { models: ['judge-model'] },
43
+ });
44
+ });
45
+
46
+ test('records an unavailable observed identity without inventing one', () => {
47
+ const provenance = createExecutionProvenance({
48
+ provider: 'custom',
49
+ requestedModel: 'requested-model',
50
+ cases: [],
51
+ });
52
+
53
+ expect(provenance.target).toEqual({
54
+ provider: 'custom',
55
+ requested_models: ['requested-model'],
56
+ });
57
+ expect(provenance.evaluator).toBeUndefined();
58
+ });
59
+ });
@@ -0,0 +1,53 @@
1
+ /** Bounded execution provenance derived from declared and observed run evidence. */
2
+
3
+ import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
4
+
5
+ export function createExecutionProvenance(options: {
6
+ provider: string;
7
+ requestedModel?: string;
8
+ temperature?: number;
9
+ maxTokens?: number;
10
+ seed?: number;
11
+ cases: CaseResult[];
12
+ }): ExecutionProvenance {
13
+ const requestedModels = uniqueStrings([
14
+ options.requestedModel,
15
+ ...options.cases.map((caseResult) => caseResult.target?.requested_model),
16
+ ]);
17
+ const observedModels = uniqueStrings(
18
+ options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? [])
19
+ );
20
+ const evaluatorModels = uniqueStrings(
21
+ options.cases.map((caseResult) => caseResult.evidence?.model)
22
+ );
23
+ const generation = omitUndefined({
24
+ temperature: options.temperature,
25
+ max_tokens: options.maxTokens,
26
+ seed: options.seed,
27
+ });
28
+
29
+ return {
30
+ schema_version: '1',
31
+ target: {
32
+ provider: boundedString(options.provider, 100) ?? 'unknown',
33
+ ...(requestedModels.length ? { requested_models: requestedModels } : {}),
34
+ ...(observedModels.length ? { observed_models: observedModels } : {}),
35
+ ...(Object.keys(generation).length ? { generation } : {}),
36
+ },
37
+ ...(evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}),
38
+ };
39
+ }
40
+
41
+ function uniqueStrings(values: unknown[]): string[] {
42
+ return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))] as string[];
43
+ }
44
+
45
+ function boundedString(value: unknown, maxLength: number): string | undefined {
46
+ return typeof value === 'string' && value.length > 0 ? value.slice(0, maxLength) : undefined;
47
+ }
48
+
49
+ function omitUndefined<T extends Record<string, number | undefined>>(value: T): Partial<T> {
50
+ return Object.fromEntries(
51
+ Object.entries(value).filter(([, item]) => item !== undefined)
52
+ ) as Partial<T>;
53
+ }
@@ -5,3 +5,4 @@
5
5
  export { getGitInfo } from './git';
6
6
  export { getEnvironmentInfo } from './environment';
7
7
  export { createWorkloadIdentity } from './workload-identity';
8
+ export { createExecutionProvenance } from './execution-provenance';
@@ -170,6 +170,11 @@ describe('executeCase tool loop', () => {
170
170
  expect(result.error).toBe('TOOL_EXECUTOR_REQUIRED');
171
171
  expect(result.latencyMs).toBe(4);
172
172
  expect(result.tokens).toEqual({ prompt: 7, completion: 2, total: 9 });
173
+ expect(result.target).toEqual({
174
+ provider: 'ling',
175
+ requested_model: 'Ling-3.0-flash',
176
+ observed_models: ['Ling-3.0-flash'],
177
+ });
173
178
  });
174
179
 
175
180
  it('retains prior generation metrics when a later generation rejects', async () => {
@@ -201,6 +206,7 @@ describe('executeCase tool loop', () => {
201
206
  steps: 1,
202
207
  terminationReason: 'tool_error',
203
208
  });
209
+ expect(result.target?.observed_models).toEqual(['Ling-3.0-flash']);
204
210
  });
205
211
 
206
212
  it('retains prior generation metrics when a later generation times out', async () => {
@@ -304,7 +310,11 @@ describe('executeCase measurement integrity', () => {
304
310
  status: 'error',
305
311
  response: '',
306
312
  error: 'provider unavailable',
313
+ target: {
314
+ provider: 'test',
315
+ },
307
316
  });
317
+ expect(result.target?.observed_models).toBeUndefined();
308
318
  });
309
319
 
310
320
  it('retains only the bounded evidence contract rather than evaluator details', async () => {
@@ -100,6 +100,7 @@ export async function executeCase(
100
100
  ): Promise<CaseResult> {
101
101
  const { timeout, retries = 0 } = context;
102
102
  const caseStartTime = Date.now();
103
+ const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
103
104
 
104
105
  let lastError: Error | null = null;
105
106
 
@@ -135,6 +136,7 @@ export async function executeCase(
135
136
  expected: testCase.expected,
136
137
  tags: testCase.tags,
137
138
  error: lastError?.message,
139
+ target: targetEvidence(context.client.provider, requestedModel),
138
140
  };
139
141
  }
140
142
 
@@ -143,7 +145,7 @@ async function executeCaseAttempt(
143
145
  context: ExecutorContext,
144
146
  timeout?: number
145
147
  ): Promise<CaseResult> {
146
- const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
148
+ const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
147
149
 
148
150
  // Merge scenario-level and case-level variables (case overrides scenario)
149
151
  const variables = mergeVariables(scenario.variables, testCase.variables);
@@ -171,7 +173,7 @@ async function executeCaseAttempt(
171
173
  const generate = () =>
172
174
  client.generate({
173
175
  prompt: loopPrompt,
174
- model: testCase.model || scenario.model,
176
+ model: testCase.model || requestedModel || scenario.model,
175
177
  temperature: scenario.temperature,
176
178
  maxTokens: scenario.maxTokens,
177
179
  seed: scenario.seed,
@@ -182,6 +184,7 @@ async function executeCaseAttempt(
182
184
  let result = timeout
183
185
  ? await Promise.race([generatePromise, createTimeout(timeout)])
184
186
  : await generatePromise;
187
+ const observedModels = [result.model];
185
188
  const generationMetrics = {
186
189
  latencyMs: result.latencyMs,
187
190
  tokens: { ...result.tokens },
@@ -198,7 +201,12 @@ async function executeCaseAttempt(
198
201
  terminationReason: 'tool_error',
199
202
  },
200
203
  generationMetrics,
201
- 'TOOL_EXECUTOR_REQUIRED'
204
+ 'TOOL_EXECUTOR_REQUIRED',
205
+ targetEvidence(
206
+ client.provider,
207
+ testCase.model || requestedModel || scenario.model,
208
+ observedModels
209
+ )
202
210
  );
203
211
  }
204
212
  const executor =
@@ -225,7 +233,12 @@ async function executeCaseAttempt(
225
233
  terminationReason: 'duplicate_call',
226
234
  },
227
235
  generationMetrics,
228
- 'TOOL_DUPLICATE_CALL'
236
+ 'TOOL_DUPLICATE_CALL',
237
+ targetEvidence(
238
+ client.provider,
239
+ testCase.model || requestedModel || scenario.model,
240
+ observedModels
241
+ )
229
242
  );
230
243
  }
231
244
  seenCalls.add(fingerprint);
@@ -263,7 +276,12 @@ async function executeCaseAttempt(
263
276
  : 'tool_error',
264
277
  },
265
278
  generationMetrics,
266
- execution.error?.code ?? 'TOOL_EXECUTION_FAILED'
279
+ execution.error?.code ?? 'TOOL_EXECUTION_FAILED',
280
+ targetEvidence(
281
+ client.provider,
282
+ testCase.model || requestedModel || scenario.model,
283
+ observedModels
284
+ )
267
285
  );
268
286
  }
269
287
  const content = JSON.stringify(execution.result ?? {});
@@ -281,7 +299,12 @@ async function executeCaseAttempt(
281
299
  toolTrace,
282
300
  { status: 'error', steps: step + 1, terminationReason: 'timeout' },
283
301
  generationMetrics,
284
- 'TOOL_LOOP_TIMEOUT'
302
+ 'TOOL_LOOP_TIMEOUT',
303
+ targetEvidence(
304
+ client.provider,
305
+ testCase.model || requestedModel || scenario.model,
306
+ observedModels
307
+ )
285
308
  );
286
309
  }
287
310
  const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
@@ -298,9 +321,15 @@ async function executeCaseAttempt(
298
321
  terminationReason: timedOut ? 'timeout' : 'tool_error',
299
322
  },
300
323
  generationMetrics,
301
- timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED'
324
+ timedOut ? 'TOOL_LOOP_TIMEOUT' : 'TOOL_GENERATION_FAILED',
325
+ targetEvidence(
326
+ client.provider,
327
+ testCase.model || requestedModel || scenario.model,
328
+ observedModels
329
+ )
302
330
  );
303
331
  }
332
+ observedModels.push(result.model);
304
333
  generationMetrics.latencyMs += result.latencyMs;
305
334
  generationMetrics.tokens.prompt += result.tokens.prompt;
306
335
  generationMetrics.tokens.completion += result.tokens.completion;
@@ -316,7 +345,12 @@ async function executeCaseAttempt(
316
345
  terminationReason: 'max_steps',
317
346
  },
318
347
  generationMetrics,
319
- 'TOOL_LOOP_MAX_STEPS'
348
+ 'TOOL_LOOP_MAX_STEPS',
349
+ targetEvidence(
350
+ client.provider,
351
+ testCase.model || requestedModel || scenario.model,
352
+ observedModels
353
+ )
320
354
  );
321
355
  }
322
356
  toolLoop = { status: 'completed', steps: toolTrace.length, terminationReason: 'completed' };
@@ -435,17 +469,44 @@ async function executeCaseAttempt(
435
469
  tags: testCase.tags,
436
470
  redaction: redactionInfo,
437
471
  evidence: finalEvidence,
472
+ target: targetEvidence(
473
+ client.provider,
474
+ testCase.model || requestedModel || scenario.model,
475
+ observedModels
476
+ ),
438
477
  toolTrace: toolTrace.length ? toolTrace : undefined,
439
478
  toolLoop,
440
479
  };
441
480
  }
442
481
 
482
+ function targetEvidence(
483
+ provider: string,
484
+ requestedModel?: string,
485
+ observedModels?: unknown[]
486
+ ): NonNullable<CaseResult['target']> {
487
+ const observed = [
488
+ ...new Set(
489
+ (observedModels ?? []).filter(
490
+ (model): model is string => typeof model === 'string' && model.length > 0
491
+ )
492
+ ),
493
+ ]
494
+ .map((model) => sanitizeArtifactText(model, 200))
495
+ .filter((model): model is string => Boolean(model));
496
+ return {
497
+ provider: sanitizeArtifactText(provider, 100) ?? 'unknown',
498
+ ...(requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {}),
499
+ ...(observed.length ? { observed_models: observed } : {}),
500
+ };
501
+ }
502
+
443
503
  function createToolLoopError(
444
504
  testCase: TestCase,
445
505
  toolTrace: ToolTraceEntry[],
446
506
  toolLoop: ToolLoopSummary,
447
507
  generationMetrics: Pick<CaseResult, 'latencyMs' | 'tokens'>,
448
- code: string
508
+ code: string,
509
+ target?: CaseResult['target']
449
510
  ): ToolLoopError {
450
511
  return new ToolLoopError(code, {
451
512
  id: testCase.id,
@@ -462,6 +523,7 @@ function createToolLoopError(
462
523
  expected: testCase.expected,
463
524
  tags: testCase.tags,
464
525
  error: code,
526
+ target,
465
527
  toolTrace,
466
528
  toolLoop,
467
529
  });
@@ -96,6 +96,13 @@ describe('release validation: fixture-backed workflow cases', () => {
96
96
  workload: { algorithm: 'sha256' },
97
97
  rubric: { algorithm: 'sha256' },
98
98
  });
99
+ expect(result.manifest.execution_provenance).toMatchObject({
100
+ schema_version: '1',
101
+ target: {
102
+ provider: 'fixture',
103
+ observed_models: ['fixture-model'],
104
+ },
105
+ });
99
106
  });
100
107
 
101
108
  test('RV-02 retains independent logistics tool evidence through a fixture-backed workflow', async () => {
@@ -171,4 +178,33 @@ describe('release validation: fixture-backed workflow cases', () => {
171
178
  toolTrace: [{ toolCall: { id: 'capacity-1' }, result: { available: true } }],
172
179
  });
173
180
  });
181
+
182
+ test('records a CLI-resolved model for each case when the scenario does not declare one', async () => {
183
+ const scenario = ScenarioSchema.parse({
184
+ name: 'resolved model evidence',
185
+ cases: [
186
+ {
187
+ id: 'cli-model-override',
188
+ prompt: 'Respond with assured',
189
+ expected: { type: 'exact', value: 'assured' },
190
+ },
191
+ ],
192
+ });
193
+
194
+ const result = await runScenario({
195
+ scenario,
196
+ client: fixtureClient(['assured']),
197
+ resolvedConfig: {
198
+ provider: 'fixture',
199
+ model: 'fixture-cli-model',
200
+ source: { provider: 'cli', model: 'cli' },
201
+ },
202
+ });
203
+
204
+ expect(result.cases[0].target).toEqual({
205
+ provider: 'fixture',
206
+ requested_model: 'fixture-cli-model',
207
+ observed_models: ['fixture-model'],
208
+ });
209
+ });
174
210
  });
@@ -4,7 +4,7 @@
4
4
 
5
5
  import { createRunManifest } from '../artifacts/manifest';
6
6
  import type { CaseResult, ManifestRedactionInfo } from '../artifacts/types';
7
- import { createWorkloadIdentity } from '../provenance';
7
+ import { createExecutionProvenance, createWorkloadIdentity } from '../provenance';
8
8
  import { Redactor } from '../redaction';
9
9
  import { executeCase } from './executor';
10
10
  import type { RunOptions, RunResult } from './types';
@@ -51,6 +51,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
51
51
  const result = await executeCase(testCase, {
52
52
  client,
53
53
  scenario,
54
+ requestedModel: resolvedConfig?.model,
54
55
  timeout: testCase.timeout || timeout,
55
56
  retries: testCase.retries ?? retries,
56
57
  redaction,
@@ -70,6 +71,7 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
70
71
  const result = await executeCase(testCase, {
71
72
  client,
72
73
  scenario,
74
+ requestedModel: resolvedConfig?.model,
73
75
  timeout: testCase.timeout || timeout,
74
76
  retries: testCase.retries ?? retries,
75
77
  redaction,
@@ -122,6 +124,14 @@ export async function runScenario(options: RunOptions): Promise<RunResult> {
122
124
  },
123
125
  resolvedConfig,
124
126
  workloadIdentity: createWorkloadIdentity(scenario),
127
+ executionProvenance: createExecutionProvenance({
128
+ provider: client.provider,
129
+ requestedModel: resolvedConfig?.model || scenario.model,
130
+ temperature: resolvedConfig?.temperature ?? scenario.temperature,
131
+ maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
132
+ seed: scenario.seed,
133
+ cases: results,
134
+ }),
125
135
  cases: results,
126
136
  startTime,
127
137
  endTime,
@@ -56,6 +56,8 @@ export interface RunResult {
56
56
  export interface ExecutorContext {
57
57
  client: ModelClient;
58
58
  scenario: Scenario;
59
+ /** Effective model selected outside the scenario, such as a CLI override. */
60
+ requestedModel?: string;
59
61
  timeout?: number;
60
62
  retries?: number;
61
63
  /** Redaction configuration for this execution */