@artemiskit/core 0.4.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/artifacts/manifest.d.ts +3 -1
- package/dist/artifacts/manifest.d.ts.map +1 -1
- package/dist/artifacts/types.d.ts +48 -0
- package/dist/artifacts/types.d.ts.map +1 -1
- package/dist/index.js +186 -12
- package/dist/provenance/execution-provenance.d.ts +11 -0
- package/dist/provenance/execution-provenance.d.ts.map +1 -0
- package/dist/provenance/index.d.ts +2 -0
- package/dist/provenance/index.d.ts.map +1 -1
- package/dist/provenance/workload-identity.d.ts +12 -0
- package/dist/provenance/workload-identity.d.ts.map +1 -0
- package/dist/runner/executor.d.ts.map +1 -1
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/types.d.ts +2 -0
- package/dist/runner/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/artifacts/manifest.test.ts +67 -1
- package/src/artifacts/manifest.ts +9 -1
- package/src/artifacts/types.ts +139 -0
- package/src/provenance/execution-provenance.test.ts +59 -0
- package/src/provenance/execution-provenance.ts +53 -0
- package/src/provenance/index.ts +2 -0
- package/src/provenance/workload-identity.test.ts +69 -0
- package/src/provenance/workload-identity.ts +92 -0
- package/src/runner/executor.test.ts +10 -0
- package/src/runner/executor.ts +71 -9
- package/src/runner/release-validation.test.ts +41 -0
- package/src/runner/runner.ts +12 -0
- package/src/runner/types.ts +2 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# @artemiskit/core
|
|
2
2
|
|
|
3
|
+
## 0.5.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Preserve bounded requested and observed target-model evidence per case and per run, separately from
|
|
8
|
+
evaluator identities. Normal manifests now use v1.3 execution provenance while retaining readable
|
|
9
|
+
historical artifacts. CI JSON output is machine-readable without progress text.
|
|
10
|
+
|
|
11
|
+
## 0.5.0
|
|
12
|
+
|
|
13
|
+
### Minor Changes
|
|
14
|
+
|
|
15
|
+
- b826698: Add versioned, redacted SHA-256 workload and rubric identities to normal scenario-run manifests.
|
|
16
|
+
The new manifest v1.2 field distinguishes changed declared inputs from changed evaluation criteria
|
|
17
|
+
without inferring identity for historical artifacts.
|
|
18
|
+
|
|
3
19
|
## 0.4.2
|
|
4
20
|
|
|
5
21
|
### Patch Changes
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Run manifest generation utilities
|
|
3
3
|
*/
|
|
4
|
-
import type { CaseResult, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest } from './types';
|
|
4
|
+
import type { CaseResult, ExecutionProvenance, ManifestRedactionInfo, ResolvedConfig, RunConfig, RunManifest, WorkloadIdentity } from './types';
|
|
5
5
|
/**
|
|
6
6
|
* Create a new run manifest
|
|
7
7
|
*/
|
|
@@ -9,6 +9,8 @@ export declare function createRunManifest(options: {
|
|
|
9
9
|
project: string;
|
|
10
10
|
config: RunConfig;
|
|
11
11
|
resolvedConfig?: ResolvedConfig;
|
|
12
|
+
workloadIdentity?: WorkloadIdentity;
|
|
13
|
+
executionProvenance?: ExecutionProvenance;
|
|
12
14
|
cases: CaseResult[];
|
|
13
15
|
startTime: Date;
|
|
14
16
|
endTime: Date;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,
|
|
1
|
+
{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../src/artifacts/manifest.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EACV,UAAU,EAEV,mBAAmB,EACnB,qBAAqB,EACrB,cAAc,EACd,SAAS,EACT,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAGjB;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC,GAAG,WAAW,CA2Cd"}
|
|
@@ -35,6 +35,48 @@ export interface ManifestRedactionInfo {
|
|
|
35
35
|
totalRedactions: number;
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
|
+
/** A versioned SHA-256 digest of canonical, redacted assessment material. */
|
|
39
|
+
export interface ContentIdentity {
|
|
40
|
+
schema_version: '1';
|
|
41
|
+
algorithm: 'sha256';
|
|
42
|
+
digest: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Separates the scenario workload from the criteria used to judge it.
|
|
46
|
+
*
|
|
47
|
+
* The digests prove matching declared, sanitized inputs. They are not a
|
|
48
|
+
* signature or an attestation of provider behaviour.
|
|
49
|
+
*/
|
|
50
|
+
export interface WorkloadIdentity {
|
|
51
|
+
schema_version: '1';
|
|
52
|
+
workload: ContentIdentity;
|
|
53
|
+
rubric: ContentIdentity;
|
|
54
|
+
}
|
|
55
|
+
/** Bounded target identity captured from a case execution. */
|
|
56
|
+
export interface CaseTargetEvidence {
|
|
57
|
+
provider: string;
|
|
58
|
+
requested_model?: string;
|
|
59
|
+
/** Model identifiers returned by the target provider during this case. */
|
|
60
|
+
observed_models?: string[];
|
|
61
|
+
}
|
|
62
|
+
/** Requested and observed execution configuration for a complete run. */
|
|
63
|
+
export interface ExecutionProvenance {
|
|
64
|
+
schema_version: '1';
|
|
65
|
+
target: {
|
|
66
|
+
provider: string;
|
|
67
|
+
requested_models?: string[];
|
|
68
|
+
observed_models?: string[];
|
|
69
|
+
generation?: {
|
|
70
|
+
temperature?: number;
|
|
71
|
+
max_tokens?: number;
|
|
72
|
+
seed?: number;
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
/** Judge/evaluator model identities, never combined with target identity. */
|
|
76
|
+
evaluator?: {
|
|
77
|
+
models?: string[];
|
|
78
|
+
};
|
|
79
|
+
}
|
|
38
80
|
/**
|
|
39
81
|
* Terminal status of a case measurement.
|
|
40
82
|
*
|
|
@@ -82,6 +124,8 @@ export interface CaseResult {
|
|
|
82
124
|
error?: string;
|
|
83
125
|
/** Sanitized evaluator evidence; arbitrary evaluator details are never stored here. */
|
|
84
126
|
evidence?: CaseEvaluationEvidence;
|
|
127
|
+
/** Requested and observed target identity for this case. */
|
|
128
|
+
target?: CaseTargetEvidence;
|
|
85
129
|
/** Redaction information for this case */
|
|
86
130
|
redaction?: CaseRedactionInfo;
|
|
87
131
|
/** Ordered tool activity captured for an enabled tool loop. */
|
|
@@ -237,6 +281,10 @@ export interface RunManifest {
|
|
|
237
281
|
config: RunConfig;
|
|
238
282
|
/** Resolved configuration with full provider details and source tracking */
|
|
239
283
|
resolved_config?: ResolvedConfig;
|
|
284
|
+
/** Versioned identities for the declared workload and evaluation rubric. */
|
|
285
|
+
workload_identity?: WorkloadIdentity;
|
|
286
|
+
/** Requested and observed target/evaluator configuration for this run. */
|
|
287
|
+
execution_provenance?: ExecutionProvenance;
|
|
240
288
|
metrics: RunMetrics;
|
|
241
289
|
git: GitInfo;
|
|
242
290
|
provenance: ProvenanceInfo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAwB7F;AAgDD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/artifacts/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,EAAE,OAAO,CAAC;IAClB,kCAAkC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB,oCAAoC;IACpC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,iDAAiD;IACjD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,yDAAyD;IACzD,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sDAAsD;QACtD,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAMD,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,GAAG,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,GAAG,CAAC;IACpB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,8DAA8D;AAC9D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0EAA0E;IAC1E,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,yEAAyE;AACzE,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,GAAG,CAAC;IACpB,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;QAC5B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3B,UAAU,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,IAAI,CAAC,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC;IACF,6EAA6E;IAC7E,SAAS,CAAC,EAAE;QACV,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;KACnB,CAAC;CACH;AAMD;;;;;GAKG;AACH,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAE7E,uEAAuE;AACvE,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAK9E,CAAC;AAEF,uEAAuE;AACvE,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,mFAAmF;IACnF,MAAM,CAAC,EAAE,oBAAoB,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,4DAA4D;IAC5D,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,OAAO,UAAU,EAAE,cAAc,EAAE,CAAC;IAChD,gDAAgD;IAChD,QAAQ,CAAC,EAAE,OAAO,UAAU,EAAE,eAAe,CAAC;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,iCAAiC;IACjC,IAAI,CAAC,EAAE,gBAAgB,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,EAAE,CAAC,EAAE;QACH,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,KAAK,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAC;AAE7E;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IAGf,6BAA6B;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,0BAA0B;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B,0CAA0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,qCAAqC;IACrC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAG3B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kBAAkB;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,mDAAmD;IACnD,MAAM,EAAE;QACN,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,QAAQ,CAAC,EAAE,YAAY,CAAC;QACxB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,eAAe,CAAC,EAAE,YAAY,CAAC;QAC/B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,mBAAmB,CAAC,EAAE,YAAY,CAAC;QACnC,IAAI,CAAC,EAAE,YAAY,CAAC;QACpB,aAAa,CAAC,EAAE,YAAY,CAAC;QAC7B,cAAc,CAAC,EAAE,YAAY,CAAC;QAC9B,gBAAgB,CAAC,EAAE,YAAY,CAAC;QAChC,OAAO,CAAC,EAAE,YAAY,CAAC;QACvB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,UAAU,CAAC,EAAE,YAAY,CAAC;KAC3B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,SAAS,CAAC;IAClB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,4EAA4E;IAC5E,iBAAiB,CAAC,EAAE,gBAAgB,CAAC;IACrC,0EAA0E;IAC1E,oBAAoB,CAAC,EAAE,mBAAmB,CAAC;IAC3C,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,UAAU,GAAG,oBAAoB,CAWpF;AAED,4EAA4E;AAC5E,wBAAgB,4BAA4B,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAE3E;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAkC7F;AAyHD;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;AAE9E;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,yBAAyB;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAC;IACzB,yBAAyB;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,YAAY,EAAE,MAAM,CAAC;IACrB,4BAA4B;IAC5B,WAAW,EAAE;QACX,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,aAAa,CAAC;IACtB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,cAAc,CAAC;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oCAAoC;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,MAAM,CAAC,EAAE;QACP,0BAA0B;QAC1B,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,UAAU,EAAE,MAAM,CAAC;QACnB,mBAAmB;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,0BAA0B;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sBAAsB;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,8BAA8B;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE;QACP,oDAAoD;QACpD,mBAAmB,EAAE,MAAM,CAAC;QAC5B,yDAAyD;QACzD,uBAAuB,EAAE,MAAM,CAAC;QAChC,yCAAyC;QACzC,YAAY,EAAE,MAAM,CAAC;QACrB,iCAAiC;QACjC,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,+EAA+E;IAC/E,IAAI,CAAC,EAAE;QACL,kCAAkC;QAClC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,mCAAmC;QACnC,SAAS,EAAE;YACT,mCAAmC;YACnC,eAAe,EAAE,MAAM,CAAC;YACxB,wCAAwC;YACxC,mBAAmB,EAAE,MAAM,CAAC;SAC7B,CAAC;QACF,sCAAsC;QACtC,KAAK,EAAE,MAAM,CAAC;QACd,mCAAmC;QACnC,OAAO,EAAE;YACP,aAAa,EAAE,MAAM,CAAC;YACtB,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,OAAO,EAAE,aAAa,CAAC;IACvB,GAAG,EAAE,OAAO,CAAC;IACb,UAAU,EAAE,cAAc,CAAC;IAC3B,mEAAmE;IACnE,cAAc,EAAE,mBAAmB,EAAE,CAAC;IACtC,WAAW,EAAE;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,yCAAyC;IACzC,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,eAAe,GAAG,cAAc,CAAC;AAEzE;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAE5E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAEpF;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAElF"}
|
package/dist/index.js
CHANGED
|
@@ -20525,6 +20525,7 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
|
|
|
20525
20525
|
async function executeCase(testCase, context) {
|
|
20526
20526
|
const { timeout, retries = 0 } = context;
|
|
20527
20527
|
const caseStartTime = Date.now();
|
|
20528
|
+
const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
|
|
20528
20529
|
let lastError = null;
|
|
20529
20530
|
for (let attempt = 0;attempt <= retries; attempt++) {
|
|
20530
20531
|
try {
|
|
@@ -20555,11 +20556,12 @@ async function executeCase(testCase, context) {
|
|
|
20555
20556
|
response: "",
|
|
20556
20557
|
expected: testCase.expected,
|
|
20557
20558
|
tags: testCase.tags,
|
|
20558
|
-
error: lastError?.message
|
|
20559
|
+
error: lastError?.message,
|
|
20560
|
+
target: targetEvidence(context.client.provider, requestedModel)
|
|
20559
20561
|
};
|
|
20560
20562
|
}
|
|
20561
20563
|
async function executeCaseAttempt(testCase, context, timeout) {
|
|
20562
|
-
const { client, scenario, redaction: cliRedaction, toolExecutor } = context;
|
|
20564
|
+
const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
|
|
20563
20565
|
const variables = mergeVariables(scenario.variables, testCase.variables);
|
|
20564
20566
|
let prompt = substituteVariables(testCase.prompt, variables);
|
|
20565
20567
|
if (scenario.setup?.systemPrompt && typeof prompt === "string") {
|
|
@@ -20577,7 +20579,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20577
20579
|
const loopPrompt = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt;
|
|
20578
20580
|
const generate = () => client.generate({
|
|
20579
20581
|
prompt: loopPrompt,
|
|
20580
|
-
model: testCase.model || scenario.model,
|
|
20582
|
+
model: testCase.model || requestedModel || scenario.model,
|
|
20581
20583
|
temperature: scenario.temperature,
|
|
20582
20584
|
maxTokens: scenario.maxTokens,
|
|
20583
20585
|
seed: scenario.seed,
|
|
@@ -20585,6 +20587,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20585
20587
|
});
|
|
20586
20588
|
const generatePromise = generate();
|
|
20587
20589
|
let result = timeout ? await Promise.race([generatePromise, createTimeout(timeout)]) : await generatePromise;
|
|
20590
|
+
const observedModels = [result.model];
|
|
20588
20591
|
const generationMetrics = {
|
|
20589
20592
|
latencyMs: result.latencyMs,
|
|
20590
20593
|
tokens: { ...result.tokens }
|
|
@@ -20595,7 +20598,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20595
20598
|
status: "error",
|
|
20596
20599
|
steps: 0,
|
|
20597
20600
|
terminationReason: "tool_error"
|
|
20598
|
-
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED");
|
|
20601
|
+
}, generationMetrics, "TOOL_EXECUTOR_REQUIRED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20599
20602
|
}
|
|
20600
20603
|
const executor = toolExecutor ?? new FixtureToolExecutor({
|
|
20601
20604
|
tools,
|
|
@@ -20614,7 +20617,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20614
20617
|
status: "error",
|
|
20615
20618
|
steps: step,
|
|
20616
20619
|
terminationReason: "duplicate_call"
|
|
20617
|
-
}, generationMetrics, "TOOL_DUPLICATE_CALL");
|
|
20620
|
+
}, generationMetrics, "TOOL_DUPLICATE_CALL", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20618
20621
|
}
|
|
20619
20622
|
seenCalls.add(fingerprint);
|
|
20620
20623
|
const toolStart = Date.now();
|
|
@@ -20641,7 +20644,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20641
20644
|
status: "error",
|
|
20642
20645
|
steps: step + 1,
|
|
20643
20646
|
terminationReason: execution.error?.code === "TOOL_UNKNOWN" ? "unknown_tool" : execution.error?.code?.startsWith("TOOL_ARGUMENTS") ? "invalid_arguments" : "tool_error"
|
|
20644
|
-
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED");
|
|
20647
|
+
}, generationMetrics, execution.error?.code ?? "TOOL_EXECUTION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20645
20648
|
}
|
|
20646
20649
|
const content = JSON.stringify(execution.result ?? {});
|
|
20647
20650
|
loopPrompt.push({
|
|
@@ -20653,7 +20656,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20653
20656
|
}
|
|
20654
20657
|
const remainingLoopTime = policy.timeoutMs - (Date.now() - loopStartedAt);
|
|
20655
20658
|
if (remainingLoopTime <= 0) {
|
|
20656
|
-
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT");
|
|
20659
|
+
throw createToolLoopError(testCase, toolTrace, { status: "error", steps: step + 1, terminationReason: "timeout" }, generationMetrics, "TOOL_LOOP_TIMEOUT", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20657
20660
|
}
|
|
20658
20661
|
const requestTimeout = timeout ? Math.min(timeout, remainingLoopTime) : remainingLoopTime;
|
|
20659
20662
|
try {
|
|
@@ -20664,8 +20667,9 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20664
20667
|
status: "error",
|
|
20665
20668
|
steps: step + 1,
|
|
20666
20669
|
terminationReason: timedOut ? "timeout" : "tool_error"
|
|
20667
|
-
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED");
|
|
20670
|
+
}, generationMetrics, timedOut ? "TOOL_LOOP_TIMEOUT" : "TOOL_GENERATION_FAILED", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20668
20671
|
}
|
|
20672
|
+
observedModels.push(result.model);
|
|
20669
20673
|
generationMetrics.latencyMs += result.latencyMs;
|
|
20670
20674
|
generationMetrics.tokens.prompt += result.tokens.prompt;
|
|
20671
20675
|
generationMetrics.tokens.completion += result.tokens.completion;
|
|
@@ -20676,7 +20680,7 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20676
20680
|
status: "error",
|
|
20677
20681
|
steps: policy.maxSteps,
|
|
20678
20682
|
terminationReason: "max_steps"
|
|
20679
|
-
}, generationMetrics, "TOOL_LOOP_MAX_STEPS");
|
|
20683
|
+
}, generationMetrics, "TOOL_LOOP_MAX_STEPS", targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels));
|
|
20680
20684
|
}
|
|
20681
20685
|
toolLoop = { status: "completed", steps: toolTrace.length, terminationReason: "completed" };
|
|
20682
20686
|
}
|
|
@@ -20772,11 +20776,22 @@ async function executeCaseAttempt(testCase, context, timeout) {
|
|
|
20772
20776
|
tags: testCase.tags,
|
|
20773
20777
|
redaction: redactionInfo,
|
|
20774
20778
|
evidence: finalEvidence,
|
|
20779
|
+
target: targetEvidence(client.provider, testCase.model || requestedModel || scenario.model, observedModels),
|
|
20775
20780
|
toolTrace: toolTrace.length ? toolTrace : undefined,
|
|
20776
20781
|
toolLoop
|
|
20777
20782
|
};
|
|
20778
20783
|
}
|
|
20779
|
-
function
|
|
20784
|
+
function targetEvidence(provider, requestedModel, observedModels) {
|
|
20785
|
+
const observed = [
|
|
20786
|
+
...new Set((observedModels ?? []).filter((model) => typeof model === "string" && model.length > 0))
|
|
20787
|
+
].map((model) => sanitizeArtifactText(model, 200)).filter((model) => Boolean(model));
|
|
20788
|
+
return {
|
|
20789
|
+
provider: sanitizeArtifactText(provider, 100) ?? "unknown",
|
|
20790
|
+
...requestedModel ? { requested_model: sanitizeArtifactText(requestedModel, 200) } : {},
|
|
20791
|
+
...observed.length ? { observed_models: observed } : {}
|
|
20792
|
+
};
|
|
20793
|
+
}
|
|
20794
|
+
function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, code, target) {
|
|
20780
20795
|
return new ToolLoopError(code, {
|
|
20781
20796
|
id: testCase.id,
|
|
20782
20797
|
name: testCase.name,
|
|
@@ -20792,6 +20807,7 @@ function createToolLoopError(testCase, toolTrace, toolLoop, generationMetrics, c
|
|
|
20792
20807
|
expected: testCase.expected,
|
|
20793
20808
|
tags: testCase.tags,
|
|
20794
20809
|
error: code,
|
|
20810
|
+
target,
|
|
20795
20811
|
toolTrace,
|
|
20796
20812
|
toolLoop
|
|
20797
20813
|
});
|
|
@@ -21292,6 +21308,12 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21292
21308
|
if (!isRecord2(manifest) || !Array.isArray(manifest.cases)) {
|
|
21293
21309
|
throw new Error("Invalid run manifest: expected an object with a cases array");
|
|
21294
21310
|
}
|
|
21311
|
+
if (manifest.workload_identity !== undefined) {
|
|
21312
|
+
assertWorkloadIdentity(manifest.workload_identity);
|
|
21313
|
+
}
|
|
21314
|
+
if (manifest.execution_provenance !== undefined) {
|
|
21315
|
+
assertExecutionProvenance(manifest.execution_provenance);
|
|
21316
|
+
}
|
|
21295
21317
|
for (const [index, caseResult] of manifest.cases.entries()) {
|
|
21296
21318
|
if (!isRecord2(caseResult)) {
|
|
21297
21319
|
throw new Error(`Invalid run manifest: case ${index} is not an object`);
|
|
@@ -21302,8 +21324,46 @@ function assertRunManifestIntegrity(manifest) {
|
|
|
21302
21324
|
if (caseResult.evidence !== undefined) {
|
|
21303
21325
|
assertCaseEvaluationEvidence(caseResult.evidence, index);
|
|
21304
21326
|
}
|
|
21327
|
+
if (caseResult.target !== undefined) {
|
|
21328
|
+
assertCaseTargetEvidence(caseResult.target);
|
|
21329
|
+
}
|
|
21330
|
+
}
|
|
21331
|
+
}
|
|
21332
|
+
function assertCaseTargetEvidence(target) {
|
|
21333
|
+
if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
|
|
21334
|
+
throw new Error("Invalid run manifest: malformed target evidence");
|
|
21335
|
+
}
|
|
21336
|
+
}
|
|
21337
|
+
function assertExecutionProvenance(provenance) {
|
|
21338
|
+
if (!isRecord2(provenance) || provenance.schema_version !== "1" || !isRecord2(provenance.target)) {
|
|
21339
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
21340
|
+
}
|
|
21341
|
+
const target = provenance.target;
|
|
21342
|
+
if (typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || !isBoundedStringList(target.requested_models) || !isBoundedStringList(target.observed_models) || target.generation !== undefined && !isGenerationConfig(target.generation)) {
|
|
21343
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
21344
|
+
}
|
|
21345
|
+
if (provenance.evaluator !== undefined) {
|
|
21346
|
+
if (!isRecord2(provenance.evaluator) || !isBoundedStringList(provenance.evaluator.models)) {
|
|
21347
|
+
throw new Error("Invalid run manifest: malformed execution provenance");
|
|
21348
|
+
}
|
|
21349
|
+
}
|
|
21350
|
+
}
|
|
21351
|
+
function isBoundedStringList(value) {
|
|
21352
|
+
return value === undefined || Array.isArray(value) && value.length <= 100 && value.every((item) => typeof item === "string" && item.length > 0 && item.length <= 200);
|
|
21353
|
+
}
|
|
21354
|
+
function isGenerationConfig(value) {
|
|
21355
|
+
if (!isRecord2(value))
|
|
21356
|
+
return false;
|
|
21357
|
+
return [value.temperature, value.max_tokens, value.seed].every((item) => item === undefined || typeof item === "number" && Number.isFinite(item));
|
|
21358
|
+
}
|
|
21359
|
+
function assertWorkloadIdentity(identity) {
|
|
21360
|
+
if (!isRecord2(identity) || identity.schema_version !== "1" || !isContentIdentity(identity.workload) || !isContentIdentity(identity.rubric)) {
|
|
21361
|
+
throw new Error("Invalid run manifest: malformed workload identity");
|
|
21305
21362
|
}
|
|
21306
21363
|
}
|
|
21364
|
+
function isContentIdentity(value) {
|
|
21365
|
+
return isRecord2(value) && value.schema_version === "1" && value.algorithm === "sha256" && typeof value.digest === "string" && /^[a-f0-9]{64}$/.test(value.digest);
|
|
21366
|
+
}
|
|
21307
21367
|
function assertCaseEvaluationEvidence(evidence, caseIndex) {
|
|
21308
21368
|
if (!isRecord2(evidence) || typeof evidence.evaluator !== "string" || evidence.evaluator.length > 100) {
|
|
21309
21369
|
throw new Error(`Invalid run manifest: case ${caseIndex} has malformed evaluator evidence`);
|
|
@@ -21345,6 +21405,8 @@ function createRunManifest(options) {
|
|
|
21345
21405
|
project,
|
|
21346
21406
|
config,
|
|
21347
21407
|
resolvedConfig,
|
|
21408
|
+
workloadIdentity,
|
|
21409
|
+
executionProvenance,
|
|
21348
21410
|
cases,
|
|
21349
21411
|
startTime,
|
|
21350
21412
|
endTime,
|
|
@@ -21357,7 +21419,7 @@ function createRunManifest(options) {
|
|
|
21357
21419
|
const git = getGitInfo();
|
|
21358
21420
|
const environment = getEnvironmentInfo();
|
|
21359
21421
|
return {
|
|
21360
|
-
version: "1.
|
|
21422
|
+
version: "1.3",
|
|
21361
21423
|
run_id: nanoid(12),
|
|
21362
21424
|
project,
|
|
21363
21425
|
start_time: startTime.toISOString(),
|
|
@@ -21365,6 +21427,8 @@ function createRunManifest(options) {
|
|
|
21365
21427
|
duration_ms: endTime.getTime() - startTime.getTime(),
|
|
21366
21428
|
config,
|
|
21367
21429
|
resolved_config: resolvedConfig,
|
|
21430
|
+
workload_identity: workloadIdentity,
|
|
21431
|
+
execution_provenance: executionProvenance,
|
|
21368
21432
|
metrics,
|
|
21369
21433
|
git,
|
|
21370
21434
|
provenance: {
|
|
@@ -21452,7 +21516,104 @@ function detectCIEnvironment() {
|
|
|
21452
21516
|
}
|
|
21453
21517
|
return;
|
|
21454
21518
|
}
|
|
21455
|
-
|
|
21519
|
+
// src/provenance/workload-identity.ts
|
|
21520
|
+
import { createHash as createHash2 } from "crypto";
|
|
21521
|
+
var SENSITIVE_KEY = /(?:api[-_]?key|authorization|auth|credential|password|secret|token)$/i;
|
|
21522
|
+
var REDACTED_SECRET = "[REDACTED_SECRET]";
|
|
21523
|
+
function createWorkloadIdentity(scenario) {
|
|
21524
|
+
const patterns = resolvePatterns([
|
|
21525
|
+
...DEFAULT_REDACTION_PATTERNS,
|
|
21526
|
+
...scenario.redaction?.patterns ?? []
|
|
21527
|
+
]).map(({ regex }) => regex);
|
|
21528
|
+
const workload = {
|
|
21529
|
+
name: scenario.name,
|
|
21530
|
+
version: scenario.version,
|
|
21531
|
+
description: scenario.description,
|
|
21532
|
+
tags: scenario.tags,
|
|
21533
|
+
variables: scenario.variables,
|
|
21534
|
+
setup: scenario.setup,
|
|
21535
|
+
cases: scenario.cases.map((testCase) => ({
|
|
21536
|
+
id: testCase.id,
|
|
21537
|
+
name: testCase.name,
|
|
21538
|
+
description: testCase.description,
|
|
21539
|
+
prompt: testCase.prompt,
|
|
21540
|
+
tags: testCase.tags,
|
|
21541
|
+
metadata: testCase.metadata,
|
|
21542
|
+
timeout: testCase.timeout,
|
|
21543
|
+
retries: testCase.retries,
|
|
21544
|
+
variables: testCase.variables
|
|
21545
|
+
}))
|
|
21546
|
+
};
|
|
21547
|
+
const rubric = {
|
|
21548
|
+
cases: scenario.cases.map((testCase) => ({
|
|
21549
|
+
id: testCase.id,
|
|
21550
|
+
expected: testCase.expected
|
|
21551
|
+
}))
|
|
21552
|
+
};
|
|
21553
|
+
return {
|
|
21554
|
+
schema_version: "1",
|
|
21555
|
+
workload: createContentIdentity(workload, patterns),
|
|
21556
|
+
rubric: createContentIdentity(rubric, patterns)
|
|
21557
|
+
};
|
|
21558
|
+
}
|
|
21559
|
+
function createContentIdentity(value, patterns) {
|
|
21560
|
+
const canonical = JSON.stringify(canonicalize(value, patterns));
|
|
21561
|
+
return {
|
|
21562
|
+
schema_version: "1",
|
|
21563
|
+
algorithm: "sha256",
|
|
21564
|
+
digest: createHash2("sha256").update(canonical).digest("hex")
|
|
21565
|
+
};
|
|
21566
|
+
}
|
|
21567
|
+
function canonicalize(value, patterns, key) {
|
|
21568
|
+
if (key && SENSITIVE_KEY.test(key))
|
|
21569
|
+
return REDACTED_SECRET;
|
|
21570
|
+
if (typeof value === "string") {
|
|
21571
|
+
return redactText(value, patterns, REDACTED_SECRET).text;
|
|
21572
|
+
}
|
|
21573
|
+
if (Array.isArray(value)) {
|
|
21574
|
+
return value.map((item) => canonicalize(item, patterns));
|
|
21575
|
+
}
|
|
21576
|
+
if (value && typeof value === "object") {
|
|
21577
|
+
return Object.fromEntries(Object.entries(value).filter(([, nestedValue]) => nestedValue !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([nestedKey, nestedValue]) => [
|
|
21578
|
+
nestedKey,
|
|
21579
|
+
canonicalize(nestedValue, patterns, nestedKey)
|
|
21580
|
+
]));
|
|
21581
|
+
}
|
|
21582
|
+
return value;
|
|
21583
|
+
}
|
|
21584
|
+
// src/provenance/execution-provenance.ts
|
|
21585
|
+
function createExecutionProvenance(options) {
|
|
21586
|
+
const requestedModels = uniqueStrings([
|
|
21587
|
+
options.requestedModel,
|
|
21588
|
+
...options.cases.map((caseResult) => caseResult.target?.requested_model)
|
|
21589
|
+
]);
|
|
21590
|
+
const observedModels = uniqueStrings(options.cases.flatMap((caseResult) => caseResult.target?.observed_models ?? []));
|
|
21591
|
+
const evaluatorModels = uniqueStrings(options.cases.map((caseResult) => caseResult.evidence?.model));
|
|
21592
|
+
const generation = omitUndefined({
|
|
21593
|
+
temperature: options.temperature,
|
|
21594
|
+
max_tokens: options.maxTokens,
|
|
21595
|
+
seed: options.seed
|
|
21596
|
+
});
|
|
21597
|
+
return {
|
|
21598
|
+
schema_version: "1",
|
|
21599
|
+
target: {
|
|
21600
|
+
provider: boundedString(options.provider, 100) ?? "unknown",
|
|
21601
|
+
...requestedModels.length ? { requested_models: requestedModels } : {},
|
|
21602
|
+
...observedModels.length ? { observed_models: observedModels } : {},
|
|
21603
|
+
...Object.keys(generation).length ? { generation } : {}
|
|
21604
|
+
},
|
|
21605
|
+
...evaluatorModels.length ? { evaluator: { models: evaluatorModels } } : {}
|
|
21606
|
+
};
|
|
21607
|
+
}
|
|
21608
|
+
function uniqueStrings(values) {
|
|
21609
|
+
return [...new Set(values.map((value) => boundedString(value, 200)).filter(Boolean))];
|
|
21610
|
+
}
|
|
21611
|
+
function boundedString(value, maxLength) {
|
|
21612
|
+
return typeof value === "string" && value.length > 0 ? value.slice(0, maxLength) : undefined;
|
|
21613
|
+
}
|
|
21614
|
+
function omitUndefined(value) {
|
|
21615
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
|
21616
|
+
}
|
|
21456
21617
|
// src/runner/runner.ts
|
|
21457
21618
|
async function runScenario(options) {
|
|
21458
21619
|
const {
|
|
@@ -21486,6 +21647,7 @@ async function runScenario(options) {
|
|
|
21486
21647
|
const result = await executeCase(testCase, {
|
|
21487
21648
|
client,
|
|
21488
21649
|
scenario,
|
|
21650
|
+
requestedModel: resolvedConfig?.model,
|
|
21489
21651
|
timeout: testCase.timeout || timeout,
|
|
21490
21652
|
retries: testCase.retries ?? retries,
|
|
21491
21653
|
redaction,
|
|
@@ -21502,6 +21664,7 @@ async function runScenario(options) {
|
|
|
21502
21664
|
const result = await executeCase(testCase, {
|
|
21503
21665
|
client,
|
|
21504
21666
|
scenario,
|
|
21667
|
+
requestedModel: resolvedConfig?.model,
|
|
21505
21668
|
timeout: testCase.timeout || timeout,
|
|
21506
21669
|
retries: testCase.retries ?? retries,
|
|
21507
21670
|
redaction,
|
|
@@ -21545,6 +21708,15 @@ async function runScenario(options) {
|
|
|
21545
21708
|
seed: scenario.seed
|
|
21546
21709
|
},
|
|
21547
21710
|
resolvedConfig,
|
|
21711
|
+
workloadIdentity: createWorkloadIdentity(scenario),
|
|
21712
|
+
executionProvenance: createExecutionProvenance({
|
|
21713
|
+
provider: client.provider,
|
|
21714
|
+
requestedModel: resolvedConfig?.model || scenario.model,
|
|
21715
|
+
temperature: resolvedConfig?.temperature ?? scenario.temperature,
|
|
21716
|
+
maxTokens: resolvedConfig?.max_tokens ?? scenario.maxTokens,
|
|
21717
|
+
seed: scenario.seed,
|
|
21718
|
+
cases: results
|
|
21719
|
+
}),
|
|
21548
21720
|
cases: results,
|
|
21549
21721
|
startTime,
|
|
21550
21722
|
endTime,
|
|
@@ -32627,11 +32799,13 @@ export {
|
|
|
32627
32799
|
assertRunManifestIntegrity,
|
|
32628
32800
|
createAdapter,
|
|
32629
32801
|
createDefaultRedactor,
|
|
32802
|
+
createExecutionProvenance,
|
|
32630
32803
|
createNoOpRedactor,
|
|
32631
32804
|
createRedactionOptions,
|
|
32632
32805
|
createRunManifest,
|
|
32633
32806
|
createStorageAdapter,
|
|
32634
32807
|
createStorageFromEnv,
|
|
32808
|
+
createWorkloadIdentity,
|
|
32635
32809
|
discoverScenarios,
|
|
32636
32810
|
estimateCost,
|
|
32637
32811
|
executeCase,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Bounded execution provenance derived from declared and observed run evidence. */
|
|
2
|
+
import type { CaseResult, ExecutionProvenance } from '../artifacts/types';
|
|
3
|
+
export declare function createExecutionProvenance(options: {
|
|
4
|
+
provider: string;
|
|
5
|
+
requestedModel?: string;
|
|
6
|
+
temperature?: number;
|
|
7
|
+
maxTokens?: number;
|
|
8
|
+
seed?: number;
|
|
9
|
+
cases: CaseResult[];
|
|
10
|
+
}): ExecutionProvenance;
|
|
11
|
+
//# sourceMappingURL=execution-provenance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"execution-provenance.d.ts","sourceRoot":"","sources":["../../src/provenance/execution-provenance.ts"],"names":[],"mappings":"AAAA,oFAAoF;AAEpF,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE1E,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,GAAG,mBAAmB,CA2BtB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/provenance/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical, redacted workload identities for reproducible assessment evidence.
|
|
3
|
+
*/
|
|
4
|
+
import type { WorkloadIdentity } from '../artifacts/types';
|
|
5
|
+
import type { Scenario } from '../scenario/schema';
|
|
6
|
+
/**
|
|
7
|
+
* Create independent identities for the declared workload and its evaluation
|
|
8
|
+
* criteria. Input is sorted, redacted, and hashed locally; raw material is
|
|
9
|
+
* never retained in the identity artifact.
|
|
10
|
+
*/
|
|
11
|
+
export declare function createWorkloadIdentity(scenario: Scenario): WorkloadIdentity;
|
|
12
|
+
//# sourceMappingURL=workload-identity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"workload-identity.d.ts","sourceRoot":"","sources":["../../src/provenance/workload-identity.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAKnD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,gBAAgB,CAsC3E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/runner/executor.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAIV,UAAU,EACX,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAwE/C;;GAEG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CAyCrB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAOH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAErD;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAsIzE;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASlF"}
|
package/dist/runner/types.d.ts
CHANGED
|
@@ -52,6 +52,8 @@ export interface RunResult {
|
|
|
52
52
|
export interface ExecutorContext {
|
|
53
53
|
client: ModelClient;
|
|
54
54
|
scenario: Scenario;
|
|
55
|
+
/** Effective model selected outside the scenario, such as a CLI override. */
|
|
56
|
+
requestedModel?: string;
|
|
55
57
|
timeout?: number;
|
|
56
58
|
retries?: number;
|
|
57
59
|
/** Redaction configuration for this execution */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ,CAAC;IACnB,0BAA0B;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,4DAA4D;IAC5D,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oCAAoC;IACpC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5E,oCAAoC;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,6EAA6E;IAC7E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B"}
|