@artemiskit/sdk 0.4.4 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  # @artemiskit/sdk
2
2
 
3
+ ## 0.4.6
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies
8
+ - @artemiskit/core@0.5.3
9
+ - @artemiskit/redteam@0.3.7
10
+
11
+ ## 0.4.5
12
+
13
+ ### Patch Changes
14
+
15
+ - Updated dependencies
16
+ - @artemiskit/core@0.5.2
17
+ - @artemiskit/redteam@0.3.6
18
+
3
19
  ## 0.4.4
4
20
 
5
21
  ### Patch Changes
@@ -1 +1 @@
1
- {"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuCH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAuBzC;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA6F9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
1
+ {"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyCH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAuBzC;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA+DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA2H9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
package/dist/index.js CHANGED
@@ -26484,7 +26484,7 @@ var TestCaseSchema = objectType({
26484
26484
  tags: arrayType(stringType()).optional().default([]),
26485
26485
  metadata: recordType(unknownType()).optional().default({}),
26486
26486
  timeout: numberType().optional(),
26487
- retries: numberType().optional().default(0),
26487
+ retries: numberType().int().min(0).max(99).optional().default(0),
26488
26488
  provider: ProviderSchema.optional(),
26489
26489
  model: stringType().optional(),
26490
26490
  variables: VariablesSchema.optional(),
@@ -27129,17 +27129,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
27129
27129
  }
27130
27130
  async function executeCase(testCase, context) {
27131
27131
  const { timeout, retries = 0 } = context;
27132
+ if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
27133
+ throw new RangeError("retries must be a whole number between 0 and 99");
27134
+ }
27132
27135
  const caseStartTime = Date.now();
27133
27136
  const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
27137
+ const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
27138
+ const repetitionIndex = context.repetition?.index ?? 1;
27139
+ const attemptEvidence = [];
27134
27140
  let lastError = null;
27135
27141
  for (let attempt = 0;attempt <= retries; attempt++) {
27142
+ const attemptStartTime = Date.now();
27136
27143
  try {
27137
27144
  const result = await executeCaseAttempt(testCase, context, timeout);
27138
- return { ...result, attempts: attempt + 1 };
27145
+ return withAttemptEvidence(result, attemptEvidence, {
27146
+ retryChainId,
27147
+ repetitionIndex,
27148
+ attemptNumber: attempt + 1,
27149
+ includedInOutcome: true,
27150
+ latencyMs: result.latencyMs
27151
+ });
27139
27152
  } catch (error) {
27140
27153
  lastError = error;
27141
- if (error instanceof ToolLoopError)
27142
- return { ...error.caseResult, attempts: attempt + 1 };
27154
+ if (error instanceof ToolLoopError) {
27155
+ return withAttemptEvidence(error.caseResult, attemptEvidence, {
27156
+ retryChainId,
27157
+ repetitionIndex,
27158
+ attemptNumber: attempt + 1,
27159
+ includedInOutcome: true,
27160
+ latencyMs: error.caseResult.latencyMs,
27161
+ errorCode: "tool_error"
27162
+ });
27163
+ }
27164
+ attemptEvidence.push({
27165
+ attempt_id: `${retryChainId}:${attempt + 1}`,
27166
+ retry_chain_id: retryChainId,
27167
+ repetition_index: repetitionIndex,
27168
+ attempt_number: attempt + 1,
27169
+ status: "error",
27170
+ included_in_outcome: false,
27171
+ latency_ms: Date.now() - attemptStartTime,
27172
+ error_code: error instanceof TimeoutError ? "timeout" : "target_error"
27173
+ });
27143
27174
  if (attempt < retries) {
27144
27175
  await sleep(2 ** attempt * 1000);
27145
27176
  }
@@ -27152,6 +27183,10 @@ async function executeCase(testCase, context) {
27152
27183
  ok: false,
27153
27184
  status: "error",
27154
27185
  attempts: retries + 1,
27186
+ attempt_evidence: attemptEvidence.map((entry, index) => ({
27187
+ ...entry,
27188
+ included_in_outcome: index === attemptEvidence.length - 1
27189
+ })),
27155
27190
  score: 0,
27156
27191
  matcherType: testCase.expected.type,
27157
27192
  reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
@@ -27165,6 +27200,28 @@ async function executeCase(testCase, context) {
27165
27200
  target: targetEvidence(context.client.provider, requestedModel)
27166
27201
  };
27167
27202
  }
27203
+ function withAttemptEvidence(result, priorAttempts, input) {
27204
+ return {
27205
+ ...result,
27206
+ attempts: input.attemptNumber,
27207
+ attempt_evidence: [
27208
+ ...priorAttempts,
27209
+ {
27210
+ attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
27211
+ retry_chain_id: input.retryChainId,
27212
+ repetition_index: input.repetitionIndex,
27213
+ attempt_number: input.attemptNumber,
27214
+ status: getTerminalStatus(result),
27215
+ included_in_outcome: input.includedInOutcome,
27216
+ latency_ms: input.latencyMs,
27217
+ ...input.errorCode ? { error_code: input.errorCode } : {}
27218
+ }
27219
+ ]
27220
+ };
27221
+ }
27222
+ function getTerminalStatus(result) {
27223
+ return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
27224
+ }
27168
27225
  async function executeCaseAttempt(testCase, context, timeout) {
27169
27226
  const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
27170
27227
  const variables = mergeVariables(scenario.variables, testCase.variables);
@@ -27526,302 +27583,6 @@ function nanoid(size = 21) {
27526
27583
  }
27527
27584
  return id;
27528
27585
  }
27529
- var MODEL_PRICING = {
27530
- "gpt-5": {
27531
- promptPer1K: 0.00125,
27532
- completionPer1K: 0.01,
27533
- lastUpdated: "2026-01",
27534
- notes: "400K context window"
27535
- },
27536
- "gpt-5.1": {
27537
- promptPer1K: 0.00125,
27538
- completionPer1K: 0.01,
27539
- lastUpdated: "2026-01"
27540
- },
27541
- "gpt-5.2": {
27542
- promptPer1K: 0.00175,
27543
- completionPer1K: 0.014,
27544
- lastUpdated: "2026-01"
27545
- },
27546
- "gpt-5-mini": {
27547
- promptPer1K: 0.00025,
27548
- completionPer1K: 0.002,
27549
- lastUpdated: "2026-01"
27550
- },
27551
- "gpt-5-nano": {
27552
- promptPer1K: 0.00005,
27553
- completionPer1K: 0.0004,
27554
- lastUpdated: "2026-01"
27555
- },
27556
- "gpt-4.1": {
27557
- promptPer1K: 0.002,
27558
- completionPer1K: 0.008,
27559
- lastUpdated: "2026-01",
27560
- notes: "1M context window"
27561
- },
27562
- "gpt-4.1-mini": {
27563
- promptPer1K: 0.0004,
27564
- completionPer1K: 0.0016,
27565
- lastUpdated: "2026-01"
27566
- },
27567
- "gpt-4.1-nano": {
27568
- promptPer1K: 0.0001,
27569
- completionPer1K: 0.0004,
27570
- lastUpdated: "2026-01"
27571
- },
27572
- "gpt-4o": {
27573
- promptPer1K: 0.0025,
27574
- completionPer1K: 0.01,
27575
- lastUpdated: "2026-01",
27576
- notes: "128K context window"
27577
- },
27578
- "gpt-4o-mini": {
27579
- promptPer1K: 0.00015,
27580
- completionPer1K: 0.0006,
27581
- lastUpdated: "2026-01",
27582
- notes: "128K context window"
27583
- },
27584
- o1: {
27585
- promptPer1K: 0.015,
27586
- completionPer1K: 0.06,
27587
- lastUpdated: "2026-01",
27588
- notes: "Reasoning model - internal thinking tokens billed as output"
27589
- },
27590
- o3: {
27591
- promptPer1K: 0.002,
27592
- completionPer1K: 0.008,
27593
- lastUpdated: "2026-01"
27594
- },
27595
- "o3-mini": {
27596
- promptPer1K: 0.0011,
27597
- completionPer1K: 0.0044,
27598
- lastUpdated: "2026-01"
27599
- },
27600
- "o4-mini": {
27601
- promptPer1K: 0.0011,
27602
- completionPer1K: 0.0044,
27603
- lastUpdated: "2026-01"
27604
- },
27605
- "gpt-4-turbo": {
27606
- promptPer1K: 0.01,
27607
- completionPer1K: 0.03,
27608
- lastUpdated: "2026-01"
27609
- },
27610
- "gpt-4": {
27611
- promptPer1K: 0.03,
27612
- completionPer1K: 0.06,
27613
- lastUpdated: "2026-01"
27614
- },
27615
- "gpt-3.5-turbo": {
27616
- promptPer1K: 0.0005,
27617
- completionPer1K: 0.0015,
27618
- lastUpdated: "2026-01"
27619
- },
27620
- "claude-opus-4.5": {
27621
- promptPer1K: 0.005,
27622
- completionPer1K: 0.025,
27623
- lastUpdated: "2026-01",
27624
- notes: "Most capable Claude model"
27625
- },
27626
- "claude-sonnet-4.5": {
27627
- promptPer1K: 0.003,
27628
- completionPer1K: 0.015,
27629
- lastUpdated: "2026-01",
27630
- notes: "Balanced performance and cost"
27631
- },
27632
- "claude-haiku-4.5": {
27633
- promptPer1K: 0.001,
27634
- completionPer1K: 0.005,
27635
- lastUpdated: "2026-01",
27636
- notes: "Fastest Claude model"
27637
- },
27638
- "claude-opus-4": {
27639
- promptPer1K: 0.015,
27640
- completionPer1K: 0.075,
27641
- lastUpdated: "2026-01"
27642
- },
27643
- "claude-opus-4.1": {
27644
- promptPer1K: 0.015,
27645
- completionPer1K: 0.075,
27646
- lastUpdated: "2026-01"
27647
- },
27648
- "claude-sonnet-4": {
27649
- promptPer1K: 0.003,
27650
- completionPer1K: 0.015,
27651
- lastUpdated: "2026-01"
27652
- },
27653
- "claude-sonnet-3.7": {
27654
- promptPer1K: 0.003,
27655
- completionPer1K: 0.015,
27656
- lastUpdated: "2026-01"
27657
- },
27658
- "claude-3-7-sonnet": {
27659
- promptPer1K: 0.003,
27660
- completionPer1K: 0.015,
27661
- lastUpdated: "2026-01"
27662
- },
27663
- "claude-3-5-sonnet-20241022": {
27664
- promptPer1K: 0.003,
27665
- completionPer1K: 0.015,
27666
- lastUpdated: "2026-01"
27667
- },
27668
- "claude-3-5-haiku-20241022": {
27669
- promptPer1K: 0.0008,
27670
- completionPer1K: 0.004,
27671
- lastUpdated: "2026-01"
27672
- },
27673
- "claude-haiku-3.5": {
27674
- promptPer1K: 0.0008,
27675
- completionPer1K: 0.004,
27676
- lastUpdated: "2026-01"
27677
- },
27678
- "claude-3-opus": {
27679
- promptPer1K: 0.015,
27680
- completionPer1K: 0.075,
27681
- lastUpdated: "2026-01"
27682
- },
27683
- "claude-3-sonnet": {
27684
- promptPer1K: 0.003,
27685
- completionPer1K: 0.015,
27686
- lastUpdated: "2026-01"
27687
- },
27688
- "claude-3-haiku": {
27689
- promptPer1K: 0.00025,
27690
- completionPer1K: 0.00125,
27691
- lastUpdated: "2026-01"
27692
- },
27693
- "claude-3.5-sonnet": {
27694
- promptPer1K: 0.003,
27695
- completionPer1K: 0.015,
27696
- lastUpdated: "2026-01"
27697
- },
27698
- "claude-3.5-haiku": {
27699
- promptPer1K: 0.0008,
27700
- completionPer1K: 0.004,
27701
- lastUpdated: "2026-01"
27702
- }
27703
- };
27704
- var DEFAULT_PRICING = {
27705
- promptPer1K: 0.003,
27706
- completionPer1K: 0.015,
27707
- lastUpdated: "2026-01",
27708
- notes: "Default pricing - verify with provider"
27709
- };
27710
- function getModelPricing(model) {
27711
- if (MODEL_PRICING[model]) {
27712
- return MODEL_PRICING[model];
27713
- }
27714
- const lowerModel = model.toLowerCase();
27715
- for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
27716
- if (key.toLowerCase() === lowerModel) {
27717
- return pricing;
27718
- }
27719
- }
27720
- if (lowerModel.includes("gpt-5.2")) {
27721
- return MODEL_PRICING["gpt-5.2"];
27722
- }
27723
- if (lowerModel.includes("gpt-5.1")) {
27724
- return MODEL_PRICING["gpt-5.1"];
27725
- }
27726
- if (lowerModel.includes("gpt-5-mini")) {
27727
- return MODEL_PRICING["gpt-5-mini"];
27728
- }
27729
- if (lowerModel.includes("gpt-5-nano")) {
27730
- return MODEL_PRICING["gpt-5-nano"];
27731
- }
27732
- if (lowerModel.includes("gpt-5")) {
27733
- return MODEL_PRICING["gpt-5"];
27734
- }
27735
- if (lowerModel.includes("gpt-4.1-mini")) {
27736
- return MODEL_PRICING["gpt-4.1-mini"];
27737
- }
27738
- if (lowerModel.includes("gpt-4.1-nano")) {
27739
- return MODEL_PRICING["gpt-4.1-nano"];
27740
- }
27741
- if (lowerModel.includes("gpt-4.1")) {
27742
- return MODEL_PRICING["gpt-4.1"];
27743
- }
27744
- if (lowerModel.includes("gpt-4o-mini")) {
27745
- return MODEL_PRICING["gpt-4o-mini"];
27746
- }
27747
- if (lowerModel.includes("gpt-4o")) {
27748
- return MODEL_PRICING["gpt-4o"];
27749
- }
27750
- if (lowerModel.includes("o4-mini")) {
27751
- return MODEL_PRICING["o4-mini"];
27752
- }
27753
- if (lowerModel.includes("o3-mini")) {
27754
- return MODEL_PRICING["o3-mini"];
27755
- }
27756
- if (lowerModel.includes("o3")) {
27757
- return MODEL_PRICING.o3;
27758
- }
27759
- if (lowerModel.includes("o1")) {
27760
- return MODEL_PRICING.o1;
27761
- }
27762
- if (lowerModel.includes("gpt-4-turbo")) {
27763
- return MODEL_PRICING["gpt-4-turbo"];
27764
- }
27765
- if (lowerModel.includes("gpt-4")) {
27766
- return MODEL_PRICING["gpt-4"];
27767
- }
27768
- if (lowerModel.includes("gpt-3.5")) {
27769
- return MODEL_PRICING["gpt-3.5-turbo"];
27770
- }
27771
- if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
27772
- return MODEL_PRICING["claude-opus-4.5"];
27773
- }
27774
- if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
27775
- return MODEL_PRICING["claude-sonnet-4.5"];
27776
- }
27777
- if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
27778
- return MODEL_PRICING["claude-haiku-4.5"];
27779
- }
27780
- if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
27781
- return MODEL_PRICING["claude-opus-4.1"];
27782
- }
27783
- if (lowerModel.includes("opus-4")) {
27784
- return MODEL_PRICING["claude-opus-4"];
27785
- }
27786
- if (lowerModel.includes("sonnet-4")) {
27787
- return MODEL_PRICING["claude-sonnet-4"];
27788
- }
27789
- if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
27790
- return MODEL_PRICING["claude-sonnet-3.7"];
27791
- }
27792
- if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
27793
- return MODEL_PRICING["claude-3.5-sonnet"];
27794
- }
27795
- if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
27796
- return MODEL_PRICING["claude-3.5-haiku"];
27797
- }
27798
- if (lowerModel.includes("claude-3-opus")) {
27799
- return MODEL_PRICING["claude-3-opus"];
27800
- }
27801
- if (lowerModel.includes("claude-3-sonnet")) {
27802
- return MODEL_PRICING["claude-3-sonnet"];
27803
- }
27804
- if (lowerModel.includes("claude-3-haiku")) {
27805
- return MODEL_PRICING["claude-3-haiku"];
27806
- }
27807
- if (lowerModel.includes("claude")) {
27808
- return MODEL_PRICING["claude-sonnet-4.5"];
27809
- }
27810
- return DEFAULT_PRICING;
27811
- }
27812
- function estimateCost(promptTokens, completionTokens, model) {
27813
- const pricing = getModelPricing(model);
27814
- const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
27815
- const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
27816
- const totalUsd = promptCostUsd + completionCostUsd;
27817
- return {
27818
- totalUsd,
27819
- promptCostUsd,
27820
- completionCostUsd,
27821
- model,
27822
- pricing
27823
- };
27824
- }
27825
27586
  function getEnvironmentInfo() {
27826
27587
  return {
27827
27588
  node_version: process.version,
@@ -27880,6 +27641,12 @@ function assertRunManifestIntegrity(manifest) {
27880
27641
  if (manifest.execution_provenance !== undefined) {
27881
27642
  assertExecutionProvenance(manifest.execution_provenance);
27882
27643
  }
27644
+ if (manifest.attempt_evidence !== undefined) {
27645
+ assertRunAttemptEvidence(manifest.attempt_evidence);
27646
+ }
27647
+ if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
27648
+ assertCostProvenance(manifest.metrics.cost_provenance);
27649
+ }
27883
27650
  for (const [index, caseResult] of manifest.cases.entries()) {
27884
27651
  if (!isRecord2(caseResult)) {
27885
27652
  throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -27893,7 +27660,60 @@ function assertRunManifestIntegrity(manifest) {
27893
27660
  if (caseResult.target !== undefined) {
27894
27661
  assertCaseTargetEvidence(caseResult.target);
27895
27662
  }
27663
+ if (caseResult.attempt_evidence !== undefined) {
27664
+ assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
27665
+ }
27666
+ }
27667
+ }
27668
+ function assertRunAttemptEvidence(evidence) {
27669
+ if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
27670
+ throw new Error("Invalid run manifest: malformed attempt evidence");
27671
+ }
27672
+ }
27673
+ function assertCaseAttemptEvidence(evidence, index) {
27674
+ if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
27675
+ throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
27676
+ }
27677
+ for (const attempt of evidence) {
27678
+ if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
27679
+ throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
27680
+ }
27681
+ }
27682
+ }
27683
+ function assertCostProvenance(cost) {
27684
+ if (!isRecord2(cost) || cost.schema_version !== "1") {
27685
+ throw new Error("Invalid run manifest: malformed cost provenance");
27686
+ }
27687
+ if (cost.status === "unavailable") {
27688
+ if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
27689
+ throw new Error("Invalid run manifest: malformed cost provenance");
27690
+ }
27691
+ return;
27896
27692
  }
27693
+ if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
27694
+ throw new Error("Invalid run manifest: malformed cost provenance");
27695
+ }
27696
+ }
27697
+ function isCaseEvaluationStatus(value) {
27698
+ return value === "passed" || value === "failed" || value === "invalid" || value === "error";
27699
+ }
27700
+ function isBoundedNonemptyString(value, maxLength) {
27701
+ return typeof value === "string" && value.length > 0 && value.length <= maxLength;
27702
+ }
27703
+ function isNonnegativeFiniteNumber(value) {
27704
+ return typeof value === "number" && Number.isFinite(value) && value >= 0;
27705
+ }
27706
+ function isPositiveFiniteNumber(value) {
27707
+ return isNonnegativeFiniteNumber(value) && value > 0;
27708
+ }
27709
+ function isNonnegativeSafeInteger(value) {
27710
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
27711
+ }
27712
+ function isPositiveSafeInteger(value) {
27713
+ return isNonnegativeSafeInteger(value) && value > 0;
27714
+ }
27715
+ function isIsoTimestamp(value) {
27716
+ return typeof value === "string" && Number.isFinite(Date.parse(value));
27897
27717
  }
27898
27718
  function assertCaseTargetEvidence(target) {
27899
27719
  if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
@@ -27971,6 +27791,9 @@ function createRunManifest(options) {
27971
27791
  resolvedConfig,
27972
27792
  workloadIdentity,
27973
27793
  executionProvenance,
27794
+ attemptEvidence,
27795
+ costProvenance,
27796
+ runId,
27974
27797
  cases,
27975
27798
  startTime,
27976
27799
  endTime,
@@ -27978,13 +27801,12 @@ function createRunManifest(options) {
27978
27801
  runReason,
27979
27802
  redaction
27980
27803
  } = options;
27981
- const modelForCost = resolvedConfig?.model || config.model;
27982
- const metrics = calculateMetrics(cases, modelForCost);
27804
+ const metrics = calculateMetrics(cases, costProvenance);
27983
27805
  const git = getGitInfo();
27984
27806
  const environment = getEnvironmentInfo();
27985
27807
  return {
27986
- version: "1.3",
27987
- run_id: nanoid(12),
27808
+ version: "1.4",
27809
+ run_id: runId ?? nanoid(12),
27988
27810
  project,
27989
27811
  start_time: startTime.toISOString(),
27990
27812
  end_time: endTime.toISOString(),
@@ -27993,6 +27815,7 @@ function createRunManifest(options) {
27993
27815
  resolved_config: resolvedConfig,
27994
27816
  workload_identity: workloadIdentity,
27995
27817
  execution_provenance: executionProvenance,
27818
+ attempt_evidence: attemptEvidence,
27996
27819
  metrics,
27997
27820
  git,
27998
27821
  provenance: {
@@ -28005,7 +27828,7 @@ function createRunManifest(options) {
28005
27828
  redaction
28006
27829
  };
28007
27830
  }
28008
- function calculateMetrics(cases, model) {
27831
+ function calculateMetrics(cases, costProvenance) {
28009
27832
  const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
28010
27833
  const validCases = cases.filter((c) => {
28011
27834
  const status = getCaseEvaluationStatus(c);
@@ -28017,21 +27840,11 @@ function calculateMetrics(cases, model) {
28017
27840
  const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
28018
27841
  const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
28019
27842
  const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
28020
- let cost;
28021
- if (model && !model.toLowerCase().includes("ling-") && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
28022
- const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
28023
- const pricing = getModelPricing(model);
28024
- cost = {
28025
- total_usd: costEstimate.totalUsd,
28026
- prompt_cost_usd: costEstimate.promptCostUsd,
28027
- completion_cost_usd: costEstimate.completionCostUsd,
28028
- model: costEstimate.model,
28029
- pricing: {
28030
- prompt_per_1k: pricing.promptPer1K,
28031
- completion_per_1k: pricing.completionPer1K
28032
- }
28033
- };
28034
- }
27843
+ const cost_provenance = costProvenance ?? {
27844
+ schema_version: "1",
27845
+ status: "unavailable",
27846
+ unavailable_reason: "provider_billing_not_recorded"
27847
+ };
28035
27848
  return {
28036
27849
  success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
28037
27850
  total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
@@ -28046,7 +27859,7 @@ function calculateMetrics(cases, model) {
28046
27859
  total_tokens: totalPromptTokens + totalCompletionTokens,
28047
27860
  total_prompt_tokens: totalPromptTokens,
28048
27861
  total_completion_tokens: totalCompletionTokens,
28049
- cost
27862
+ cost_provenance
28050
27863
  };
28051
27864
  }
28052
27865
  function detectCIEnvironment() {
@@ -28185,6 +27998,8 @@ async function runScenario(options) {
28185
27998
  concurrency = 1,
28186
27999
  timeout,
28187
28000
  retries,
28001
+ repetition = { index: 1, total: 1 },
28002
+ costProvenance,
28188
28003
  redaction,
28189
28004
  toolExecutor,
28190
28005
  onCaseComplete,
@@ -28200,6 +28015,7 @@ async function runScenario(options) {
28200
28015
  }
28201
28016
  onProgress?.(`Running ${cases.length} test cases...`);
28202
28017
  const startTime = new Date;
28018
+ const runId = nanoid(12);
28203
28019
  const results = [];
28204
28020
  if (concurrency === 1) {
28205
28021
  for (let i = 0;i < cases.length; i++) {
@@ -28210,6 +28026,8 @@ async function runScenario(options) {
28210
28026
  requestedModel: resolvedConfig?.model,
28211
28027
  timeout: testCase.timeout || timeout,
28212
28028
  retries: testCase.retries ?? retries,
28029
+ runId,
28030
+ repetition,
28213
28031
  redaction,
28214
28032
  toolExecutor
28215
28033
  });
@@ -28227,6 +28045,8 @@ async function runScenario(options) {
28227
28045
  requestedModel: resolvedConfig?.model,
28228
28046
  timeout: testCase.timeout || timeout,
28229
28047
  retries: testCase.retries ?? retries,
28048
+ runId,
28049
+ repetition,
28230
28050
  redaction,
28231
28051
  toolExecutor
28232
28052
  });
@@ -28277,9 +28097,21 @@ async function runScenario(options) {
28277
28097
  seed: scenario.seed,
28278
28098
  cases: results
28279
28099
  }),
28100
+ attemptEvidence: {
28101
+ schema_version: "1",
28102
+ repetition,
28103
+ retry_policy: {
28104
+ default_max_retries: retries ?? 0,
28105
+ backoff: "exponential",
28106
+ initial_delay_ms: 1000
28107
+ },
28108
+ ...timeout ? { timeout: { default_ms: timeout } } : {}
28109
+ },
28110
+ costProvenance,
28280
28111
  cases: results,
28281
28112
  startTime,
28282
28113
  endTime,
28114
+ runId,
28283
28115
  redaction: redactionInfo
28284
28116
  });
28285
28117
  const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
@@ -28296,6 +28128,62 @@ function chunkArray(array, size) {
28296
28128
  }
28297
28129
  return chunks;
28298
28130
  }
28131
+ function assessComparisonEligibility(baseline, current) {
28132
+ const reasons = [];
28133
+ if (baseline.config.scenario !== current.config.scenario) {
28134
+ reasons.push({ code: "scenario_mismatch" });
28135
+ }
28136
+ const baselineIdentity = baseline.workload_identity;
28137
+ const currentIdentity = current.workload_identity;
28138
+ if (!baselineIdentity || !currentIdentity) {
28139
+ if (!baselineIdentity || !currentIdentity)
28140
+ reasons.push({ code: "workload_identity_missing" });
28141
+ if (!baselineIdentity || !currentIdentity)
28142
+ reasons.push({ code: "rubric_identity_missing" });
28143
+ } else {
28144
+ if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
28145
+ reasons.push({ code: "workload_mismatch" });
28146
+ }
28147
+ if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
28148
+ reasons.push({ code: "rubric_mismatch" });
28149
+ }
28150
+ }
28151
+ if (reasons.some((reason) => isIncomparableReason(reason.code))) {
28152
+ return { schema_version: "1", status: "incomparable", reasons };
28153
+ }
28154
+ const baselineExecution = baseline.execution_provenance;
28155
+ const currentExecution = current.execution_provenance;
28156
+ if (!baselineExecution || !currentExecution) {
28157
+ reasons.push({ code: "execution_provenance_missing" });
28158
+ } else {
28159
+ if (baselineExecution.target.provider !== currentExecution.target.provider) {
28160
+ reasons.push({ code: "target_provider_changed" });
28161
+ }
28162
+ if (!sameStringSet(baselineExecution.target.requested_models, currentExecution.target.requested_models)) {
28163
+ reasons.push({ code: "target_model_changed" });
28164
+ }
28165
+ if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
28166
+ reasons.push({ code: "generation_settings_changed" });
28167
+ }
28168
+ }
28169
+ return {
28170
+ schema_version: "1",
28171
+ status: reasons.length === 0 ? "compatible" : "qualified",
28172
+ reasons
28173
+ };
28174
+ }
28175
+ function isComparisonAvailable(eligibility) {
28176
+ return eligibility.status !== "incomparable";
28177
+ }
28178
+ function isIncomparableReason(code) {
28179
+ return code === "scenario_mismatch" || code === "workload_mismatch" || code === "rubric_mismatch";
28180
+ }
28181
+ function sameStringSet(left, right) {
28182
+ return JSON.stringify([...left ?? []].sort()) === JSON.stringify([...right ?? []].sort());
28183
+ }
28184
+ function sameGeneration(left, right) {
28185
+ return left?.temperature === right?.temperature && left?.max_tokens === right?.max_tokens && left?.seed === right?.seed;
28186
+ }
28299
28187
  function getManifestType(manifest) {
28300
28188
  if ("type" in manifest) {
28301
28189
  if (manifest.type === "redteam")
@@ -28435,9 +28323,14 @@ class LocalStorageAdapter {
28435
28323
  this.loadRun(baselineId),
28436
28324
  this.loadRun(currentId)
28437
28325
  ]);
28326
+ const eligibility = assessComparisonEligibility(baseline, current);
28327
+ if (!isComparisonAvailable(eligibility)) {
28328
+ return { baseline, current, eligibility };
28329
+ }
28438
28330
  return {
28439
28331
  baseline,
28440
28332
  current,
28333
+ eligibility,
28441
28334
  delta: {
28442
28335
  successRate: current.metrics.success_rate - baseline.metrics.success_rate,
28443
28336
  latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -28536,7 +28429,7 @@ class LocalStorageAdapter {
28536
28429
  return null;
28537
28430
  }
28538
28431
  const comparison = await this.compare(baseline.runId, runId);
28539
- const hasRegression = comparison.delta.successRate < -regressionThreshold;
28432
+ const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
28540
28433
  return {
28541
28434
  baseline,
28542
28435
  comparison,
@@ -37127,9 +37020,14 @@ class SupabaseStorageAdapter {
37127
37020
  }
37128
37021
  async compare(baselineId, currentId) {
37129
37022
  const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
37023
+ const eligibility = assessComparisonEligibility(baseline, current);
37024
+ if (!isComparisonAvailable(eligibility)) {
37025
+ return { baseline, current, eligibility };
37026
+ }
37130
37027
  return {
37131
37028
  baseline,
37132
37029
  current,
37030
+ eligibility,
37133
37031
  delta: {
37134
37032
  successRate: current.metrics.success_rate - baseline.metrics.success_rate,
37135
37033
  latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -37260,7 +37158,7 @@ class SupabaseStorageAdapter {
37260
37158
  return null;
37261
37159
  }
37262
37160
  const comparison = await this.compare(baseline.runId, runId);
37263
- const hasRegression = comparison.delta.successRate < -regressionThreshold;
37161
+ const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
37264
37162
  return {
37265
37163
  baseline,
37266
37164
  comparison,
@@ -51643,6 +51541,29 @@ class ArtemisKit {
51643
51541
  });
51644
51542
  const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
51645
51543
  const currentMetrics = this.extractRunMetrics(currentManifest, current);
51544
+ const eligibility = assessComparisonEligibility(baselineManifest, currentManifest);
51545
+ if (!isComparisonAvailable(eligibility)) {
51546
+ this.emit("progress", {
51547
+ message: `Comparison unavailable: ${eligibility.reasons.map((reason) => reason.code).join(", ")}`,
51548
+ phase: "teardown",
51549
+ progress: 100
51550
+ });
51551
+ return {
51552
+ baseline: baselineMetrics,
51553
+ current: currentMetrics,
51554
+ comparison: {
51555
+ comparisonAvailable: false,
51556
+ newFailures: [],
51557
+ newPasses: [],
51558
+ unchanged: [],
51559
+ addedCases: [],
51560
+ removedCases: []
51561
+ },
51562
+ eligibility,
51563
+ hasRegression: false,
51564
+ threshold
51565
+ };
51566
+ }
51646
51567
  const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
51647
51568
  this.emit("progress", {
51648
51569
  message: "Comparing cases...",
@@ -51660,9 +51581,11 @@ class ArtemisKit {
51660
51581
  baseline: baselineMetrics,
51661
51582
  current: currentMetrics,
51662
51583
  comparison: {
51584
+ comparisonAvailable: true,
51663
51585
  successRateDelta,
51664
51586
  ...caseComparison
51665
51587
  },
51588
+ eligibility,
51666
51589
  hasRegression,
51667
51590
  threshold
51668
51591
  };
package/dist/types.d.ts CHANGED
@@ -2,7 +2,7 @@
2
2
  * @artemiskit/sdk
3
3
  * Type definitions for the programmatic SDK
4
4
  */
5
- import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
5
+ import type { AdapterConfig, CaseResult, ComparisonEligibility, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
6
6
  import type { RedactionConfig } from '@artemiskit/core';
7
7
  import type { Scenario } from '@artemiskit/core';
8
8
  /**
@@ -327,8 +327,10 @@ export interface RunSummary {
327
327
  * Comparison details between two runs
328
328
  */
329
329
  export interface ComparisonDetails {
330
- /** Change in success rate (current - baseline) */
331
- successRateDelta: number;
330
+ /** True when the runs are eligible for a metric/case delta. */
331
+ comparisonAvailable: boolean;
332
+ /** Change in success rate (current - baseline), absent for incomparable runs. */
333
+ successRateDelta?: number;
332
334
  /** Cases that passed in baseline but failed in current */
333
335
  newFailures: Array<{
334
336
  caseId: string;
@@ -371,7 +373,9 @@ export interface CompareResult {
371
373
  current: RunSummary;
372
374
  /** Detailed comparison */
373
375
  comparison: ComparisonDetails;
374
- /** Whether a regression was detected */
376
+ /** Versioned compatibility decision that governs the comparison. */
377
+ eligibility: ComparisonEligibility;
378
+ /** Whether a regression was detected; always false for incomparable runs. */
375
379
  hasRegression: boolean;
376
380
  /** Threshold used for regression detection */
377
381
  threshold: number;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,wCAAwC;IACxC,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,qBAAqB,EACrB,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,mBAAmB,EAAE,OAAO,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,oEAAoE;IACpE,WAAW,EAAE,qBAAqB,CAAC;IACnC,6EAA6E;IAC7E,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.4.4",
3
+ "version": "0.4.6",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -71,8 +71,8 @@
71
71
  "test:coverage": "bun test --coverage"
72
72
  },
73
73
  "dependencies": {
74
- "@artemiskit/core": "0.5.1",
75
- "@artemiskit/redteam": "0.3.5"
74
+ "@artemiskit/core": "0.5.3",
75
+ "@artemiskit/redteam": "0.3.7"
76
76
  },
77
77
  "devDependencies": {
78
78
  "@types/bun": "^1.1.0",
package/src/artemiskit.ts CHANGED
@@ -15,10 +15,12 @@ import {
15
15
  type StressManifest,
16
16
  type StressMetrics,
17
17
  type StressRequestResult,
18
+ assessComparisonEligibility,
18
19
  runScenario as coreRunScenario,
19
20
  createAdapter,
20
21
  createStorageAdapter,
21
22
  getGitInfo,
23
+ isComparisonAvailable,
22
24
  parseScenarioFile,
23
25
  resolveScenarioPaths,
24
26
  } from '@artemiskit/core';
@@ -1025,6 +1027,34 @@ export class ArtemisKit {
1025
1027
  // Extract metrics from manifests
1026
1028
  const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
1027
1029
  const currentMetrics = this.extractRunMetrics(currentManifest, current);
1030
+ const eligibility = assessComparisonEligibility(
1031
+ baselineManifest as RunManifest,
1032
+ currentManifest as RunManifest
1033
+ );
1034
+
1035
+ if (!isComparisonAvailable(eligibility)) {
1036
+ this.emit('progress', {
1037
+ message: `Comparison unavailable: ${eligibility.reasons.map((reason) => reason.code).join(', ')}`,
1038
+ phase: 'teardown',
1039
+ progress: 100,
1040
+ });
1041
+
1042
+ return {
1043
+ baseline: baselineMetrics,
1044
+ current: currentMetrics,
1045
+ comparison: {
1046
+ comparisonAvailable: false,
1047
+ newFailures: [],
1048
+ newPasses: [],
1049
+ unchanged: [],
1050
+ addedCases: [],
1051
+ removedCases: [],
1052
+ },
1053
+ eligibility,
1054
+ hasRegression: false,
1055
+ threshold,
1056
+ };
1057
+ }
1028
1058
 
1029
1059
  // Calculate success rate delta
1030
1060
  const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
@@ -1054,9 +1084,11 @@ export class ArtemisKit {
1054
1084
  baseline: baselineMetrics,
1055
1085
  current: currentMetrics,
1056
1086
  comparison: {
1087
+ comparisonAvailable: true,
1057
1088
  successRateDelta,
1058
1089
  ...caseComparison,
1059
1090
  },
1091
+ eligibility,
1060
1092
  hasRegression,
1061
1093
  threshold,
1062
1094
  };
package/src/types.ts CHANGED
@@ -6,6 +6,7 @@
6
6
  import type {
7
7
  AdapterConfig,
8
8
  CaseResult,
9
+ ComparisonEligibility,
9
10
  ModelClient,
10
11
  RedTeamManifest,
11
12
  RunManifest,
@@ -393,8 +394,10 @@ export interface RunSummary {
393
394
  * Comparison details between two runs
394
395
  */
395
396
  export interface ComparisonDetails {
396
- /** Change in success rate (current - baseline) */
397
- successRateDelta: number;
397
+ /** True when the runs are eligible for a metric/case delta. */
398
+ comparisonAvailable: boolean;
399
+ /** Change in success rate (current - baseline), absent for incomparable runs. */
400
+ successRateDelta?: number;
398
401
  /** Cases that passed in baseline but failed in current */
399
402
  newFailures: Array<{
400
403
  caseId: string;
@@ -438,7 +441,9 @@ export interface CompareResult {
438
441
  current: RunSummary;
439
442
  /** Detailed comparison */
440
443
  comparison: ComparisonDetails;
441
- /** Whether a regression was detected */
444
+ /** Versioned compatibility decision that governs the comparison. */
445
+ eligibility: ComparisonEligibility;
446
+ /** Whether a regression was detected; always false for incomparable runs. */
442
447
  hasRegression: boolean;
443
448
  /** Threshold used for regression detection */
444
449
  threshold: number;