@artemiskit/core 0.5.2 → 0.5.3

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @artemiskit/core
 
+## 0.5.3
+
+### Patch Changes
+
+- Qualify standard-run comparisons from bounded workload, rubric, and execution evidence. Changed
+  workloads or rubrics withhold deltas, while incomplete historical evidence and changed target
+  configuration remain explicitly qualified across storage, CLI, SDK, and reports.
+
 ## 0.5.2
 
 ### Patch Changes

package/dist/comparison/eligibility.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Compatibility decisions for comparisons of saved scenario-evaluation runs.
+ *
+ * This contract compares declared evidence only. A matching digest proves the
+ * same canonical workload/rubric was declared; it does not attest to provider
+ * behaviour or replace a future assessment-profile compatibility decision.
+ */
+import type { RunManifest } from '../artifacts/types';
+export type ComparisonEligibilityStatus = 'compatible' | 'qualified' | 'incomparable';
+export type ComparisonEligibilityReasonCode = 'scenario_mismatch' | 'workload_identity_missing' | 'rubric_identity_missing' | 'workload_mismatch' | 'rubric_mismatch' | 'execution_provenance_missing' | 'target_provider_changed' | 'target_model_changed' | 'generation_settings_changed';
+/** A bounded, machine-readable reason for a comparison decision. */
+export interface ComparisonEligibilityReason {
+    code: ComparisonEligibilityReasonCode;
+}
+/**
+ * Versioned decision that accompanies every comparison. `qualified` permits
+ * a visibly-qualified delta; `incomparable` prohibits a delta entirely.
+ */
+export interface ComparisonEligibility {
+    schema_version: '1';
+    status: ComparisonEligibilityStatus;
+    reasons: ComparisonEligibilityReason[];
+}
+export declare function assessComparisonEligibility(baseline: RunManifest, current: RunManifest): ComparisonEligibility;
+export declare function isComparisonAvailable(eligibility: ComparisonEligibility): boolean;
+//# sourceMappingURL=eligibility.d.ts.map

package/dist/comparison/eligibility.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eligibility.d.ts","sourceRoot":"","sources":["../../src/comparison/eligibility.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD,MAAM,MAAM,2BAA2B,GAAG,YAAY,GAAG,WAAW,GAAG,cAAc,CAAC;AAEtF,MAAM,MAAM,+BAA+B,GACvC,mBAAmB,GACnB,2BAA2B,GAC3B,yBAAyB,GACzB,mBAAmB,GACnB,iBAAiB,GACjB,8BAA8B,GAC9B,yBAAyB,GACzB,sBAAsB,GACtB,6BAA6B,CAAC;AAElC,oEAAoE;AACpE,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,+BAA+B,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,cAAc,EAAE,GAAG,CAAC;IACpB,MAAM,EAAE,2BAA2B,CAAC;IACpC,OAAO,EAAE,2BAA2B,EAAE,CAAC;CACxC;AAED,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,WAAW,GACnB,qBAAqB,CAmDvB;AAED,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,qBAAqB,GAAG,OAAO,CAEjF"}

package/dist/comparison/index.d.ts

@@ -0,0 +1,2 @@
+export { assessComparisonEligibility, isComparisonAvailable, type ComparisonEligibility, type ComparisonEligibilityReason, type ComparisonEligibilityReasonCode, type ComparisonEligibilityStatus, } from './eligibility';
+//# sourceMappingURL=index.d.ts.map

package/dist/comparison/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/comparison/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,2BAA2B,EAC3B,qBAAqB,EACrB,KAAK,qBAAqB,EAC1B,KAAK,2BAA2B,EAChC,KAAK,+BAA+B,EACpC,KAAK,2BAA2B,GACjC,MAAM,eAAe,CAAC"}

package/dist/index.d.ts

@@ -9,6 +9,7 @@ export * from './runner';
 export * from './storage';
 export * from './artifacts';
 export * from './provenance';
+export * from './comparison';
 export * from './utils';
 export * from './redaction';
 export * from './cost';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC;AAGvB,cAAc,SAAS,CAAC;AAGxB,cAAc,oBAAoB,CAAC;AAGnC,cAAc,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,YAAY,CAAC;AAG3B,cAAc,cAAc,CAAC;AAG7B,cAAc,UAAU,CAAC;AAGzB,cAAc,WAAW,CAAC;AAG1B,cAAc,aAAa,CAAC;AAG5B,cAAc,cAAc,CAAC;AAG7B,cAAc,cAAc,CAAC;AAG7B,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,QAAQ,CAAC;AAGvB,cAAc,SAAS,CAAC;AAGxB,cAAc,oBAAoB,CAAC;AAGnC,cAAc,aAAa,CAAC"}

package/dist/index.js

@@ -21562,6 +21562,65 @@ function chunkArray(array, size) {
 // src/storage/local.ts
 import { mkdir, readFile as readFile2, readdir as readdir2, unlink, writeFile } from "fs/promises";
 import { join as join2, resolve as resolve2 } from "path";
+
+// src/comparison/eligibility.ts
+function assessComparisonEligibility(baseline, current) {
+  const reasons = [];
+  if (baseline.config.scenario !== current.config.scenario) {
+    reasons.push({ code: "scenario_mismatch" });
+  }
+  const baselineIdentity = baseline.workload_identity;
+  const currentIdentity = current.workload_identity;
+  if (!baselineIdentity || !currentIdentity) {
+    if (!baselineIdentity || !currentIdentity)
+      reasons.push({ code: "workload_identity_missing" });
+    if (!baselineIdentity || !currentIdentity)
+      reasons.push({ code: "rubric_identity_missing" });
+  } else {
+    if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
+      reasons.push({ code: "workload_mismatch" });
+    }
+    if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
+      reasons.push({ code: "rubric_mismatch" });
+    }
+  }
+  if (reasons.some((reason) => isIncomparableReason(reason.code))) {
+    return { schema_version: "1", status: "incomparable", reasons };
+  }
+  const baselineExecution = baseline.execution_provenance;
+  const currentExecution = current.execution_provenance;
+  if (!baselineExecution || !currentExecution) {
+    reasons.push({ code: "execution_provenance_missing" });
+  } else {
+    if (baselineExecution.target.provider !== currentExecution.target.provider) {
+      reasons.push({ code: "target_provider_changed" });
+    }
+    if (!sameStringSet(baselineExecution.target.requested_models, currentExecution.target.requested_models)) {
+      reasons.push({ code: "target_model_changed" });
+    }
+    if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
+      reasons.push({ code: "generation_settings_changed" });
+    }
+  }
+  return {
+    schema_version: "1",
+    status: reasons.length === 0 ? "compatible" : "qualified",
+    reasons
+  };
+}
+function isComparisonAvailable(eligibility) {
+  return eligibility.status !== "incomparable";
+}
+function isIncomparableReason(code) {
+  return code === "scenario_mismatch" || code === "workload_mismatch" || code === "rubric_mismatch";
+}
+function sameStringSet(left, right) {
+  return JSON.stringify([...left ?? []].sort()) === JSON.stringify([...right ?? []].sort());
+}
+function sameGeneration(left, right) {
+  return left?.temperature === right?.temperature && left?.max_tokens === right?.max_tokens && left?.seed === right?.seed;
+}
+// src/storage/local.ts
 function getManifestType(manifest) {
   if ("type" in manifest) {
     if (manifest.type === "redteam")
@@ -21701,9 +21760,14 @@ class LocalStorageAdapter {
       this.loadRun(baselineId),
       this.loadRun(currentId)
     ]);
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -21802,7 +21866,7 @@ class LocalStorageAdapter {
       return null;
     }
     const comparison = await this.compare(baseline.runId, runId);
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
     return {
       baseline,
       comparison,
@@ -30448,9 +30512,14 @@ class SupabaseStorageAdapter {
   }
   async compare(baselineId, currentId) {
     const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -30581,7 +30650,7 @@ class SupabaseStorageAdapter {
       return null;
     }
     const comparison = await this.compare(baseline.runId, runId);
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
     return {
       baseline,
       comparison,
@@ -32924,6 +32993,7 @@ export {
   actionBudgetExceeded,
   adapterRegistry,
   assertRunManifestIntegrity,
+  assessComparisonEligibility,
   createAdapter,
   createDefaultRedactor,
   createExecutionProvenance,
@@ -32945,6 +33015,7 @@ export {
   getModelPricing,
   hashText,
   isArtemisError,
+  isComparisonAvailable,
   isRedTeamManifest,
   isRunManifest,
   isStressManifest,

package/dist/storage/local.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/storage/local.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,KAAK,cAAc,EAGpB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EACV,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,WAAW,EACX,WAAW,EACZ,MAAM,SAAS,CAAC;AAyDjB,qBAAa,mBAAoB,YAAW,sBAAsB;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;gBAElB,QAAQ,SAAmB;IAKjC,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAa5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAczC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAS5C,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQpD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAQlD,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA0DnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAiBjE,eAAe;YASf,SAAS;IAWvB;;OAEG;YACW,iBAAiB;IAS/B;;OAEG;YACW,iBAAiB;IAM/B;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4B3F;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKrE;;OAEG;IACG,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAMzE;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAOlD;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUxD;;OAEG;IACG,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW5D;;OAEG;IACG,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;CAwBV"}
+{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/storage/local.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,KAAK,cAAc,EAGpB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EACV,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,WAAW,EACX,WAAW,EACZ,MAAM,SAAS,CAAC;AAyDjB,qBAAa,mBAAoB,YAAW,sBAAsB;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;gBAElB,QAAQ,SAAmB;IAKjC,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAa5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAczC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAS5C,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQpD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAQlD,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA0DnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAuBjE,eAAe;YASf,SAAS;IAWvB;;OAEG;YACW,iBAAiB;IAS/B;;OAEG;YACW,iBAAiB;IAM/B;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4B3F;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKrE;;OAEG;IACG,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAMzE;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAOlD;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUxD;;OAEG;IACG,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAW5D;;OAEG;IACG,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;CAyBV"}

package/dist/storage/supabase.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"supabase.d.ts","sourceRoot":"","sources":["../../src/storage/supabase.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAEL,KAAK,WAAW,EAGjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,mBAAmB,EACnB,WAAW,EACX,cAAc,EACf,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA0BD,qBAAa,sBAAuB,YAAW,uBAAuB;IACpE,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,MAAM;IAUrD,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAwD5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAyBzC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAiCnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAepC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkBzE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkDrF,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA4B/D,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA2BnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA2B5C,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAclD,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUtD,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;IAoCH,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAiCzD,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoC/D,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAiC1D,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwD9E,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAgC/D,eAAe,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAsCxE,kBAAkB,CACtB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAyC5B,qBAAqB,CACzB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,CAAC;CA0E5B"}
+{"version":3,"file":"supabase.d.ts","sourceRoot":"","sources":["../../src/storage/supabase.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAEL,KAAK,WAAW,EAGjB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAChB,WAAW,EACX,eAAe,EACf,mBAAmB,EACnB,WAAW,EACX,cAAc,EACf,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AA0BD,qBAAa,sBAAuB,YAAW,uBAAuB;IACpE,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,MAAM;IAUrD,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAwD5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAyBzC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAiCnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAepC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwBzE,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkDrF,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA4B/D,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA2BnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA2B5C,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAclD,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAUtD,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,mBAAmB,SAAO,GACzB,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC;IAqCH,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC;IAiCzD,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAoC/D,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAiC1D,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwD9E,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAgC/D,eAAe,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAsCxE,kBAAkB,CACtB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAyC5B,qBAAqB,CACzB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,CAAC;CA0E5B"}

package/dist/storage/types.d.ts

@@ -2,6 +2,7 @@
  * Storage types and interfaces
  */
 import type { AnyManifest, CaseEvaluationEvidence, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
+import type { ComparisonEligibility } from '../comparison';
 /**
  * Run listing item
  */
@@ -21,7 +22,10 @@ export interface RunListItem {
 export interface ComparisonResult {
     baseline: RunManifest;
     current: RunManifest;
-    delta: {
+    /** Compatibility decision made before any metric delta is calculated. */
+    eligibility: ComparisonEligibility;
+    /** Absent when workloads or rubrics are incomparable. */
+    delta?: {
         successRate: number;
         latency: number;
         tokens: number;

package/dist/storage/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,sBAAsB,EACtB,eAAe,EACf,WAAW,EACX,cAAc,EACf,MAAM,oBAAoB,CAAC;AAE5B;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC;IACpC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,WAAW,CAAC;IACtB,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE;QACL,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC;IACpC,0CAA0C;IAC1C,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7C;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE1C;;OAEG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9C;;OAEG;IACH,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEtD;;OAEG;IACH,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAEpD;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEpD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;OAEG;IACH,OAAO,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,OAAO,EAAE;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,cAAc;IAC5D;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAEtF;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAEhE;;OAEG;IACH,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAEpE;;OAEG;IACH,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAE7C;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEnD;;OAEG;IACH,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEvD;;;;OAIG;IACH,iBAAiB,CAAC,CAChB,KAAK,EAAE,MAAM,EACb,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC,CAAC;CACX;AAMD;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,iDAAiD;IACjD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,MAAM,EAAE,gBAAgB,CAAC;IACzB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,iCAAiC;IACjC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gCAAgC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,iDAAiD;IACjD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,mBAAmB;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,2BAA2B;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,2BAA2B;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2BAA2B;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,4BAA4B;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gCAAgC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,sBAAsB;IACrE;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1D;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEhE;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAE3D;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAE/E;;OAEG;IACH,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhE;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAEzE;;OAEG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAEnC;;;OAGG;IACH,qBAAqB,CAAC,CACpB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,sBAAsB,EACtB,eAAe,EACf,WAAW,EACX,cAAc,EACf,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC;IACpC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,WAAW,CAAC;IACtB,OAAO,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,WAAW,EAAE,qBAAqB,CAAC;IACnC,yDAAyD;IACzD,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC;IACpC,0CAA0C;IAC1C,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7C;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE1C;;OAEG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9C;;OAEG;IACH,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEtD;;OAEG;IACH,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAEpD;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEpD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;OAEG;IACH,OAAO,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,OAAO,EAAE;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,cAAc;IAC5D;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAEtF;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAEhE;;OAEG;IACH,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAEpE;;OAEG;IACH,aAAa,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAE7C;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEnD;;OAEG;IACH,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEvD;;;;OAIG;IACH,iBAAiB,CAAC,CAChB,KAAK,EAAE,MAAM,EACb,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC;QACT,QAAQ,EAAE,gBAAgB,CAAC;QAC3B,UAAU,EAAE,gBAAgB,CAAC;QAC7B,aAAa,EAAE,OAAO,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,IAAI,CAAC,CAAC;CACX;AAMD;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;AAEzE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,iDAAiD;IACjD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,MAAM,EAAE,gBAAgB,CAAC;IACzB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,yBAAyB;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,sBAAsB,CAAC;IAClC,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,gBAAgB,CAAC;IAC1B,iCAAiC;IACjC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gCAAgC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAMD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,iDAAiD;IACjD,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,mBAAmB;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,2BAA2B;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,2BAA2B;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,2BAA2B;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,4BAA4B;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gCAAgC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAMD;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,sBAAsB;IACrE;;OAEG;IACH,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1D;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEhE;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAE3D;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAE/E;;OAEG;IACH,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhE;;OAEG;IACH,eAAe,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAEzE;;OAEG;IACH,kBAAkB,CAChB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAEnC;;;OAGG;IACH,qBAAqB,CAAC,CACpB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artemiskit/core",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
   "type": "module",
   "license": "Apache-2.0",

package/src/comparison/eligibility.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, test } from 'bun:test';
+import type { RunManifest } from '../artifacts';
+import { assessComparisonEligibility, isComparisonAvailable } from './eligibility';
+
+function manifest(overrides: Partial<RunManifest> = {}): RunManifest {
+  return {
+    version: '1.3',
+    run_id: 'run-id',
+    project: 'project',
+    start_time: '2026-09-10T00:00:00.000Z',
+    end_time: '2026-09-10T00:00:01.000Z',
+    duration_ms: 1000,
+    config: { scenario: 'customer-service', provider: 'openai', model: 'model-a' },
+    workload_identity: {
+      schema_version: '1',
+      workload: { schema_version: '1', algorithm: 'sha256', digest: 'a'.repeat(64) },
+      rubric: { schema_version: '1', algorithm: 'sha256', digest: 'b'.repeat(64) },
+    },
+    execution_provenance: {
+      schema_version: '1',
+      target: { provider: 'openai', requested_models: ['model-a'], generation: { temperature: 0 } },
+    },
+    metrics: {
+      success_rate: 1,
+      total_cases: 1,
+      passed_cases: 1,
+      failed_cases: 0,
+      median_latency_ms: 1,
+      p95_latency_ms: 1,
+      total_tokens: 1,
+      total_prompt_tokens: 1,
+      total_completion_tokens: 0,
+    },
+    git: { commit: 'commit', branch: 'main', dirty: false },
+    provenance: { run_by: 'test' },
+    cases: [],
+    environment: { node_version: 'test', platform: 'test', arch: 'test' },
+    ...overrides,
+  };
+}
+
+describe('assessComparisonEligibility', () => {
+  test('accepts matching declared workload, rubric, and execution configuration', () => {
+    const eligibility = assessComparisonEligibility(manifest(), manifest({ run_id: 'current' }));
+
+    expect(eligibility).toEqual({ schema_version: '1', status: 'compatible', reasons: [] });
+    expect(isComparisonAvailable(eligibility)).toBe(true);
+  });
+
+  test('qualifies legacy artifacts instead of presenting them as fully compatible', () => {
+    const legacy = manifest({ workload_identity: undefined, execution_provenance: undefined });
+    const eligibility = assessComparisonEligibility(legacy, manifest({ run_id: 'current' }));
+
+    expect(eligibility.status).toBe('qualified');
+    expect(eligibility.reasons.map((reason) => reason.code)).toEqual([
+      'workload_identity_missing',
+      'rubric_identity_missing',
+      'execution_provenance_missing',
+    ]);
+    expect(isComparisonAvailable(eligibility)).toBe(true);
+  });
+
+  test('qualifies a deliberate target-model change without treating it as the same execution', () => {
+    const current = manifest({
+      run_id: 'current',
+      execution_provenance: {
+        schema_version: '1',
+        target: {
+          provider: 'openai',
+          requested_models: ['model-b'],
+          generation: { temperature: 0 },
+        },
+      },
+    });
+
+    const eligibility = assessComparisonEligibility(manifest(), current);
+
+    expect(eligibility.status).toBe('qualified');
+    expect(eligibility.reasons).toEqual([{ code: 'target_model_changed' }]);
+  });
+
+  test('refuses deltas when workload or rubric evidence differs', () => {
+    const current = manifest({
+      run_id: 'current',
+      workload_identity: {
+        schema_version: '1',
+        workload: { schema_version: '1', algorithm: 'sha256', digest: 'c'.repeat(64) },
+        rubric: { schema_version: '1', algorithm: 'sha256', digest: 'd'.repeat(64) },
+      },
+    });
+
+    const eligibility = assessComparisonEligibility(manifest(), current);
+
+    expect(eligibility.status).toBe('incomparable');
+    expect(eligibility.reasons.map((reason) => reason.code)).toEqual([
+      'workload_mismatch',
+      'rubric_mismatch',
+    ]);
+    expect(isComparisonAvailable(eligibility)).toBe(false);
+  });
+});

package/src/comparison/eligibility.ts

@@ -0,0 +1,116 @@
+/**
+ * Compatibility decisions for comparisons of saved scenario-evaluation runs.
+ *
+ * This contract compares declared evidence only. A matching digest proves the
+ * same canonical workload/rubric was declared; it does not attest to provider
+ * behaviour or replace a future assessment-profile compatibility decision.
+ */
+
+import type { RunManifest } from '../artifacts/types';
+
+export type ComparisonEligibilityStatus = 'compatible' | 'qualified' | 'incomparable';
+
+export type ComparisonEligibilityReasonCode =
+  | 'scenario_mismatch'
+  | 'workload_identity_missing'
+  | 'rubric_identity_missing'
+  | 'workload_mismatch'
+  | 'rubric_mismatch'
+  | 'execution_provenance_missing'
+  | 'target_provider_changed'
+  | 'target_model_changed'
+  | 'generation_settings_changed';
+
+/** A bounded, machine-readable reason for a comparison decision. */
+export interface ComparisonEligibilityReason {
+  code: ComparisonEligibilityReasonCode;
+}
+
+/**
+ * Versioned decision that accompanies every comparison. `qualified` permits
+ * a visibly-qualified delta; `incomparable` prohibits a delta entirely.
+ */
+export interface ComparisonEligibility {
+  schema_version: '1';
+  status: ComparisonEligibilityStatus;
+  reasons: ComparisonEligibilityReason[];
+}
+
+export function assessComparisonEligibility(
+  baseline: RunManifest,
+  current: RunManifest
+): ComparisonEligibility {
+  const reasons: ComparisonEligibilityReason[] = [];
+
+  if (baseline.config.scenario !== current.config.scenario) {
+    reasons.push({ code: 'scenario_mismatch' });
+  }
+
+  const baselineIdentity = baseline.workload_identity;
+  const currentIdentity = current.workload_identity;
+  if (!baselineIdentity || !currentIdentity) {
+    if (!baselineIdentity || !currentIdentity) reasons.push({ code: 'workload_identity_missing' });
+    if (!baselineIdentity || !currentIdentity) reasons.push({ code: 'rubric_identity_missing' });
+  } else {
+    if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
+      reasons.push({ code: 'workload_mismatch' });
+    }
+    if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
+      reasons.push({ code: 'rubric_mismatch' });
+    }
+  }
+
+  if (reasons.some((reason) => isIncomparableReason(reason.code))) {
+    return { schema_version: '1', status: 'incomparable', reasons };
+  }
+
+  const baselineExecution = baseline.execution_provenance;
+  const currentExecution = current.execution_provenance;
+  if (!baselineExecution || !currentExecution) {
+    reasons.push({ code: 'execution_provenance_missing' });
+  } else {
+    if (baselineExecution.target.provider !== currentExecution.target.provider) {
+      reasons.push({ code: 'target_provider_changed' });
+    }
+    if (
+      !sameStringSet(
+        baselineExecution.target.requested_models,
+        currentExecution.target.requested_models
+      )
+    ) {
+      reasons.push({ code: 'target_model_changed' });
+    }
+    if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
+      reasons.push({ code: 'generation_settings_changed' });
+    }
+  }
+
+  return {
+    schema_version: '1',
+    status: reasons.length === 0 ? 'compatible' : 'qualified',
+    reasons,
+  };
+}
+
+export function isComparisonAvailable(eligibility: ComparisonEligibility): boolean {
+  return eligibility.status !== 'incomparable';
+}
+
+function isIncomparableReason(code: ComparisonEligibilityReasonCode): boolean {
+  return code === 'scenario_mismatch' || code === 'workload_mismatch' || code === 'rubric_mismatch';
+}
+
+function sameStringSet(left?: string[], right?: string[]): boolean {
+  return JSON.stringify([...(left ?? [])].sort()) === JSON.stringify([...(right ?? [])].sort());
+}
+
+function sameGeneration(
+  left?: { temperature?: number; max_tokens?: number; seed?: number },
+  right?: { temperature?: number; max_tokens?: number; seed?: number }
+): boolean {
+  return (
+    left?.temperature === right?.temperature &&
+    left?.max_tokens === right?.max_tokens &&
+    left?.seed === right?.seed
+  );
+}

package/src/comparison/index.ts

@@ -0,0 +1,8 @@
+export {
+  assessComparisonEligibility,
+  isComparisonAvailable,
+  type ComparisonEligibility,
+  type ComparisonEligibilityReason,
+  type ComparisonEligibilityReasonCode,
+  type ComparisonEligibilityStatus,
+} from './eligibility';

package/src/index.ts

@@ -24,6 +24,9 @@ export * from './artifacts';
 // Provenance
 export * from './provenance';
 
+// Comparison eligibility
+export * from './comparison';
+
 // Utilities
 export * from './utils';
 

package/src/storage/local.test.ts

@@ -179,6 +179,45 @@ describe('LocalStorageAdapter', () => {
     expect(comparison.delta.successRate).toBeCloseTo(0.1, 2);
     expect(comparison.delta.latency).toBe(-30);
     expect(comparison.delta.tokens).toBe(100);
+    expect(comparison.eligibility.status).toBe('qualified');
+  });
+
+  test('withholds deltas for incompatible workload evidence', async () => {
+    const baseline = {
+      ...mockManifest,
+      run_id: 'incompatible-baseline',
+      workload_identity: {
+        schema_version: '1' as const,
+        workload: {
+          schema_version: '1' as const,
+          algorithm: 'sha256' as const,
+          digest: 'a'.repeat(64),
+        },
+        rubric: {
+          schema_version: '1' as const,
+          algorithm: 'sha256' as const,
+          digest: 'b'.repeat(64),
+        },
+      },
+      execution_provenance: { schema_version: '1' as const, target: { provider: 'openai' } },
+    };
+    const current = {
+      ...baseline,
+      run_id: 'incompatible-current',
+      workload_identity: {
+        ...baseline.workload_identity,
+        workload: { ...baseline.workload_identity.workload, digest: 'c'.repeat(64) },
+      },
+    };
+
+    await storage.save(baseline);
+    await storage.save(current);
+
+    const comparison = await storage.compare('incompatible-baseline', 'incompatible-current');
+
+    expect(comparison.eligibility.status).toBe('incomparable');
+    expect(comparison.eligibility.reasons).toEqual([{ code: 'workload_mismatch' }]);
+    expect(comparison.delta).toBeUndefined();
   });
 
   test('handles empty storage gracefully', async () => {

package/src/storage/local.ts

@@ -12,6 +12,7 @@ import {
   assertRunManifestIntegrity,
   isRunManifest,
 } from '../artifacts/types';
+import { assessComparisonEligibility, isComparisonAvailable } from '../comparison';
 import type {
   BaselineMetadata,
   BaselineStorageAdapter,
@@ -212,9 +213,15 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
       this.loadRun(currentId),
     ]);
 
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
+
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -375,7 +382,8 @@ export class LocalStorageAdapter implements BaselineStorageAdapter {
     const comparison = await this.compare(baseline.runId, runId);
 
     // Check for regression (negative delta in success rate)
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression =
+      comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
 
     return {
       baseline,

package/src/storage/supabase.ts

@@ -9,6 +9,7 @@ import {
   assertRunManifestIntegrity,
   getCaseEvaluationStatus,
 } from '../artifacts/types';
+import { assessComparisonEligibility, isComparisonAvailable } from '../comparison';
 import type {
   AnalyticsStorageAdapter,
   BaselineMetadata,
@@ -199,9 +200,15 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
   async compare(baselineId: string, currentId: string): Promise<ComparisonResult> {
     const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
 
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
+
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -400,7 +407,8 @@ export class SupabaseStorageAdapter implements AnalyticsStorageAdapter {
     const comparison = await this.compare(baseline.runId, runId);
 
     // Check for regression (success rate dropped by more than threshold)
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression =
+      comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
 
     return {
       baseline,

package/src/storage/types.ts

@@ -9,6 +9,7 @@ import type {
   RunManifest,
   StressManifest,
 } from '../artifacts/types';
+import type { ComparisonEligibility } from '../comparison';
 
 /**
  * Run listing item
@@ -30,7 +31,10 @@ export interface RunListItem {
 export interface ComparisonResult {
   baseline: RunManifest;
   current: RunManifest;
-  delta: {
+  /** Compatibility decision made before any metric delta is calculated. */
+  eligibility: ComparisonEligibility;
+  /** Absent when workloads or rubrics are incomparable. */
+  delta?: {
     successRate: number;
     latency: number;
     tokens: number;