@artemiskit/core 0.5.1 → 0.5.3

package/dist/index.js

@@ -19824,7 +19824,7 @@ var TestCaseSchema = objectType({
   tags: arrayType(stringType()).optional().default([]),
   metadata: recordType(unknownType()).optional().default({}),
   timeout: numberType().optional(),
-  retries: numberType().optional().default(0),
+  retries: numberType().int().min(0).max(99).optional().default(0),
   provider: ProviderSchema.optional(),
   model: stringType().optional(),
   variables: VariablesSchema.optional(),
@@ -20524,17 +20524,48 @@ function mergeRedactionConfig(scenarioConfig, caseConfig, cliConfig) {
 }
 async function executeCase(testCase, context) {
   const { timeout, retries = 0 } = context;
+  if (!Number.isSafeInteger(retries) || retries < 0 || retries > 99) {
+    throw new RangeError("retries must be a whole number between 0 and 99");
+  }
   const caseStartTime = Date.now();
   const requestedModel = testCase.model || context.requestedModel || context.scenario.model;
+  const retryChainId = `${context.runId ?? "untracked"}:${testCase.id}`;
+  const repetitionIndex = context.repetition?.index ?? 1;
+  const attemptEvidence = [];
   let lastError = null;
   for (let attempt = 0;attempt <= retries; attempt++) {
+    const attemptStartTime = Date.now();
     try {
       const result = await executeCaseAttempt(testCase, context, timeout);
-      return { ...result, attempts: attempt + 1 };
+      return withAttemptEvidence(result, attemptEvidence, {
+        retryChainId,
+        repetitionIndex,
+        attemptNumber: attempt + 1,
+        includedInOutcome: true,
+        latencyMs: result.latencyMs
+      });
     } catch (error) {
       lastError = error;
-      if (error instanceof ToolLoopError)
-        return { ...error.caseResult, attempts: attempt + 1 };
+      if (error instanceof ToolLoopError) {
+        return withAttemptEvidence(error.caseResult, attemptEvidence, {
+          retryChainId,
+          repetitionIndex,
+          attemptNumber: attempt + 1,
+          includedInOutcome: true,
+          latencyMs: error.caseResult.latencyMs,
+          errorCode: "tool_error"
+        });
+      }
+      attemptEvidence.push({
+        attempt_id: `${retryChainId}:${attempt + 1}`,
+        retry_chain_id: retryChainId,
+        repetition_index: repetitionIndex,
+        attempt_number: attempt + 1,
+        status: "error",
+        included_in_outcome: false,
+        latency_ms: Date.now() - attemptStartTime,
+        error_code: error instanceof TimeoutError ? "timeout" : "target_error"
+      });
       if (attempt < retries) {
         await sleep(2 ** attempt * 1000);
       }
@@ -20547,6 +20578,10 @@ async function executeCase(testCase, context) {
     ok: false,
     status: "error",
     attempts: retries + 1,
+    attempt_evidence: attemptEvidence.map((entry, index) => ({
+      ...entry,
+      included_in_outcome: index === attemptEvidence.length - 1
+    })),
     score: 0,
     matcherType: testCase.expected.type,
     reason: `Failed after ${retries + 1} attempts: ${lastError?.message}`,
@@ -20560,6 +20595,28 @@ async function executeCase(testCase, context) {
     target: targetEvidence(context.client.provider, requestedModel)
   };
 }
+function withAttemptEvidence(result, priorAttempts, input) {
+  return {
+    ...result,
+    attempts: input.attemptNumber,
+    attempt_evidence: [
+      ...priorAttempts,
+      {
+        attempt_id: `${input.retryChainId}:${input.attemptNumber}`,
+        retry_chain_id: input.retryChainId,
+        repetition_index: input.repetitionIndex,
+        attempt_number: input.attemptNumber,
+        status: getTerminalStatus(result),
+        included_in_outcome: input.includedInOutcome,
+        latency_ms: input.latencyMs,
+        ...input.errorCode ? { error_code: input.errorCode } : {}
+      }
+    ]
+  };
+}
+function getTerminalStatus(result) {
+  return result.status ?? (result.ok ? "passed" : result.error ? "error" : "failed");
+}
 async function executeCaseAttempt(testCase, context, timeout) {
   const { client, scenario, requestedModel, redaction: cliRedaction, toolExecutor } = context;
   const variables = mergeVariables(scenario.variables, testCase.variables);
@@ -20928,374 +20985,61 @@ function nanoid(size = 21) {
   return id;
 }
 
-// src/cost/pricing.ts
-var MODEL_PRICING = {
-  "gpt-5": {
-    promptPer1K: 0.00125,
-    completionPer1K: 0.01,
-    lastUpdated: "2026-01",
-    notes: "400K context window"
-  },
-  "gpt-5.1": {
-    promptPer1K: 0.00125,
-    completionPer1K: 0.01,
-    lastUpdated: "2026-01"
-  },
-  "gpt-5.2": {
-    promptPer1K: 0.00175,
-    completionPer1K: 0.014,
-    lastUpdated: "2026-01"
-  },
-  "gpt-5-mini": {
-    promptPer1K: 0.00025,
-    completionPer1K: 0.002,
-    lastUpdated: "2026-01"
-  },
-  "gpt-5-nano": {
-    promptPer1K: 0.00005,
-    completionPer1K: 0.0004,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4.1": {
-    promptPer1K: 0.002,
-    completionPer1K: 0.008,
-    lastUpdated: "2026-01",
-    notes: "1M context window"
-  },
-  "gpt-4.1-mini": {
-    promptPer1K: 0.0004,
-    completionPer1K: 0.0016,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4.1-nano": {
-    promptPer1K: 0.0001,
-    completionPer1K: 0.0004,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4o": {
-    promptPer1K: 0.0025,
-    completionPer1K: 0.01,
-    lastUpdated: "2026-01",
-    notes: "128K context window"
-  },
-  "gpt-4o-mini": {
-    promptPer1K: 0.00015,
-    completionPer1K: 0.0006,
-    lastUpdated: "2026-01",
-    notes: "128K context window"
-  },
-  o1: {
-    promptPer1K: 0.015,
-    completionPer1K: 0.06,
-    lastUpdated: "2026-01",
-    notes: "Reasoning model - internal thinking tokens billed as output"
-  },
-  o3: {
-    promptPer1K: 0.002,
-    completionPer1K: 0.008,
-    lastUpdated: "2026-01"
-  },
-  "o3-mini": {
-    promptPer1K: 0.0011,
-    completionPer1K: 0.0044,
-    lastUpdated: "2026-01"
-  },
-  "o4-mini": {
-    promptPer1K: 0.0011,
-    completionPer1K: 0.0044,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4-turbo": {
-    promptPer1K: 0.01,
-    completionPer1K: 0.03,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4": {
-    promptPer1K: 0.03,
-    completionPer1K: 0.06,
-    lastUpdated: "2026-01"
-  },
-  "gpt-3.5-turbo": {
-    promptPer1K: 0.0005,
-    completionPer1K: 0.0015,
-    lastUpdated: "2026-01"
-  },
-  "claude-opus-4.5": {
-    promptPer1K: 0.005,
-    completionPer1K: 0.025,
-    lastUpdated: "2026-01",
-    notes: "Most capable Claude model"
-  },
-  "claude-sonnet-4.5": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01",
-    notes: "Balanced performance and cost"
-  },
-  "claude-haiku-4.5": {
-    promptPer1K: 0.001,
-    completionPer1K: 0.005,
-    lastUpdated: "2026-01",
-    notes: "Fastest Claude model"
-  },
-  "claude-opus-4": {
-    promptPer1K: 0.015,
-    completionPer1K: 0.075,
-    lastUpdated: "2026-01"
-  },
-  "claude-opus-4.1": {
-    promptPer1K: 0.015,
-    completionPer1K: 0.075,
-    lastUpdated: "2026-01"
-  },
-  "claude-sonnet-4": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-sonnet-3.7": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-7-sonnet": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-5-sonnet-20241022": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-5-haiku-20241022": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.004,
-    lastUpdated: "2026-01"
-  },
-  "claude-haiku-3.5": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.004,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-opus": {
-    promptPer1K: 0.015,
-    completionPer1K: 0.075,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-sonnet": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-haiku": {
-    promptPer1K: 0.00025,
-    completionPer1K: 0.00125,
-    lastUpdated: "2026-01"
-  },
-  "claude-3.5-sonnet": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3.5-haiku": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.004,
-    lastUpdated: "2026-01"
-  }
-};
-var DEFAULT_PRICING = {
-  promptPer1K: 0.003,
-  completionPer1K: 0.015,
-  lastUpdated: "2026-01",
-  notes: "Default pricing - verify with provider"
-};
-function getModelPricing(model) {
-  if (MODEL_PRICING[model]) {
-    return MODEL_PRICING[model];
+// src/provenance/environment.ts
+function getEnvironmentInfo() {
+  return {
+    node_version: process.version,
+    platform: process.platform,
+    arch: process.arch
+  };
+}
+
+// src/provenance/git.ts
+import { execSync } from "child_process";
+function getGitInfo() {
+  try {
+    const commit = execGit("rev-parse HEAD");
+    const branch = execGit("rev-parse --abbrev-ref HEAD");
+    const dirty = execGit("status --porcelain").length > 0;
+    const remote = execGit("remote get-url origin", true);
+    return {
+      commit,
+      branch,
+      dirty,
+      remote: remote || undefined
+    };
+  } catch {
+    return {
+      commit: "unknown",
+      branch: "unknown",
+      dirty: false
+    };
   }
-  const lowerModel = model.toLowerCase();
-  for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
-    if (key.toLowerCase() === lowerModel) {
-      return pricing;
+}
+function execGit(command, allowFailure = false) {
+  try {
+    return execSync(`git ${command}`, {
+      encoding: "utf-8",
+      stdio: ["pipe", "pipe", "pipe"]
+    }).trim();
+  } catch {
+    if (allowFailure) {
+      return "";
     }
+    throw new Error(`Git command failed: ${command}`);
   }
-  if (lowerModel.includes("gpt-5.2")) {
-    return MODEL_PRICING["gpt-5.2"];
-  }
-  if (lowerModel.includes("gpt-5.1")) {
-    return MODEL_PRICING["gpt-5.1"];
-  }
-  if (lowerModel.includes("gpt-5-mini")) {
-    return MODEL_PRICING["gpt-5-mini"];
-  }
-  if (lowerModel.includes("gpt-5-nano")) {
-    return MODEL_PRICING["gpt-5-nano"];
-  }
-  if (lowerModel.includes("gpt-5")) {
-    return MODEL_PRICING["gpt-5"];
-  }
-  if (lowerModel.includes("gpt-4.1-mini")) {
-    return MODEL_PRICING["gpt-4.1-mini"];
-  }
-  if (lowerModel.includes("gpt-4.1-nano")) {
-    return MODEL_PRICING["gpt-4.1-nano"];
-  }
-  if (lowerModel.includes("gpt-4.1")) {
-    return MODEL_PRICING["gpt-4.1"];
-  }
-  if (lowerModel.includes("gpt-4o-mini")) {
-    return MODEL_PRICING["gpt-4o-mini"];
-  }
-  if (lowerModel.includes("gpt-4o")) {
-    return MODEL_PRICING["gpt-4o"];
-  }
-  if (lowerModel.includes("o4-mini")) {
-    return MODEL_PRICING["o4-mini"];
-  }
-  if (lowerModel.includes("o3-mini")) {
-    return MODEL_PRICING["o3-mini"];
-  }
-  if (lowerModel.includes("o3")) {
-    return MODEL_PRICING.o3;
-  }
-  if (lowerModel.includes("o1")) {
-    return MODEL_PRICING.o1;
-  }
-  if (lowerModel.includes("gpt-4-turbo")) {
-    return MODEL_PRICING["gpt-4-turbo"];
-  }
-  if (lowerModel.includes("gpt-4")) {
-    return MODEL_PRICING["gpt-4"];
-  }
-  if (lowerModel.includes("gpt-3.5")) {
-    return MODEL_PRICING["gpt-3.5-turbo"];
-  }
-  if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
-    return MODEL_PRICING["claude-opus-4.5"];
-  }
-  if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
-    return MODEL_PRICING["claude-sonnet-4.5"];
-  }
-  if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
-    return MODEL_PRICING["claude-haiku-4.5"];
-  }
-  if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
-    return MODEL_PRICING["claude-opus-4.1"];
-  }
-  if (lowerModel.includes("opus-4")) {
-    return MODEL_PRICING["claude-opus-4"];
-  }
-  if (lowerModel.includes("sonnet-4")) {
-    return MODEL_PRICING["claude-sonnet-4"];
-  }
-  if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
-    return MODEL_PRICING["claude-sonnet-3.7"];
-  }
-  if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
-    return MODEL_PRICING["claude-3.5-sonnet"];
-  }
-  if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
-    return MODEL_PRICING["claude-3.5-haiku"];
-  }
-  if (lowerModel.includes("claude-3-opus")) {
-    return MODEL_PRICING["claude-3-opus"];
-  }
-  if (lowerModel.includes("claude-3-sonnet")) {
-    return MODEL_PRICING["claude-3-sonnet"];
-  }
-  if (lowerModel.includes("claude-3-haiku")) {
-    return MODEL_PRICING["claude-3-haiku"];
-  }
-  if (lowerModel.includes("claude")) {
-    return MODEL_PRICING["claude-sonnet-4.5"];
-  }
-  return DEFAULT_PRICING;
-}
-function estimateCost(promptTokens, completionTokens, model) {
-  const pricing = getModelPricing(model);
-  const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
-  const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
-  const totalUsd = promptCostUsd + completionCostUsd;
-  return {
-    totalUsd,
-    promptCostUsd,
-    completionCostUsd,
-    model,
-    pricing
-  };
-}
-function formatCost(costUsd) {
-  if (costUsd < 0.01) {
-    return `$${(costUsd * 100).toFixed(4)} cents`;
-  }
-  if (costUsd < 1) {
-    return `$${costUsd.toFixed(4)}`;
-  }
-  return `$${costUsd.toFixed(2)}`;
-}
-function listKnownModels() {
-  return Object.entries(MODEL_PRICING).map(([model, pricing]) => ({
-    model,
-    pricing
-  }));
-}
-
-// src/provenance/environment.ts
-function getEnvironmentInfo() {
-  return {
-    node_version: process.version,
-    platform: process.platform,
-    arch: process.arch
-  };
-}
-
-// src/provenance/git.ts
-import { execSync } from "child_process";
-function getGitInfo() {
-  try {
-    const commit = execGit("rev-parse HEAD");
-    const branch = execGit("rev-parse --abbrev-ref HEAD");
-    const dirty = execGit("status --porcelain").length > 0;
-    const remote = execGit("remote get-url origin", true);
-    return {
-      commit,
-      branch,
-      dirty,
-      remote: remote || undefined
-    };
-  } catch {
-    return {
-      commit: "unknown",
-      branch: "unknown",
-      dirty: false
-    };
-  }
-}
-function execGit(command, allowFailure = false) {
-  try {
-    return execSync(`git ${command}`, {
-      encoding: "utf-8",
-      stdio: ["pipe", "pipe", "pipe"]
-    }).trim();
-  } catch {
-    if (allowFailure) {
-      return "";
-    }
-    throw new Error(`Git command failed: ${command}`);
-  }
-}
-
-// src/artifacts/types.ts
-var CASE_EVALUATION_STATUS_LABELS = {
-  passed: "Passed",
-  failed: "Failed criteria",
-  invalid: "Invalid measurement",
-  error: "Execution error"
-};
-function getCaseEvaluationStatus(caseResult) {
-  if (caseResult.status === "passed" || caseResult.status === "failed" || caseResult.status === "invalid" || caseResult.status === "error") {
-    return caseResult.status;
+}
+
+// src/artifacts/types.ts
+var CASE_EVALUATION_STATUS_LABELS = {
+  passed: "Passed",
+  failed: "Failed criteria",
+  invalid: "Invalid measurement",
+  error: "Execution error"
+};
+function getCaseEvaluationStatus(caseResult) {
+  if (caseResult.status === "passed" || caseResult.status === "failed" || caseResult.status === "invalid" || caseResult.status === "error") {
+    return caseResult.status;
   }
   if (caseResult.ok)
     return "passed";
@@ -21314,6 +21058,12 @@ function assertRunManifestIntegrity(manifest) {
   if (manifest.execution_provenance !== undefined) {
     assertExecutionProvenance(manifest.execution_provenance);
   }
+  if (manifest.attempt_evidence !== undefined) {
+    assertRunAttemptEvidence(manifest.attempt_evidence);
+  }
+  if (isRecord2(manifest.metrics) && manifest.metrics.cost_provenance !== undefined) {
+    assertCostProvenance(manifest.metrics.cost_provenance);
+  }
   for (const [index, caseResult] of manifest.cases.entries()) {
     if (!isRecord2(caseResult)) {
       throw new Error(`Invalid run manifest: case ${index} is not an object`);
@@ -21327,8 +21077,61 @@ function assertRunManifestIntegrity(manifest) {
     if (caseResult.target !== undefined) {
       assertCaseTargetEvidence(caseResult.target);
     }
+    if (caseResult.attempt_evidence !== undefined) {
+      assertCaseAttemptEvidence(caseResult.attempt_evidence, index);
+    }
+  }
+}
+function assertRunAttemptEvidence(evidence) {
+  if (!isRecord2(evidence) || evidence.schema_version !== "1" || !isRecord2(evidence.repetition) || !isPositiveSafeInteger(evidence.repetition.index) || !isPositiveSafeInteger(evidence.repetition.total) || evidence.repetition.index > evidence.repetition.total || !isRecord2(evidence.retry_policy) || !isNonnegativeSafeInteger(evidence.retry_policy.default_max_retries) || evidence.retry_policy.backoff !== "exponential" || !isNonnegativeFiniteNumber(evidence.retry_policy.initial_delay_ms) || evidence.timeout !== undefined && (!isRecord2(evidence.timeout) || !isPositiveFiniteNumber(evidence.timeout.default_ms))) {
+    throw new Error("Invalid run manifest: malformed attempt evidence");
+  }
+}
+function assertCaseAttemptEvidence(evidence, index) {
+  if (!Array.isArray(evidence) || evidence.length === 0 || evidence.length > 100) {
+    throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
+  }
+  for (const attempt of evidence) {
+    if (!isRecord2(attempt) || !isBoundedNonemptyString(attempt.attempt_id, 200) || !isBoundedNonemptyString(attempt.retry_chain_id, 200) || !isPositiveSafeInteger(attempt.repetition_index) || !isPositiveSafeInteger(attempt.attempt_number) || !isCaseEvaluationStatus(attempt.status) || typeof attempt.included_in_outcome !== "boolean" || !isNonnegativeFiniteNumber(attempt.latency_ms) || attempt.error_code !== undefined && attempt.error_code !== "timeout" && attempt.error_code !== "target_error" && attempt.error_code !== "tool_error") {
+      throw new Error(`Invalid run manifest: case ${index} has malformed attempt evidence`);
+    }
+  }
+}
+function assertCostProvenance(cost) {
+  if (!isRecord2(cost) || cost.schema_version !== "1") {
+    throw new Error("Invalid run manifest: malformed cost provenance");
+  }
+  if (cost.status === "unavailable") {
+    if (cost.amount !== undefined || cost.currency !== undefined || cost.source !== undefined || cost.recorded_at !== undefined || cost.unavailable_reason !== "provider_billing_not_recorded" && cost.unavailable_reason !== "unsupported_provider" && cost.unavailable_reason !== "not_requested") {
+      throw new Error("Invalid run manifest: malformed cost provenance");
+    }
+    return;
+  }
+  if (cost.status !== "known" && cost.status !== "user_supplied" || !isNonnegativeFiniteNumber(cost.amount) || !isBoundedNonemptyString(cost.currency, 3) || cost.status === "known" && cost.source !== "provider_billing" || cost.status === "user_supplied" && cost.source !== "operator_input" || !isIsoTimestamp(cost.recorded_at) || cost.unavailable_reason !== undefined) {
+    throw new Error("Invalid run manifest: malformed cost provenance");
   }
 }
+function isCaseEvaluationStatus(value) {
+  return value === "passed" || value === "failed" || value === "invalid" || value === "error";
+}
+function isBoundedNonemptyString(value, maxLength) {
+  return typeof value === "string" && value.length > 0 && value.length <= maxLength;
+}
+function isNonnegativeFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+function isPositiveFiniteNumber(value) {
+  return isNonnegativeFiniteNumber(value) && value > 0;
+}
+function isNonnegativeSafeInteger(value) {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
+}
+function isPositiveSafeInteger(value) {
+  return isNonnegativeSafeInteger(value) && value > 0;
+}
+function isIsoTimestamp(value) {
+  return typeof value === "string" && Number.isFinite(Date.parse(value));
+}
 function assertCaseTargetEvidence(target) {
   if (!isRecord2(target) || typeof target.provider !== "string" || target.provider.length === 0 || target.provider.length > 100 || target.requested_model !== undefined && (typeof target.requested_model !== "string" || target.requested_model.length > 200) || !isBoundedStringList(target.observed_models)) {
     throw new Error("Invalid run manifest: malformed target evidence");
@@ -21407,6 +21210,9 @@ function createRunManifest(options) {
     resolvedConfig,
     workloadIdentity,
     executionProvenance,
+    attemptEvidence,
+    costProvenance,
+    runId,
     cases,
     startTime,
     endTime,
@@ -21414,13 +21220,12 @@ function createRunManifest(options) {
     runReason,
     redaction
   } = options;
-  const modelForCost = resolvedConfig?.model || config.model;
-  const metrics = calculateMetrics(cases, modelForCost);
+  const metrics = calculateMetrics(cases, costProvenance);
   const git = getGitInfo();
   const environment = getEnvironmentInfo();
   return {
-    version: "1.3",
-    run_id: nanoid(12),
+    version: "1.4",
+    run_id: runId ?? nanoid(12),
     project,
     start_time: startTime.toISOString(),
     end_time: endTime.toISOString(),
@@ -21429,6 +21234,7 @@ function createRunManifest(options) {
     resolved_config: resolvedConfig,
     workload_identity: workloadIdentity,
     execution_provenance: executionProvenance,
+    attempt_evidence: attemptEvidence,
     metrics,
     git,
     provenance: {
@@ -21441,7 +21247,7 @@ function createRunManifest(options) {
     redaction
   };
 }
-function calculateMetrics(cases, model) {
+function calculateMetrics(cases, costProvenance) {
   const passedCases = cases.filter((c) => getCaseEvaluationStatus(c) === "passed");
   const validCases = cases.filter((c) => {
     const status = getCaseEvaluationStatus(c);
@@ -21453,21 +21259,11 @@ function calculateMetrics(cases, model) {
   const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
   const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
   const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
-  let cost;
-  if (model && !model.toLowerCase().includes("ling-") && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
-    const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
-    const pricing = getModelPricing(model);
-    cost = {
-      total_usd: costEstimate.totalUsd,
-      prompt_cost_usd: costEstimate.promptCostUsd,
-      completion_cost_usd: costEstimate.completionCostUsd,
-      model: costEstimate.model,
-      pricing: {
-        prompt_per_1k: pricing.promptPer1K,
-        completion_per_1k: pricing.completionPer1K
-      }
-    };
-  }
+  const cost_provenance = costProvenance ?? {
+    schema_version: "1",
+    status: "unavailable",
+    unavailable_reason: "provider_billing_not_recorded"
+  };
   return {
     success_rate: validCases.length > 0 ? passedCases.length / validCases.length : 0,
     total_attempts: cases.reduce((sum, c) => sum + (c.attempts ?? 1), 0),
@@ -21482,7 +21278,7 @@ function calculateMetrics(cases, model) {
     total_tokens: totalPromptTokens + totalCompletionTokens,
     total_prompt_tokens: totalPromptTokens,
     total_completion_tokens: totalCompletionTokens,
-    cost
+    cost_provenance
   };
 }
 function detectCIEnvironment() {
@@ -21625,6 +21421,8 @@ async function runScenario(options) {
     concurrency = 1,
     timeout,
     retries,
+    repetition = { index: 1, total: 1 },
+    costProvenance,
     redaction,
     toolExecutor,
     onCaseComplete,
@@ -21640,6 +21438,7 @@ async function runScenario(options) {
   }
   onProgress?.(`Running ${cases.length} test cases...`);
   const startTime = new Date;
+  const runId = nanoid(12);
   const results = [];
   if (concurrency === 1) {
     for (let i = 0;i < cases.length; i++) {
@@ -21650,6 +21449,8 @@ async function runScenario(options) {
         requestedModel: resolvedConfig?.model,
         timeout: testCase.timeout || timeout,
         retries: testCase.retries ?? retries,
+        runId,
+        repetition,
         redaction,
         toolExecutor
       });
@@ -21667,6 +21468,8 @@ async function runScenario(options) {
           requestedModel: resolvedConfig?.model,
           timeout: testCase.timeout || timeout,
           retries: testCase.retries ?? retries,
+          runId,
+          repetition,
           redaction,
           toolExecutor
         });
@@ -21717,9 +21520,21 @@ async function runScenario(options) {
       seed: scenario.seed,
       cases: results
     }),
+    attemptEvidence: {
+      schema_version: "1",
+      repetition,
+      retry_policy: {
+        default_max_retries: retries ?? 0,
+        backoff: "exponential",
+        initial_delay_ms: 1000
+      },
+      ...timeout ? { timeout: { default_ms: timeout } } : {}
+    },
+    costProvenance,
     cases: results,
     startTime,
     endTime,
+    runId,
     redaction: redactionInfo
   });
   const success = manifest.metrics.failed_cases === 0 && (manifest.metrics.invalid_evaluations ?? 0) === 0;
@@ -21747,6 +21562,65 @@ function chunkArray(array, size) {
 // src/storage/local.ts
 import { mkdir, readFile as readFile2, readdir as readdir2, unlink, writeFile } from "fs/promises";
 import { join as join2, resolve as resolve2 } from "path";
+
+// src/comparison/eligibility.ts
+function assessComparisonEligibility(baseline, current) {
+  const reasons = [];
+  if (baseline.config.scenario !== current.config.scenario) {
+    reasons.push({ code: "scenario_mismatch" });
+  }
+  const baselineIdentity = baseline.workload_identity;
+  const currentIdentity = current.workload_identity;
+  if (!baselineIdentity || !currentIdentity) {
+    if (!baselineIdentity || !currentIdentity)
+      reasons.push({ code: "workload_identity_missing" });
+    if (!baselineIdentity || !currentIdentity)
+      reasons.push({ code: "rubric_identity_missing" });
+  } else {
+    if (baselineIdentity.workload.digest !== currentIdentity.workload.digest) {
+      reasons.push({ code: "workload_mismatch" });
+    }
+    if (baselineIdentity.rubric.digest !== currentIdentity.rubric.digest) {
+      reasons.push({ code: "rubric_mismatch" });
+    }
+  }
+  if (reasons.some((reason) => isIncomparableReason(reason.code))) {
+    return { schema_version: "1", status: "incomparable", reasons };
+  }
+  const baselineExecution = baseline.execution_provenance;
+  const currentExecution = current.execution_provenance;
+  if (!baselineExecution || !currentExecution) {
+    reasons.push({ code: "execution_provenance_missing" });
+  } else {
+    if (baselineExecution.target.provider !== currentExecution.target.provider) {
+      reasons.push({ code: "target_provider_changed" });
+    }
+    if (!sameStringSet(baselineExecution.target.requested_models, currentExecution.target.requested_models)) {
+      reasons.push({ code: "target_model_changed" });
+    }
+    if (!sameGeneration(baselineExecution.target.generation, currentExecution.target.generation)) {
+      reasons.push({ code: "generation_settings_changed" });
+    }
+  }
+  return {
+    schema_version: "1",
+    status: reasons.length === 0 ? "compatible" : "qualified",
+    reasons
+  };
+}
+function isComparisonAvailable(eligibility) {
+  return eligibility.status !== "incomparable";
+}
+function isIncomparableReason(code) {
+  return code === "scenario_mismatch" || code === "workload_mismatch" || code === "rubric_mismatch";
+}
+function sameStringSet(left, right) {
+  return JSON.stringify([...left ?? []].sort()) === JSON.stringify([...right ?? []].sort());
+}
+function sameGeneration(left, right) {
+  return left?.temperature === right?.temperature && left?.max_tokens === right?.max_tokens && left?.seed === right?.seed;
+}
+// src/storage/local.ts
 function getManifestType(manifest) {
   if ("type" in manifest) {
     if (manifest.type === "redteam")
@@ -21886,9 +21760,14 @@ class LocalStorageAdapter {
       this.loadRun(baselineId),
       this.loadRun(currentId)
     ]);
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -21987,7 +21866,7 @@ class LocalStorageAdapter {
       return null;
     }
     const comparison = await this.compare(baseline.runId, runId);
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
     return {
       baseline,
       comparison,
@@ -30633,9 +30512,14 @@ class SupabaseStorageAdapter {
   }
   async compare(baselineId, currentId) {
     const [baseline, current] = await Promise.all([this.load(baselineId), this.load(currentId)]);
+    const eligibility = assessComparisonEligibility(baseline, current);
+    if (!isComparisonAvailable(eligibility)) {
+      return { baseline, current, eligibility };
+    }
     return {
       baseline,
       current,
+      eligibility,
       delta: {
         successRate: current.metrics.success_rate - baseline.metrics.success_rate,
         latency: current.metrics.median_latency_ms - baseline.metrics.median_latency_ms,
@@ -30766,7 +30650,7 @@ class SupabaseStorageAdapter {
       return null;
     }
     const comparison = await this.compare(baseline.runId, runId);
-    const hasRegression = comparison.delta.successRate < -regressionThreshold;
+    const hasRegression = comparison.delta !== undefined && comparison.delta.successRate < -regressionThreshold;
     return {
       baseline,
       comparison,
@@ -32101,6 +31985,318 @@ class Logger {
   }
 }
 var logger = new Logger("artemis");
+// src/cost/pricing.ts
+var MODEL_PRICING = {
+  "gpt-5": {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01",
+    notes: "400K context window"
+  },
+  "gpt-5.1": {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5.2": {
+    promptPer1K: 0.00175,
+    completionPer1K: 0.014,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5-mini": {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.002,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5-nano": {
+    promptPer1K: 0.00005,
+    completionPer1K: 0.0004,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4.1": {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: "2026-01",
+    notes: "1M context window"
+  },
+  "gpt-4.1-mini": {
+    promptPer1K: 0.0004,
+    completionPer1K: 0.0016,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4.1-nano": {
+    promptPer1K: 0.0001,
+    completionPer1K: 0.0004,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4o": {
+    promptPer1K: 0.0025,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01",
+    notes: "128K context window"
+  },
+  "gpt-4o-mini": {
+    promptPer1K: 0.00015,
+    completionPer1K: 0.0006,
+    lastUpdated: "2026-01",
+    notes: "128K context window"
+  },
+  o1: {
+    promptPer1K: 0.015,
+    completionPer1K: 0.06,
+    lastUpdated: "2026-01",
+    notes: "Reasoning model - internal thinking tokens billed as output"
+  },
+  o3: {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: "2026-01"
+  },
+  "o3-mini": {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: "2026-01"
+  },
+  "o4-mini": {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4-turbo": {
+    promptPer1K: 0.01,
+    completionPer1K: 0.03,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4": {
+    promptPer1K: 0.03,
+    completionPer1K: 0.06,
+    lastUpdated: "2026-01"
+  },
+  "gpt-3.5-turbo": {
+    promptPer1K: 0.0005,
+    completionPer1K: 0.0015,
+    lastUpdated: "2026-01"
+  },
+  "claude-opus-4.5": {
+    promptPer1K: 0.005,
+    completionPer1K: 0.025,
+    lastUpdated: "2026-01",
+    notes: "Most capable Claude model"
+  },
+  "claude-sonnet-4.5": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01",
+    notes: "Balanced performance and cost"
+  },
+  "claude-haiku-4.5": {
+    promptPer1K: 0.001,
+    completionPer1K: 0.005,
+    lastUpdated: "2026-01",
+    notes: "Fastest Claude model"
+  },
+  "claude-opus-4": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
+  },
+  "claude-opus-4.1": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
+  },
+  "claude-sonnet-4": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-sonnet-3.7": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-7-sonnet": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-5-sonnet-20241022": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-5-haiku-20241022": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
+  },
+  "claude-haiku-3.5": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-opus": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-sonnet": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-haiku": {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.00125,
+    lastUpdated: "2026-01"
+  },
+  "claude-3.5-sonnet": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3.5-haiku": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
+  }
+};
+var DEFAULT_PRICING = {
+  promptPer1K: 0.003,
+  completionPer1K: 0.015,
+  lastUpdated: "2026-01",
+  notes: "Default pricing - verify with provider"
+};
+function getModelPricing(model) {
+  if (MODEL_PRICING[model]) {
+    return MODEL_PRICING[model];
+  }
+  const lowerModel = model.toLowerCase();
+  for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
+    if (key.toLowerCase() === lowerModel) {
+      return pricing;
+    }
+  }
+  if (lowerModel.includes("gpt-5.2")) {
+    return MODEL_PRICING["gpt-5.2"];
+  }
+  if (lowerModel.includes("gpt-5.1")) {
+    return MODEL_PRICING["gpt-5.1"];
+  }
+  if (lowerModel.includes("gpt-5-mini")) {
+    return MODEL_PRICING["gpt-5-mini"];
+  }
+  if (lowerModel.includes("gpt-5-nano")) {
+    return MODEL_PRICING["gpt-5-nano"];
+  }
+  if (lowerModel.includes("gpt-5")) {
+    return MODEL_PRICING["gpt-5"];
+  }
+  if (lowerModel.includes("gpt-4.1-mini")) {
+    return MODEL_PRICING["gpt-4.1-mini"];
+  }
+  if (lowerModel.includes("gpt-4.1-nano")) {
+    return MODEL_PRICING["gpt-4.1-nano"];
+  }
+  if (lowerModel.includes("gpt-4.1")) {
+    return MODEL_PRICING["gpt-4.1"];
+  }
+  if (lowerModel.includes("gpt-4o-mini")) {
+    return MODEL_PRICING["gpt-4o-mini"];
+  }
+  if (lowerModel.includes("gpt-4o")) {
+    return MODEL_PRICING["gpt-4o"];
+  }
+  if (lowerModel.includes("o4-mini")) {
+    return MODEL_PRICING["o4-mini"];
+  }
+  if (lowerModel.includes("o3-mini")) {
+    return MODEL_PRICING["o3-mini"];
+  }
+  if (lowerModel.includes("o3")) {
+    return MODEL_PRICING.o3;
+  }
+  if (lowerModel.includes("o1")) {
+    return MODEL_PRICING.o1;
+  }
+  if (lowerModel.includes("gpt-4-turbo")) {
+    return MODEL_PRICING["gpt-4-turbo"];
+  }
+  if (lowerModel.includes("gpt-4")) {
+    return MODEL_PRICING["gpt-4"];
+  }
+  if (lowerModel.includes("gpt-3.5")) {
+    return MODEL_PRICING["gpt-3.5-turbo"];
+  }
+  if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
+    return MODEL_PRICING["claude-opus-4.5"];
+  }
+  if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
+    return MODEL_PRICING["claude-sonnet-4.5"];
+  }
+  if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
+    return MODEL_PRICING["claude-haiku-4.5"];
+  }
+  if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
+    return MODEL_PRICING["claude-opus-4.1"];
+  }
+  if (lowerModel.includes("opus-4")) {
+    return MODEL_PRICING["claude-opus-4"];
+  }
+  if (lowerModel.includes("sonnet-4")) {
+    return MODEL_PRICING["claude-sonnet-4"];
+  }
+  if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
+    return MODEL_PRICING["claude-sonnet-3.7"];
+  }
+  if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
+    return MODEL_PRICING["claude-3.5-sonnet"];
+  }
+  if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
+    return MODEL_PRICING["claude-3.5-haiku"];
+  }
+  if (lowerModel.includes("claude-3-opus")) {
+    return MODEL_PRICING["claude-3-opus"];
+  }
+  if (lowerModel.includes("claude-3-sonnet")) {
+    return MODEL_PRICING["claude-3-sonnet"];
+  }
+  if (lowerModel.includes("claude-3-haiku")) {
+    return MODEL_PRICING["claude-3-haiku"];
+  }
+  if (lowerModel.includes("claude")) {
+    return MODEL_PRICING["claude-sonnet-4.5"];
+  }
+  return DEFAULT_PRICING;
+}
+function estimateCost(promptTokens, completionTokens, model) {
+  const pricing = getModelPricing(model);
+  const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
+  const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
+  const totalUsd = promptCostUsd + completionCostUsd;
+  return {
+    totalUsd,
+    promptCostUsd,
+    completionCostUsd,
+    model,
+    pricing
+  };
+}
+function formatCost(costUsd) {
+  if (costUsd < 0.01) {
+    return `$${(costUsd * 100).toFixed(4)} cents`;
+  }
+  if (costUsd < 1) {
+    return `$${costUsd.toFixed(4)}`;
+  }
+  return `$${costUsd.toFixed(2)}`;
+}
+function listKnownModels() {
+  return Object.entries(MODEL_PRICING).map(([model, pricing]) => ({
+    model,
+    pricing
+  }));
+}
 // src/agent-evaluation/types.ts
 function actionBudgetExceeded(task, trace) {
   return trace.actions.length > task.maxActions;
@@ -32797,6 +32993,7 @@ export {
   actionBudgetExceeded,
   adapterRegistry,
   assertRunManifestIntegrity,
+  assessComparisonEligibility,
   createAdapter,
   createDefaultRedactor,
   createExecutionProvenance,
@@ -32818,6 +33015,7 @@ export {
   getModelPricing,
   hashText,
   isArtemisError,
+  isComparisonAvailable,
   isRedTeamManifest,
   isRunManifest,
   isStressManifest,