@artemiskit/core 0.2.2 → 0.2.4

package/dist/index.js

@@ -10896,6 +10896,55 @@ var require_public_api = __commonJS((exports) => {
   exports.stringify = stringify;
 });
 
+// ../../node_modules/.bun/yaml@2.8.2/node_modules/yaml/dist/index.js
+var require_dist = __commonJS((exports) => {
+  var composer = require_composer();
+  var Document = require_Document();
+  var Schema = require_Schema();
+  var errors2 = require_errors();
+  var Alias = require_Alias();
+  var identity = require_identity();
+  var Pair = require_Pair();
+  var Scalar = require_Scalar();
+  var YAMLMap = require_YAMLMap();
+  var YAMLSeq = require_YAMLSeq();
+  var cst = require_cst();
+  var lexer = require_lexer();
+  var lineCounter = require_line_counter();
+  var parser = require_parser();
+  var publicApi = require_public_api();
+  var visit = require_visit();
+  exports.Composer = composer.Composer;
+  exports.Document = Document.Document;
+  exports.Schema = Schema.Schema;
+  exports.YAMLError = errors2.YAMLError;
+  exports.YAMLParseError = errors2.YAMLParseError;
+  exports.YAMLWarning = errors2.YAMLWarning;
+  exports.Alias = Alias.Alias;
+  exports.isAlias = identity.isAlias;
+  exports.isCollection = identity.isCollection;
+  exports.isDocument = identity.isDocument;
+  exports.isMap = identity.isMap;
+  exports.isNode = identity.isNode;
+  exports.isPair = identity.isPair;
+  exports.isScalar = identity.isScalar;
+  exports.isSeq = identity.isSeq;
+  exports.Pair = Pair.Pair;
+  exports.Scalar = Scalar.Scalar;
+  exports.YAMLMap = YAMLMap.YAMLMap;
+  exports.YAMLSeq = YAMLSeq.YAMLSeq;
+  exports.CST = cst;
+  exports.Lexer = lexer.Lexer;
+  exports.LineCounter = lineCounter.LineCounter;
+  exports.Parser = parser.Parser;
+  exports.parse = publicApi.parse;
+  exports.parseAllDocuments = publicApi.parseAllDocuments;
+  exports.parseDocument = publicApi.parseDocument;
+  exports.stringify = publicApi.stringify;
+  exports.visit = visit.visit;
+  exports.visitAsync = visit.visitAsync;
+});
+
 // src/evaluators/combined.ts
 async function getEvaluatorForType(type) {
   const { getEvaluator } = await Promise.resolve().then(() => (init_evaluators(), exports_evaluators));
@@ -13616,55 +13665,8 @@ var ScenarioSchema = exports_external.object({
   }).optional()
 });
 // src/scenario/parser.ts
+var import_yaml = __toESM(require_dist(), 1);
 import { readFile } from "fs/promises";
-
-// ../../node_modules/.bun/yaml@2.8.2/node_modules/yaml/dist/index.js
-var composer = require_composer();
-var Document = require_Document();
-var Schema = require_Schema();
-var errors2 = require_errors();
-var Alias = require_Alias();
-var identity = require_identity();
-var Pair = require_Pair();
-var Scalar = require_Scalar();
-var YAMLMap = require_YAMLMap();
-var YAMLSeq = require_YAMLSeq();
-var cst = require_cst();
-var lexer = require_lexer();
-var lineCounter = require_line_counter();
-var parser = require_parser();
-var publicApi = require_public_api();
-var visit = require_visit();
-var $Composer = composer.Composer;
-var $Document = Document.Document;
-var $Schema = Schema.Schema;
-var $YAMLError = errors2.YAMLError;
-var $YAMLParseError = errors2.YAMLParseError;
-var $YAMLWarning = errors2.YAMLWarning;
-var $Alias = Alias.Alias;
-var $isAlias = identity.isAlias;
-var $isCollection = identity.isCollection;
-var $isDocument = identity.isDocument;
-var $isMap = identity.isMap;
-var $isNode = identity.isNode;
-var $isPair = identity.isPair;
-var $isScalar = identity.isScalar;
-var $isSeq = identity.isSeq;
-var $Pair = Pair.Pair;
-var $Scalar = Scalar.Scalar;
-var $YAMLMap = YAMLMap.YAMLMap;
-var $YAMLSeq = YAMLSeq.YAMLSeq;
-var $Lexer = lexer.Lexer;
-var $LineCounter = lineCounter.LineCounter;
-var $Parser = parser.Parser;
-var $parse = publicApi.parse;
-var $parseAllDocuments = publicApi.parseAllDocuments;
-var $parseDocument = publicApi.parseDocument;
-var $stringify = publicApi.stringify;
-var $visit = visit.visit;
-var $visitAsync = visit.visitAsync;
-
-// src/scenario/parser.ts
 function expandEnvVars(obj) {
   if (typeof obj === "string") {
     return obj.replace(/\$\{([^}]+)\}/g, (_, expr) => {
@@ -13699,7 +13701,7 @@ async function parseScenarioFile(filePath) {
 }
 function parseScenarioString(content, source) {
   try {
-    const raw = $parse(content);
+    const raw = import_yaml.parse(content);
     const expanded = expandEnvVars(raw);
     const result = ScenarioSchema.safeParse(expanded);
     if (!result.success) {
@@ -14238,54 +14240,367 @@ function nanoid(size = 21) {
   return id;
 }
 
-// src/provenance/environment.ts
-function getEnvironmentInfo() {
-  return {
-    node_version: process.version,
-    platform: process.platform,
-    arch: process.arch
-  };
-}
-
-// src/provenance/git.ts
-import { execSync } from "child_process";
-function getGitInfo() {
-  try {
-    const commit = execGit("rev-parse HEAD");
-    const branch = execGit("rev-parse --abbrev-ref HEAD");
-    const dirty = execGit("status --porcelain").length > 0;
-    const remote = execGit("remote get-url origin", true);
-    return {
-      commit,
-      branch,
-      dirty,
-      remote: remote || undefined
-    };
-  } catch {
-    return {
-      commit: "unknown",
-      branch: "unknown",
-      dirty: false
-    };
+// src/cost/pricing.ts
+var MODEL_PRICING = {
+  "gpt-5": {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01",
+    notes: "400K context window"
+  },
+  "gpt-5.1": {
+    promptPer1K: 0.00125,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5.2": {
+    promptPer1K: 0.00175,
+    completionPer1K: 0.014,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5-mini": {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.002,
+    lastUpdated: "2026-01"
+  },
+  "gpt-5-nano": {
+    promptPer1K: 0.00005,
+    completionPer1K: 0.0004,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4.1": {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: "2026-01",
+    notes: "1M context window"
+  },
+  "gpt-4.1-mini": {
+    promptPer1K: 0.0004,
+    completionPer1K: 0.0016,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4.1-nano": {
+    promptPer1K: 0.0001,
+    completionPer1K: 0.0004,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4o": {
+    promptPer1K: 0.0025,
+    completionPer1K: 0.01,
+    lastUpdated: "2026-01",
+    notes: "128K context window"
+  },
+  "gpt-4o-mini": {
+    promptPer1K: 0.00015,
+    completionPer1K: 0.0006,
+    lastUpdated: "2026-01",
+    notes: "128K context window"
+  },
+  o1: {
+    promptPer1K: 0.015,
+    completionPer1K: 0.06,
+    lastUpdated: "2026-01",
+    notes: "Reasoning model - internal thinking tokens billed as output"
+  },
+  o3: {
+    promptPer1K: 0.002,
+    completionPer1K: 0.008,
+    lastUpdated: "2026-01"
+  },
+  "o3-mini": {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: "2026-01"
+  },
+  "o4-mini": {
+    promptPer1K: 0.0011,
+    completionPer1K: 0.0044,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4-turbo": {
+    promptPer1K: 0.01,
+    completionPer1K: 0.03,
+    lastUpdated: "2026-01"
+  },
+  "gpt-4": {
+    promptPer1K: 0.03,
+    completionPer1K: 0.06,
+    lastUpdated: "2026-01"
+  },
+  "gpt-3.5-turbo": {
+    promptPer1K: 0.0005,
+    completionPer1K: 0.0015,
+    lastUpdated: "2026-01"
+  },
+  "claude-opus-4.5": {
+    promptPer1K: 0.005,
+    completionPer1K: 0.025,
+    lastUpdated: "2026-01",
+    notes: "Most capable Claude model"
+  },
+  "claude-sonnet-4.5": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01",
+    notes: "Balanced performance and cost"
+  },
+  "claude-haiku-4.5": {
+    promptPer1K: 0.001,
+    completionPer1K: 0.005,
+    lastUpdated: "2026-01",
+    notes: "Fastest Claude model"
+  },
+  "claude-opus-4": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
+  },
+  "claude-opus-4.1": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
+  },
+  "claude-sonnet-4": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-sonnet-3.7": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-7-sonnet": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-5-sonnet-20241022": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-5-haiku-20241022": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
+  },
+  "claude-haiku-3.5": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-opus": {
+    promptPer1K: 0.015,
+    completionPer1K: 0.075,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-sonnet": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3-haiku": {
+    promptPer1K: 0.00025,
+    completionPer1K: 0.00125,
+    lastUpdated: "2026-01"
+  },
+  "claude-3.5-sonnet": {
+    promptPer1K: 0.003,
+    completionPer1K: 0.015,
+    lastUpdated: "2026-01"
+  },
+  "claude-3.5-haiku": {
+    promptPer1K: 0.0008,
+    completionPer1K: 0.004,
+    lastUpdated: "2026-01"
   }
-}
-function execGit(command, allowFailure = false) {
-  try {
-    return execSync(`git ${command}`, {
-      encoding: "utf-8",
-      stdio: ["pipe", "pipe", "pipe"]
-    }).trim();
-  } catch {
-    if (allowFailure) {
-      return "";
+};
+var DEFAULT_PRICING = {
+  promptPer1K: 0.003,
+  completionPer1K: 0.015,
+  lastUpdated: "2026-01",
+  notes: "Default pricing - verify with provider"
+};
+function getModelPricing(model) {
+  if (MODEL_PRICING[model]) {
+    return MODEL_PRICING[model];
+  }
+  const lowerModel = model.toLowerCase();
+  for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
+    if (key.toLowerCase() === lowerModel) {
+      return pricing;
     }
-    throw new Error(`Git command failed: ${command}`);
   }
-}
-
-// src/artifacts/manifest.ts
-function createRunManifest(options) {
-  const {
+  if (lowerModel.includes("gpt-5.2")) {
+    return MODEL_PRICING["gpt-5.2"];
+  }
+  if (lowerModel.includes("gpt-5.1")) {
+    return MODEL_PRICING["gpt-5.1"];
+  }
+  if (lowerModel.includes("gpt-5-mini")) {
+    return MODEL_PRICING["gpt-5-mini"];
+  }
+  if (lowerModel.includes("gpt-5-nano")) {
+    return MODEL_PRICING["gpt-5-nano"];
+  }
+  if (lowerModel.includes("gpt-5")) {
+    return MODEL_PRICING["gpt-5"];
+  }
+  if (lowerModel.includes("gpt-4.1-mini")) {
+    return MODEL_PRICING["gpt-4.1-mini"];
+  }
+  if (lowerModel.includes("gpt-4.1-nano")) {
+    return MODEL_PRICING["gpt-4.1-nano"];
+  }
+  if (lowerModel.includes("gpt-4.1")) {
+    return MODEL_PRICING["gpt-4.1"];
+  }
+  if (lowerModel.includes("gpt-4o-mini")) {
+    return MODEL_PRICING["gpt-4o-mini"];
+  }
+  if (lowerModel.includes("gpt-4o")) {
+    return MODEL_PRICING["gpt-4o"];
+  }
+  if (lowerModel.includes("o4-mini")) {
+    return MODEL_PRICING["o4-mini"];
+  }
+  if (lowerModel.includes("o3-mini")) {
+    return MODEL_PRICING["o3-mini"];
+  }
+  if (lowerModel.includes("o3")) {
+    return MODEL_PRICING.o3;
+  }
+  if (lowerModel.includes("o1")) {
+    return MODEL_PRICING.o1;
+  }
+  if (lowerModel.includes("gpt-4-turbo")) {
+    return MODEL_PRICING["gpt-4-turbo"];
+  }
+  if (lowerModel.includes("gpt-4")) {
+    return MODEL_PRICING["gpt-4"];
+  }
+  if (lowerModel.includes("gpt-3.5")) {
+    return MODEL_PRICING["gpt-3.5-turbo"];
+  }
+  if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
+    return MODEL_PRICING["claude-opus-4.5"];
+  }
+  if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
+    return MODEL_PRICING["claude-sonnet-4.5"];
+  }
+  if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
+    return MODEL_PRICING["claude-haiku-4.5"];
+  }
+  if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
+    return MODEL_PRICING["claude-opus-4.1"];
+  }
+  if (lowerModel.includes("opus-4")) {
+    return MODEL_PRICING["claude-opus-4"];
+  }
+  if (lowerModel.includes("sonnet-4")) {
+    return MODEL_PRICING["claude-sonnet-4"];
+  }
+  if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
+    return MODEL_PRICING["claude-sonnet-3.7"];
+  }
+  if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
+    return MODEL_PRICING["claude-3.5-sonnet"];
+  }
+  if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
+    return MODEL_PRICING["claude-3.5-haiku"];
+  }
+  if (lowerModel.includes("claude-3-opus")) {
+    return MODEL_PRICING["claude-3-opus"];
+  }
+  if (lowerModel.includes("claude-3-sonnet")) {
+    return MODEL_PRICING["claude-3-sonnet"];
+  }
+  if (lowerModel.includes("claude-3-haiku")) {
+    return MODEL_PRICING["claude-3-haiku"];
+  }
+  if (lowerModel.includes("claude")) {
+    return MODEL_PRICING["claude-sonnet-4.5"];
+  }
+  return DEFAULT_PRICING;
+}
+function estimateCost(promptTokens, completionTokens, model) {
+  const pricing = getModelPricing(model);
+  const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
+  const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
+  const totalUsd = promptCostUsd + completionCostUsd;
+  return {
+    totalUsd,
+    promptCostUsd,
+    completionCostUsd,
+    model,
+    pricing
+  };
+}
+function formatCost(costUsd) {
+  if (costUsd < 0.01) {
+    return `$${(costUsd * 100).toFixed(4)} cents`;
+  }
+  if (costUsd < 1) {
+    return `$${costUsd.toFixed(4)}`;
+  }
+  return `$${costUsd.toFixed(2)}`;
+}
+function listKnownModels() {
+  return Object.entries(MODEL_PRICING).map(([model, pricing]) => ({
+    model,
+    pricing
+  }));
+}
+
+// src/provenance/environment.ts
+function getEnvironmentInfo() {
+  return {
+    node_version: process.version,
+    platform: process.platform,
+    arch: process.arch
+  };
+}
+
+// src/provenance/git.ts
+import { execSync } from "child_process";
+function getGitInfo() {
+  try {
+    const commit = execGit("rev-parse HEAD");
+    const branch = execGit("rev-parse --abbrev-ref HEAD");
+    const dirty = execGit("status --porcelain").length > 0;
+    const remote = execGit("remote get-url origin", true);
+    return {
+      commit,
+      branch,
+      dirty,
+      remote: remote || undefined
+    };
+  } catch {
+    return {
+      commit: "unknown",
+      branch: "unknown",
+      dirty: false
+    };
+  }
+}
+function execGit(command, allowFailure = false) {
+  try {
+    return execSync(`git ${command}`, {
+      encoding: "utf-8",
+      stdio: ["pipe", "pipe", "pipe"]
+    }).trim();
+  } catch {
+    if (allowFailure) {
+      return "";
+    }
+    throw new Error(`Git command failed: ${command}`);
+  }
+}
+
+// src/artifacts/manifest.ts
+function createRunManifest(options) {
+  const {
     project,
     config,
     resolvedConfig,
@@ -14296,7 +14611,8 @@ function createRunManifest(options) {
     runReason,
     redaction
   } = options;
-  const metrics = calculateMetrics(cases);
+  const modelForCost = resolvedConfig?.model || config.model;
+  const metrics = calculateMetrics(cases, modelForCost);
   const git = getGitInfo();
   const environment = getEnvironmentInfo();
   return {
@@ -14320,7 +14636,7 @@ function createRunManifest(options) {
     redaction
   };
 }
-function calculateMetrics(cases) {
+function calculateMetrics(cases, model) {
   const passedCases = cases.filter((c) => c.ok);
   const latencies = cases.map((c) => c.latencyMs).sort((a, b) => a - b);
   const medianLatency = latencies.length > 0 ? latencies[Math.floor(latencies.length / 2)] : 0;
@@ -14328,6 +14644,21 @@ function calculateMetrics(cases) {
   const p95Latency = latencies.length > 0 ? latencies[p95Index] : 0;
   const totalPromptTokens = cases.reduce((sum, c) => sum + c.tokens.prompt, 0);
   const totalCompletionTokens = cases.reduce((sum, c) => sum + c.tokens.completion, 0);
+  let cost;
+  if (model && (totalPromptTokens > 0 || totalCompletionTokens > 0)) {
+    const costEstimate = estimateCost(totalPromptTokens, totalCompletionTokens, model);
+    const pricing = getModelPricing(model);
+    cost = {
+      total_usd: costEstimate.totalUsd,
+      prompt_cost_usd: costEstimate.promptCostUsd,
+      completion_cost_usd: costEstimate.completionCostUsd,
+      model: costEstimate.model,
+      pricing: {
+        prompt_per_1k: pricing.promptPer1K,
+        completion_per_1k: pricing.completionPer1K
+      }
+    };
+  }
   return {
     success_rate: cases.length > 0 ? passedCases.length / cases.length : 0,
     total_cases: cases.length,
@@ -14337,7 +14668,8 @@ function calculateMetrics(cases) {
     p95_latency_ms: p95Latency,
     total_tokens: totalPromptTokens + totalCompletionTokens,
     total_prompt_tokens: totalPromptTokens,
-    total_completion_tokens: totalCompletionTokens
+    total_completion_tokens: totalCompletionTokens,
+    cost
   };
 }
 function detectCIEnvironment() {
@@ -14508,6 +14840,16 @@ function getSuccessRate(manifest) {
   }
   return manifest.metrics.success_rate;
 }
+function getEstimatedCost(manifest) {
+  const type = getManifestType(manifest);
+  if (type === "stress") {
+    return manifest.metrics.cost?.estimated_total_usd;
+  }
+  if (type === "run") {
+    return manifest.metrics.cost?.total_usd;
+  }
+  return;
+}
 function getScenario(manifest) {
   return manifest.config.scenario;
 }
@@ -14577,13 +14919,17 @@ class LocalStorageAdapter {
           if (options?.scenario && getScenario(manifest) !== options.scenario) {
             continue;
           }
-          results.push({
+          const item = {
             runId: manifest.run_id,
             scenario: getScenario(manifest),
             successRate: getSuccessRate(manifest),
             createdAt: manifest.start_time,
             type: manifestType
-          });
+          };
+          if (options?.includeCost) {
+            item.estimatedCostUsd = getEstimatedCost(manifest);
+          }
+          results.push(item);
         } catch {}
       }
     }
@@ -16585,7 +16931,7 @@ class RealtimeChannel {
       }).map((bind) => {
         if (typeof handledPayload === "object" && "ids" in handledPayload) {
           const postgresChanges = handledPayload.data;
-          const { schema: schema2, table, commit_timestamp, type: type2, errors: errors3 } = postgresChanges;
+          const { schema: schema2, table, commit_timestamp, type: type2, errors: errors2 } = postgresChanges;
           const enrichedPayload = {
             schema: schema2,
             table,
@@ -16593,7 +16939,7 @@ class RealtimeChannel {
             eventType: type2,
             new: {},
             old: {},
-            errors: errors3
+            errors: errors2
           };
           handledPayload = Object.assign(Object.assign({}, enrichedPayload), this._getPayloadRecords(postgresChanges));
         }
@@ -22164,7 +22510,7 @@ class GoTrueClient {
       }
     });
   }
-  async unlinkIdentity(identity2) {
+  async unlinkIdentity(identity) {
     try {
       return await this._useSession(async (result) => {
         var _a, _b;
@@ -22172,7 +22518,7 @@ class GoTrueClient {
         if (error) {
           throw error;
         }
-        return await _request(this.fetch, "DELETE", `${this.url}/user/identities/${identity2.identity_id}`, {
+        return await _request(this.fetch, "DELETE", `${this.url}/user/identities/${identity.identity_id}`, {
           headers: this.headers,
           jwt: (_b = (_a = data.session) === null || _a === undefined ? undefined : _a.access_token) !== null && _b !== undefined ? _b : undefined
         });
@@ -22346,20 +22692,20 @@ class GoTrueClient {
       if (this.broadcastChannel && broadcast) {
         this.broadcastChannel.postMessage({ event, session });
       }
-      const errors3 = [];
+      const errors2 = [];
       const promises = Array.from(this.stateChangeEmitters.values()).map(async (x) => {
         try {
           await x.callback(event, session);
         } catch (e) {
-          errors3.push(e);
+          errors2.push(e);
         }
       });
       await Promise.all(promises);
-      if (errors3.length > 0) {
-        for (let i = 0;i < errors3.length; i += 1) {
-          console.error(errors3[i]);
+      if (errors2.length > 0) {
+        for (let i = 0;i < errors2.length; i += 1) {
+          console.error(errors2[i]);
         }
-        throw errors3[0];
+        throw errors2[0];
       }
     } finally {
       this._debug(debugName, "end");
@@ -24407,317 +24753,250 @@ class Logger {
   }
 }
 var logger = new Logger("artemis");
-// src/cost/pricing.ts
-var MODEL_PRICING = {
-  "gpt-5": {
-    promptPer1K: 0.00125,
-    completionPer1K: 0.01,
-    lastUpdated: "2026-01",
-    notes: "400K context window"
-  },
-  "gpt-5.1": {
-    promptPer1K: 0.00125,
-    completionPer1K: 0.01,
-    lastUpdated: "2026-01"
-  },
-  "gpt-5.2": {
-    promptPer1K: 0.00175,
-    completionPer1K: 0.014,
-    lastUpdated: "2026-01"
-  },
-  "gpt-5-mini": {
-    promptPer1K: 0.00025,
-    completionPer1K: 0.002,
-    lastUpdated: "2026-01"
-  },
-  "gpt-5-nano": {
-    promptPer1K: 0.00005,
-    completionPer1K: 0.0004,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4.1": {
-    promptPer1K: 0.002,
-    completionPer1K: 0.008,
-    lastUpdated: "2026-01",
-    notes: "1M context window"
-  },
-  "gpt-4.1-mini": {
-    promptPer1K: 0.0004,
-    completionPer1K: 0.0016,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4.1-nano": {
-    promptPer1K: 0.0001,
-    completionPer1K: 0.0004,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4o": {
-    promptPer1K: 0.0025,
-    completionPer1K: 0.01,
-    lastUpdated: "2026-01",
-    notes: "128K context window"
-  },
-  "gpt-4o-mini": {
-    promptPer1K: 0.00015,
-    completionPer1K: 0.0006,
-    lastUpdated: "2026-01",
-    notes: "128K context window"
-  },
-  o1: {
-    promptPer1K: 0.015,
-    completionPer1K: 0.06,
-    lastUpdated: "2026-01",
-    notes: "Reasoning model - internal thinking tokens billed as output"
-  },
-  o3: {
-    promptPer1K: 0.002,
-    completionPer1K: 0.008,
-    lastUpdated: "2026-01"
-  },
-  "o3-mini": {
-    promptPer1K: 0.0011,
-    completionPer1K: 0.0044,
-    lastUpdated: "2026-01"
-  },
-  "o4-mini": {
-    promptPer1K: 0.0011,
-    completionPer1K: 0.0044,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4-turbo": {
-    promptPer1K: 0.01,
-    completionPer1K: 0.03,
-    lastUpdated: "2026-01"
-  },
-  "gpt-4": {
-    promptPer1K: 0.03,
-    completionPer1K: 0.06,
-    lastUpdated: "2026-01"
-  },
-  "gpt-3.5-turbo": {
-    promptPer1K: 0.0005,
-    completionPer1K: 0.0015,
-    lastUpdated: "2026-01"
-  },
-  "claude-opus-4.5": {
-    promptPer1K: 0.005,
-    completionPer1K: 0.025,
-    lastUpdated: "2026-01",
-    notes: "Most capable Claude model"
-  },
-  "claude-sonnet-4.5": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01",
-    notes: "Balanced performance and cost"
-  },
-  "claude-haiku-4.5": {
-    promptPer1K: 0.001,
-    completionPer1K: 0.005,
-    lastUpdated: "2026-01",
-    notes: "Fastest Claude model"
-  },
-  "claude-opus-4": {
-    promptPer1K: 0.015,
-    completionPer1K: 0.075,
-    lastUpdated: "2026-01"
-  },
-  "claude-opus-4.1": {
-    promptPer1K: 0.015,
-    completionPer1K: 0.075,
-    lastUpdated: "2026-01"
-  },
-  "claude-sonnet-4": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-sonnet-3.7": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-7-sonnet": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-5-sonnet-20241022": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-5-haiku-20241022": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.004,
-    lastUpdated: "2026-01"
-  },
-  "claude-haiku-3.5": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.004,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-opus": {
-    promptPer1K: 0.015,
-    completionPer1K: 0.075,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-sonnet": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3-haiku": {
-    promptPer1K: 0.00025,
-    completionPer1K: 0.00125,
-    lastUpdated: "2026-01"
-  },
-  "claude-3.5-sonnet": {
-    promptPer1K: 0.003,
-    completionPer1K: 0.015,
-    lastUpdated: "2026-01"
-  },
-  "claude-3.5-haiku": {
-    promptPer1K: 0.0008,
-    completionPer1K: 0.004,
-    lastUpdated: "2026-01"
+// src/validator/validator.ts
+var import_yaml2 = __toESM(require_dist(), 1);
+import { readFileSync } from "fs";
+class ScenarioValidator {
+  _options;
+  constructor(options = {}) {
+    this._options = options;
   }
-};
-var DEFAULT_PRICING = {
-  promptPer1K: 0.003,
-  completionPer1K: 0.015,
-  lastUpdated: "2026-01",
-  notes: "Default pricing - verify with provider"
-};
-function getModelPricing(model) {
-  if (MODEL_PRICING[model]) {
-    return MODEL_PRICING[model];
+  get options() {
+    return this._options;
   }
-  const lowerModel = model.toLowerCase();
-  for (const [key, pricing] of Object.entries(MODEL_PRICING)) {
-    if (key.toLowerCase() === lowerModel) {
-      return pricing;
+  validate(filePath) {
+    const errors4 = [];
+    const warnings = [];
+    let content;
+    try {
+      content = readFileSync(filePath, "utf-8");
+    } catch (err) {
+      const error = err;
+      errors4.push({
+        line: 1,
+        message: `Failed to read file: ${error.message}`,
+        rule: "file-read",
+        severity: "error"
+      });
+      return { file: filePath, valid: false, errors: errors4, warnings };
     }
+    let parsed;
+    try {
+      parsed = import_yaml2.default.parse(content, {
+        prettyErrors: true,
+        strict: true
+      });
+    } catch (err) {
+      if (err instanceof import_yaml2.default.YAMLError) {
+        const linePos = err.linePos?.[0];
+        errors4.push({
+          line: linePos?.line || 1,
+          column: linePos?.col,
+          message: `Invalid YAML syntax: ${err.message}`,
+          rule: "yaml-syntax",
+          severity: "error"
+        });
+      } else {
+        errors4.push({
+          line: 1,
+          message: `YAML parse error: ${err.message}`,
+          rule: "yaml-syntax",
+          severity: "error"
+        });
+      }
+      return { file: filePath, valid: false, errors: errors4, warnings };
+    }
+    if (parsed === null || typeof parsed !== "object") {
+      errors4.push({
+        line: 1,
+        message: "Scenario must be a YAML object",
+        rule: "schema-type",
+        severity: "error"
+      });
+      return { file: filePath, valid: false, errors: errors4, warnings };
+    }
+    const schemaResult = ScenarioSchema.safeParse(parsed);
+    if (!schemaResult.success) {
+      const zodErrors = this.formatZodErrors(schemaResult.error, content);
+      errors4.push(...zodErrors);
+    }
+    if (schemaResult.success) {
+      const semanticErrors = this.validateSemantics(schemaResult.data, content);
+      errors4.push(...semanticErrors);
+    }
+    const detectedWarnings = this.detectWarnings(parsed, content);
+    warnings.push(...detectedWarnings);
+    return {
+      file: filePath,
+      valid: errors4.length === 0,
+      errors: errors4,
+      warnings
+    };
   }
-  if (lowerModel.includes("gpt-5.2")) {
-    return MODEL_PRICING["gpt-5.2"];
-  }
-  if (lowerModel.includes("gpt-5.1")) {
-    return MODEL_PRICING["gpt-5.1"];
-  }
-  if (lowerModel.includes("gpt-5-mini")) {
-    return MODEL_PRICING["gpt-5-mini"];
-  }
-  if (lowerModel.includes("gpt-5-nano")) {
-    return MODEL_PRICING["gpt-5-nano"];
-  }
-  if (lowerModel.includes("gpt-5")) {
-    return MODEL_PRICING["gpt-5"];
-  }
-  if (lowerModel.includes("gpt-4.1-mini")) {
-    return MODEL_PRICING["gpt-4.1-mini"];
-  }
-  if (lowerModel.includes("gpt-4.1-nano")) {
-    return MODEL_PRICING["gpt-4.1-nano"];
-  }
-  if (lowerModel.includes("gpt-4.1")) {
-    return MODEL_PRICING["gpt-4.1"];
-  }
-  if (lowerModel.includes("gpt-4o-mini")) {
-    return MODEL_PRICING["gpt-4o-mini"];
-  }
-  if (lowerModel.includes("gpt-4o")) {
-    return MODEL_PRICING["gpt-4o"];
-  }
-  if (lowerModel.includes("o4-mini")) {
-    return MODEL_PRICING["o4-mini"];
-  }
-  if (lowerModel.includes("o3-mini")) {
-    return MODEL_PRICING["o3-mini"];
-  }
-  if (lowerModel.includes("o3")) {
-    return MODEL_PRICING.o3;
-  }
-  if (lowerModel.includes("o1")) {
-    return MODEL_PRICING.o1;
-  }
-  if (lowerModel.includes("gpt-4-turbo")) {
-    return MODEL_PRICING["gpt-4-turbo"];
-  }
-  if (lowerModel.includes("gpt-4")) {
-    return MODEL_PRICING["gpt-4"];
-  }
-  if (lowerModel.includes("gpt-3.5")) {
-    return MODEL_PRICING["gpt-3.5-turbo"];
-  }
-  if (lowerModel.includes("opus-4.5") || lowerModel.includes("opus-4-5")) {
-    return MODEL_PRICING["claude-opus-4.5"];
-  }
-  if (lowerModel.includes("sonnet-4.5") || lowerModel.includes("sonnet-4-5")) {
-    return MODEL_PRICING["claude-sonnet-4.5"];
-  }
-  if (lowerModel.includes("haiku-4.5") || lowerModel.includes("haiku-4-5")) {
-    return MODEL_PRICING["claude-haiku-4.5"];
-  }
-  if (lowerModel.includes("opus-4.1") || lowerModel.includes("opus-4-1")) {
-    return MODEL_PRICING["claude-opus-4.1"];
-  }
-  if (lowerModel.includes("opus-4")) {
-    return MODEL_PRICING["claude-opus-4"];
-  }
-  if (lowerModel.includes("sonnet-4")) {
-    return MODEL_PRICING["claude-sonnet-4"];
-  }
-  if (lowerModel.includes("sonnet-3.7") || lowerModel.includes("sonnet-3-7")) {
-    return MODEL_PRICING["claude-sonnet-3.7"];
-  }
-  if (lowerModel.includes("claude-3-5-sonnet") || lowerModel.includes("claude-3.5-sonnet")) {
-    return MODEL_PRICING["claude-3.5-sonnet"];
+  formatZodErrors(error, content) {
+    const issues = [];
+    const lines = content.split(`
+`);
+    for (const issue of error.issues) {
+      const path = issue.path.join(".");
+      const line = this.findLineForPath(lines, issue.path);
+      let message;
+      switch (issue.code) {
+        case "invalid_type":
+          message = `'${path}' expected ${issue.expected}, received ${issue.received}`;
+          break;
+        case "invalid_enum_value":
+          message = `'${path}' must be one of: ${issue.options.join(", ")}`;
+          break;
+        case "too_small":
+          if (issue.type === "array") {
+            message = `'${path}' must have at least ${issue.minimum} item(s)`;
+          } else {
+            message = `'${path}' is too small`;
+          }
+          break;
+        case "unrecognized_keys":
+          message = `Unrecognized field(s): ${issue.keys.join(", ")}`;
+          break;
+        default:
+          message = issue.message;
+      }
+      issues.push({
+        line,
+        message,
+        rule: `schema-${issue.code}`,
+        severity: "error"
+      });
+    }
+    return issues;
   }
-  if (lowerModel.includes("claude-3-5-haiku") || lowerModel.includes("claude-3.5-haiku")) {
-    return MODEL_PRICING["claude-3.5-haiku"];
+  findLineForPath(lines, path) {
+    if (path.length === 0)
+      return 1;
+    const searchKey = String(path[path.length - 1]);
+    for (let i2 = 0;i2 < lines.length; i2++) {
+      const line = lines[i2];
+      if (line.includes(`${searchKey}:`) || line.includes(`- ${searchKey}:`)) {
+        return i2 + 1;
+      }
+      if (typeof path[path.length - 1] === "number" && path.includes("cases")) {
+        if (line.trim().startsWith("- id:")) {
+          return i2 + 1;
+        }
+      }
+    }
+    return 1;
   }
-  if (lowerModel.includes("claude-3-opus")) {
-    return MODEL_PRICING["claude-3-opus"];
+  validateSemantics(scenario, content) {
+    const errors4 = [];
+    const lines = content.split(`
+`);
+    const caseIds = new Set;
+    for (const testCase of scenario.cases) {
+      if (caseIds.has(testCase.id)) {
+        const line = this.findLineForCaseId(lines, testCase.id);
+        errors4.push({
+          line,
+          message: `Duplicate case ID: '${testCase.id}'`,
+          rule: "duplicate-case-id",
+          severity: "error"
+        });
+      }
+      caseIds.add(testCase.id);
+    }
+    const globalVars = scenario.variables || {};
+    for (const testCase of scenario.cases) {
+      const caseVars = testCase.variables || {};
+      const allVars = { ...globalVars, ...caseVars };
+      const prompt2 = typeof testCase.prompt === "string" ? testCase.prompt : JSON.stringify(testCase.prompt);
+      const refs = this.extractVariableRefs(prompt2);
+      for (const ref of refs) {
+        if (!(ref in allVars)) {
+          const line = this.findLineForCaseId(lines, testCase.id);
+          errors4.push({
+            line,
+            message: `Undefined variable '{{${ref}}}' in case '${testCase.id}'`,
+            rule: "undefined-variable",
+            severity: "error",
+            suggestion: `Define '${ref}' in scenario.variables or case.variables`
+          });
+        }
+      }
+    }
+    return errors4;
   }
-  if (lowerModel.includes("claude-3-sonnet")) {
-    return MODEL_PRICING["claude-3-sonnet"];
+  findLineForCaseId(lines, caseId) {
+    for (let i2 = 0;i2 < lines.length; i2++) {
+      if (lines[i2].includes(`id: ${caseId}`) || lines[i2].includes(`id: "${caseId}"`) || lines[i2].includes(`id: '${caseId}'`)) {
+        return i2 + 1;
+      }
+    }
+    return 1;
   }
-  if (lowerModel.includes("claude-3-haiku")) {
-    return MODEL_PRICING["claude-3-haiku"];
+  extractVariableRefs(text) {
+    const regex2 = /\{\{(\w+)\}\}/g;
+    const refs = [];
+    const matches = text.matchAll(regex2);
+    for (const match of matches) {
+      refs.push(match[1]);
+    }
+    return refs;
   }
-  if (lowerModel.includes("claude")) {
-    return MODEL_PRICING["claude-sonnet-4.5"];
+  detectWarnings(parsed, content) {
+    const warnings = [];
+    const lines = content.split(`
+`);
+    if (parsed && typeof parsed === "object") {
+      const obj = parsed;
+      if (this.hasDeepKey(obj, "criteria")) {
+        const line = this.findLineForKey(lines, "criteria");
+        warnings.push({
+          line,
+          message: "'criteria' is deprecated, use 'rubric' instead (llm_grader)",
+          rule: "deprecated-field",
+          severity: "warning",
+          suggestion: "Replace 'criteria' with 'rubric'"
+        });
+      }
+      const cases = obj.cases;
+      if (Array.isArray(cases) && cases.length > 20) {
+        warnings.push({
+          line: 1,
+          message: `Scenario has ${cases.length} cases. Consider using --parallel for faster execution.`,
+          rule: "performance-hint",
+          severity: "warning"
+        });
+      }
+      if (!obj.description) {
+        warnings.push({
+          line: 1,
+          message: "Scenario is missing 'description' field. Adding a description improves documentation.",
+          rule: "missing-description",
+          severity: "warning"
+        });
+      }
+    }
+    return warnings;
   }
-  return DEFAULT_PRICING;
-}
-function estimateCost(promptTokens, completionTokens, model) {
-  const pricing = getModelPricing(model);
-  const promptCostUsd = promptTokens / 1000 * pricing.promptPer1K;
-  const completionCostUsd = completionTokens / 1000 * pricing.completionPer1K;
-  const totalUsd = promptCostUsd + completionCostUsd;
-  return {
-    totalUsd,
-    promptCostUsd,
-    completionCostUsd,
-    model,
-    pricing
-  };
-}
-function formatCost(costUsd) {
-  if (costUsd < 0.01) {
-    return `$${(costUsd * 100).toFixed(4)} cents`;
+  hasDeepKey(obj, key) {
+    if (obj === null || typeof obj !== "object")
+      return false;
+    if (key in obj)
+      return true;
+    for (const value of Object.values(obj)) {
+      if (this.hasDeepKey(value, key))
+        return true;
+    }
+    return false;
   }
-  if (costUsd < 1) {
-    return `$${costUsd.toFixed(4)}`;
+  findLineForKey(lines, key) {
+    for (let i2 = 0;i2 < lines.length; i2++) {
+      if (lines[i2].includes(`${key}:`)) {
+        return i2 + 1;
+      }
+    }
+    return 1;
   }
-  return `$${costUsd.toFixed(2)}`;
-}
-function listKnownModels() {
-  return Object.entries(MODEL_PRICING).map(([model, pricing]) => ({
-    model,
-    pricing
-  }));
 }
 export {
   wrapError,
@@ -24766,6 +25045,7 @@ export {
   TestCaseSchema,
   SupabaseStorageAdapter,
   SimilarityEvaluator,
+  ScenarioValidator,
   ScenarioSchema,
   SUPPORTED_EXPRESSIONS,
   RegexEvaluator,