@tangle-network/agent-eval 0.70.0 → 0.72.0

package/dist/index.js

@@ -1,6 +1,13 @@
 import {
-  agentProfileHash
-} from "./chunk-PQV2TKC3.js";
+  MODEL_PRICING,
+  MetricsCollector,
+  TokenCounter,
+  agentProfileHash,
+  estimateCost,
+  estimateTokens,
+  isModelPriced,
+  resolveModelPricing
+} from "./chunk-SL55X4VN.js";
 import {
   HoldoutAuditor,
   canaryLeakView,
@@ -31,12 +38,12 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-ZZCQQHW7.js";
+} from "./chunk-4QJN7RDX.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
   summarizeBackendIntegrity
-} from "./chunk-6XQIEUQ2.js";
+} from "./chunk-ZPSKPT3V.js";
 import {
   BENCHMARK_SPLIT_SEED,
   benchmarks_exports,
@@ -49,7 +56,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-3B7Y5AUR.js";
+} from "./chunk-GWGO2K6Y.js";
 import {
   exportTrainingData,
   toNdjson
@@ -204,7 +211,7 @@ import {
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-Z4ZCBC7M.js";
+} from "./chunk-ODGETRTM.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -2595,14 +2602,15 @@ async function executeScenario(tc, scenario, config) {
     const content = resp.choices?.[0]?.message?.content ?? "";
     messages.push({ role: "assistant", content });
     const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
-    let codeMatch;
-    while ((codeMatch = codeRe.exec(content)) !== null) {
+    let codeMatch = codeRe.exec(content);
+    while (codeMatch !== null) {
       allCodeBlocks.push({ language: codeMatch[1] ?? "text", code: codeMatch[2] ?? "" });
+      codeMatch = codeRe.exec(content);
     }
     const turnBlocks = [];
-    let blockMatch;
     const blockReLocal = new RegExp(blockRe.source, blockRe.flags);
-    while ((blockMatch = blockReLocal.exec(content)) !== null) {
+    let blockMatch = blockReLocal.exec(content);
+    while (blockMatch !== null) {
       const fields = {};
       for (const line of (blockMatch[2] ?? "").split("\n")) {
         const idx = line.indexOf(":");
@@ -2611,15 +2619,17 @@ async function executeScenario(tc, scenario, config) {
       const blockType = blockMatch[1] ?? "";
       allBlocks.push({ type: blockType, fields });
       turnBlocks.push({ type: blockType, title: fields.title ?? "" });
+      blockMatch = blockReLocal.exec(content);
     }
     let hasToolCall = false;
     if (config.toolCallPatterns) {
       for (const pattern of config.toolCallPatterns) {
         const re = new RegExp(pattern.source, pattern.flags);
-        let toolMatch;
-        while ((toolMatch = re.exec(content)) !== null) {
+        let toolMatch = re.exec(content);
+        while (toolMatch !== null) {
           allToolCalls.push(toolMatch[0]);
           hasToolCall = true;
+          toolMatch = re.exec(content);
         }
       }
     }
@@ -2941,14 +2951,15 @@ var ProductClient = class {
       }
     }
     const blockRe = /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
-    let match;
-    while ((match = blockRe.exec(text)) !== null) {
+    let match = blockRe.exec(text);
+    while (match !== null) {
       const fields = {};
       for (const line of match[2].split("\n")) {
         const idx = line.indexOf(":");
         if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
       }
       blocks.push({ type: match[1], title: fields.title ?? "" });
+      match = blockRe.exec(text);
     }
     return { text, blocks };
   }
@@ -3089,158 +3100,6 @@ var ConvergenceTracker = class {
   }
 };
 
-// src/metrics.ts
-var MODEL_PRICING = {
-  "gpt-4o": { input: 25e-4, output: 0.01 },
-  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
-  "gpt-4-turbo": { input: 0.01, output: 0.03 },
-  "claude-sonnet-4-20250514": { input: 3e-3, output: 0.015 },
-  "claude-opus-4-20250514": { input: 0.015, output: 0.075 },
-  "claude-3-haiku-20240307": { input: 25e-5, output: 125e-5 }
-};
-var FAMILY_PRICING = [
-  [/claude.*opus/, { input: 0.015, output: 0.075 }],
-  [/claude.*haiku/, { input: 8e-4, output: 4e-3 }],
-  [/claude.*sonnet|claude-code|claude-sonnet/, { input: 3e-3, output: 0.015 }],
-  [/gpt-4o-mini/, { input: 15e-5, output: 6e-4 }],
-  [/gpt-5|gpt-4\.1|o[134]\b/, { input: 125e-5, output: 0.01 }],
-  [/gpt-4o|gpt-4/, { input: 25e-4, output: 0.01 }],
-  [/deepseek/, { input: 3e-4, output: 11e-4 }],
-  [/glm|zhipu|zai/, { input: 6e-4, output: 22e-4 }],
-  [/kimi|moonshot/, { input: 6e-4, output: 25e-4 }],
-  [/qwen/, { input: 4e-4, output: 12e-4 }],
-  [/gemini.*flash/, { input: 1e-4, output: 4e-4 }],
-  [/gemini/, { input: 125e-5, output: 5e-3 }],
-  [/llama/, { input: 2e-4, output: 6e-4 }]
-];
-function normalizeModelId(model) {
-  return (model.split("@")[0] ?? model).trim().toLowerCase();
-}
-function resolveModelPricing(model) {
-  if (MODEL_PRICING[model]) return MODEL_PRICING[model];
-  const id = normalizeModelId(model);
-  if (MODEL_PRICING[id]) return MODEL_PRICING[id];
-  for (const [pattern, price] of FAMILY_PRICING) {
-    if (pattern.test(id)) return price;
-  }
-  return null;
-}
-function isModelPriced(model) {
-  return resolveModelPricing(model) !== null;
-}
-var warnedUnpricedModels = /* @__PURE__ */ new Set();
-function estimateTokens(text) {
-  return Math.ceil(text.length / 4);
-}
-function estimateCost(inputTokens, outputTokens, model) {
-  const pricing = resolveModelPricing(model);
-  if (!pricing) {
-    if (!warnedUnpricedModels.has(model)) {
-      warnedUnpricedModels.add(model);
-      console.warn(
-        `estimateCost: no pricing for model "${model}" \u2014 returning 0; add it to MODEL_PRICING/FAMILY_PRICING (cost/Pareto axes will be blank until then)`
-      );
-    }
-    return 0;
-  }
-  return inputTokens / 1e3 * pricing.input + outputTokens / 1e3 * pricing.output;
-}
-var TokenCounter = class {
-  totalInput = 0;
-  totalOutput = 0;
-  totalCost = 0;
-  model;
-  constructor(model = "gpt-4o") {
-    this.model = model;
-  }
-  /** Record tokens for a turn, returns per-turn cost */
-  record(inputTokens, outputTokens) {
-    this.totalInput += inputTokens;
-    this.totalOutput += outputTokens;
-    const cost = estimateCost(inputTokens, outputTokens, this.model);
-    this.totalCost += cost;
-    return cost;
-  }
-  /** Estimate and record from raw text */
-  recordFromText(inputText, outputText) {
-    const inputTokens = estimateTokens(inputText);
-    const outputTokens = estimateTokens(outputText);
-    const cost = this.record(inputTokens, outputTokens);
-    return { inputTokens, outputTokens, cost };
-  }
-  getTotalInput() {
-    return this.totalInput;
-  }
-  getTotalOutput() {
-    return this.totalOutput;
-  }
-  getTotalCost() {
-    return this.totalCost;
-  }
-};
-var MetricsCollector = class {
-  client;
-  workspaceId;
-  metrics = [];
-  constructor(client, workspaceId) {
-    this.client = client;
-    this.workspaceId = workspaceId;
-  }
-  /** Collect metrics after a turn completes */
-  async collect(turn, responseLatencyMs, responseChars, codeBlocksProduced, blocksExtracted, completionCriteriaMet, completionCriteriaTotal, qualityScore, inputTokens = 0, outputTokens = 0, estimatedCostUsd = 0) {
-    const state = await this.getState();
-    const m = {
-      turn,
-      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-      tasks: state.tasks,
-      events: state.events,
-      proposals: state.proposals,
-      vaultFiles: state.vaultFiles.length,
-      responseLatencyMs,
-      responseChars,
-      codeBlocksProduced,
-      blocksExtracted,
-      qualityScore,
-      inputTokens,
-      outputTokens,
-      estimatedCostUsd,
-      totalCostUsd: estimatedCostUsd,
-      completionPercent: completionCriteriaTotal > 0 ? completionCriteriaMet / completionCriteriaTotal * 100 : 0
-    };
-    this.metrics.push(m);
-    return m;
-  }
-  /** Get current product state */
-  async getState() {
-    const [tasks, events, approvals, vaultFiles] = await Promise.all([
-      this.client.getTasks(this.workspaceId),
-      this.client.getEvents(this.workspaceId),
-      this.client.getApprovals(this.workspaceId),
-      this.client.getVaultTree(this.workspaceId)
-    ]);
-    return {
-      tasks: tasks.length,
-      events: events.length,
-      proposals: {
-        pending: approvals.filter((a) => a.status === "pending").length,
-        approved: approvals.filter((a) => a.status === "approved").length,
-        rejected: approvals.filter((a) => a.status === "rejected").length
-      },
-      vaultFiles,
-      codeBlocks: 0,
-      generations: 0
-    };
-  }
-  /** Get all collected metrics */
-  getMetrics() {
-    return [...this.metrics];
-  }
-  /** Get convergence curve (completion% over turns) */
-  getConvergenceCurve() {
-    return this.metrics.map((m) => m.completionPercent);
-  }
-};
-
 // src/driver.ts
 var RIGOR_STANCE = {
   cooperative: "Your stance: a pragmatic early adopter. You accept reasonable answers and only push back on clear gaps or outright errors.",
@@ -4680,8 +4539,8 @@ function analyzeAntiSlop(outputs, config) {
     const lower = output.toLowerCase();
     for (const phrase of config.bannedPhrases) {
       const needle = phrase.toLowerCase();
-      let idx = 0;
-      while ((idx = lower.indexOf(needle, idx)) !== -1) {
+      let idx = lower.indexOf(needle, 0);
+      while (idx !== -1) {
         counts.banned_phrase += 1;
         if (issues.length < 20) {
           issues.push({
@@ -4690,7 +4549,7 @@ function analyzeAntiSlop(outputs, config) {
             example: snippet(output, idx, phrase.length)
           });
         }
-        idx += needle.length;
+        idx = lower.indexOf(needle, idx + needle.length);
       }
     }
     for (const re of config.bannedOpenings) {
@@ -8623,13 +8482,14 @@ function extractAssetUrls(html, baseUrl) {
   const linkRe = /<link\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>/gi;
   const scriptRe = /<script\b[^>]*\bsrc\s*=\s*["']([^"']+)["'][^>]*>/gi;
   for (const re of [linkRe, scriptRe]) {
-    let match;
-    while ((match = re.exec(html)) !== null) {
+    let match = re.exec(html);
+    while (match !== null) {
       const raw = match[1];
       try {
         urls.add(new URL(raw, baseUrl).toString());
       } catch {
       }
+      match = re.exec(html);
     }
   }
   return Array.from(urls);