agent-duelist 0.2.1 → 0.3.1

package/dist/index.js

@@ -1344,38 +1344,49 @@ var costScorer = ({ result }, providerId) => {
   };
 };
 
+// src/utils/deep-equal.ts
+function deepEqual(expected, actual) {
+  if (expected === actual) return true;
+  if (typeof expected === "string" && typeof actual === "string") {
+    return expected.trim().toLowerCase() === actual.trim().toLowerCase();
+  }
+  if (typeof expected !== typeof actual) return false;
+  if (expected === null || actual === null) return expected === actual;
+  if (Array.isArray(expected) && Array.isArray(actual)) {
+    if (expected.length !== actual.length) return false;
+    return expected.every((val, i) => deepEqual(val, actual[i]));
+  }
+  if (typeof expected === "object" && typeof actual === "object") {
+    const objExpected = expected;
+    const objActual = actual;
+    const keysExpected = Object.keys(objExpected);
+    return keysExpected.every((key) => key in objActual && deepEqual(objExpected[key], objActual[key]));
+  }
+  return expected === actual;
+}
+
 // src/scorers/correctness.ts
 var correctnessScorer = ({ task, result }) => {
   if (task.expected === void 0) {
     return { name: "correctness", value: 0.5, details: { reason: "no expected value" } };
   }
-  const match = deepEqual(task.expected, result.output);
+  const actual = normalizeOutput(task.expected, result.output);
+  const match = deepEqual(task.expected, actual);
   return {
     name: "correctness",
     value: match ? 1 : 0,
     details: { expected: task.expected, actual: result.output }
   };
 };
-function deepEqual(a, b) {
-  if (a === b) return true;
-  if (typeof a === "string" && typeof b === "string") {
-    return a.trim().toLowerCase() === b.trim().toLowerCase();
-  }
-  if (typeof a !== typeof b) return false;
-  if (a === null || b === null) return a === b;
-  if (Array.isArray(a) && Array.isArray(b)) {
-    if (a.length !== b.length) return false;
-    return a.every((val, i) => deepEqual(val, b[i]));
-  }
-  if (typeof a === "object" && typeof b === "object") {
-    const objA = a;
-    const objB = b;
-    const keysA = Object.keys(objA);
-    const keysB = Object.keys(objB);
-    if (keysA.length !== keysB.length) return false;
-    return keysA.every((key) => key in objB && deepEqual(objA[key], objB[key]));
-  }
-  return a === b;
+function normalizeOutput(expected, actual) {
+  if (Array.isArray(expected) && !Array.isArray(actual) && typeof actual === "object" && actual !== null) {
+    const entries = Object.entries(actual);
+    const arrayEntries = entries.filter(([, v]) => Array.isArray(v));
+    if (arrayEntries.length === 1) {
+      return arrayEntries[0][1];
+    }
+  }
+  return actual;
 }
 
 // src/scorers/schema-correctness.ts
@@ -1395,7 +1406,14 @@ var schemaCorrectnessScorer = ({ task, result }) => {
       };
     }
   }
-  const parsed = task.schema.safeParse(data);
+  let parsed = task.schema.safeParse(data);
+  if (!parsed.success && !Array.isArray(data) && typeof data === "object" && data !== null) {
+    const arrayEntries = Object.entries(data).filter(([, v]) => Array.isArray(v));
+    if (arrayEntries.length === 1) {
+      const unwrapped = task.schema.safeParse(arrayEntries[0][1]);
+      if (unwrapped.success) parsed = unwrapped;
+    }
+  }
   return {
     name: "schema-correctness",
     value: parsed.success ? 1 : 0,
@@ -1441,18 +1459,36 @@ import OpenAI2, { AzureOpenAI as AzureOpenAI2 } from "openai";
 
 // src/providers/openai.ts
 import OpenAI, { AzureOpenAI } from "openai";
-import { zodToJsonSchema } from "zod-to-json-schema";
+import { zodToJsonSchema as zodToJsonSchema2 } from "zod-to-json-schema";
 
 // src/providers/shared.ts
-var SCHEMA_SYSTEM_MESSAGE = "Respond with valid JSON matching the requested schema.";
+import { zodToJsonSchema } from "zod-to-json-schema";
+function buildSchemaSystemMessage(schema) {
+  if (!schema) return "Respond with valid JSON.";
+  const jsonSchema = zodToJsonSchema(schema, { target: "openAi" });
+  return [
+    "Respond with ONLY valid JSON data. No markdown, no code fences, no explanation.",
+    "",
+    "Your output must conform to this JSON Schema:",
+    JSON.stringify(jsonSchema, null, 2),
+    "",
+    "IMPORTANT: Output the actual data values, NOT the schema definition itself.",
+    'Do NOT include keys like "type", "$schema", or "items" from the schema definition in your response.'
+  ].join("\n");
+}
 function parseSchemaOutput(rawContent, hasSchema) {
   if (!hasSchema) return rawContent;
+  const cleaned = stripCodeFences(rawContent);
   try {
-    return JSON.parse(rawContent);
+    return JSON.parse(cleaned);
   } catch {
     return rawContent;
   }
 }
+function stripCodeFences(content) {
+  const match = content.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?\s*```$/m);
+  return match ? match[1] : content;
+}
 
 // src/providers/openai.ts
 var REQUEST_TIMEOUT_MS = 6e4;
@@ -1501,7 +1537,7 @@ function makeProvider(id, name, model, client, requestModel, stripThinking) {
       if (input.schema) {
         params.response_format = { type: "json_object" };
         params.messages = [
-          { role: "system", content: SCHEMA_SYSTEM_MESSAGE },
+          { role: "system", content: buildSchemaSystemMessage(input.schema) },
           ...params.messages
         ];
       }
@@ -1509,7 +1545,9 @@ function makeProvider(id, name, model, client, requestModel, stripThinking) {
         params.tools = input.tools.map(toolDefToOpenAI);
         params.tool_choice = "auto";
       }
-      const response = await client.chat.completions.create(params, { signal: input.signal });
+      const reqOpts = { signal: input.signal };
+      if (input.timeout) reqOpts.timeout = input.timeout;
+      const response = await client.chat.completions.create(params, reqOpts);
       let totalPromptTokens = response.usage?.prompt_tokens ?? 0;
       let totalCompletionTokens = response.usage?.completion_tokens ?? 0;
       const choice = response.choices[0];
@@ -1543,7 +1581,7 @@ function makeProvider(id, name, model, client, requestModel, stripThinking) {
         const followUp = await client.chat.completions.create({
           model: requestModel,
           messages: toolMessages
-        }, { signal: input.signal });
+        }, reqOpts);
         totalPromptTokens += followUp.usage?.prompt_tokens ?? 0;
         totalCompletionTokens += followUp.usage?.completion_tokens ?? 0;
         finalResponse = followUp;
@@ -1588,7 +1626,7 @@ function toolDefToOpenAI(tool) {
     function: {
       name: tool.name,
       description: tool.description,
-      parameters: zodToJsonSchema(tool.parameters, { target: "openAi" })
+      parameters: zodToJsonSchema2(tool.parameters, { target: "openAi" })
     }
   };
 }
@@ -1637,8 +1675,14 @@ function resolveJudgeClient(configModel, timeoutMs = REQUEST_TIMEOUT_MS) {
   if (!apiKey) return void 0;
   return { client: new OpenAI2({ apiKey, timeout: timeoutMs }), model };
 }
+function isTemperatureError(err) {
+  const msg = err instanceof Error ? err.message : String(err);
+  const lower = msg.toLowerCase();
+  return lower.includes("temperature") && (lower.includes("not supported") || lower.includes("is not allowed") || lower.includes("unsupported") || lower.includes("invalid"));
+}
 function createLlmJudgeScorer(judgeModel, timeoutMs = REQUEST_TIMEOUT_MS) {
   let cached = void 0;
+  let useTemperature = true;
   return async ({ task, result }) => {
     if (task.expected === void 0) {
       return { name: "llm-judge-correctness", value: -1, details: { reason: "no expected value" } };
@@ -1655,35 +1699,24 @@ function createLlmJudgeScorer(judgeModel, timeoutMs = REQUEST_TIMEOUT_MS) {
     }
     const { client, model } = cached;
     const prompt = JUDGE_PROMPT.replace("{task}", task.prompt).replace("{expected}", JSON.stringify(task.expected)).replace("{actual}", JSON.stringify(result.output));
+    const messages = [{ role: "user", content: prompt }];
     try {
-      const response = await client.chat.completions.create({
-        model,
-        messages: [{ role: "user", content: prompt }],
-        max_completion_tokens: 2048
-      });
-      const content = response.choices[0]?.message?.content?.trim() ?? "";
-      const parsed = {};
-      for (const line of content.split("\n")) {
-        const match = line.match(/^(accuracy|completeness|conciseness)\s*:\s*([\d.]+)/i);
-        if (match) parsed[match[1].toLowerCase()] = parseFloat(match[2]);
-      }
-      const accuracy = parsed.accuracy;
-      const completeness = parsed.completeness;
-      const conciseness = parsed.conciseness;
-      if (accuracy == null || completeness == null || conciseness == null || [accuracy, completeness, conciseness].some((s) => isNaN(s) || s < 0 || s > 1)) {
-        return {
-          name: "llm-judge-correctness",
-          value: -1,
-          details: { reason: `judge returned unparseable scores: "${content}"`, model }
-        };
-      }
-      const composite = Math.round((accuracy + completeness + conciseness) / 3 * 100) / 100;
-      return {
-        name: "llm-judge-correctness",
-        value: composite,
-        details: { model, accuracy, completeness, conciseness }
-      };
+      const response = await callJudge(client, model, messages, useTemperature);
+      return parseJudgeResponse(response, model);
     } catch (err) {
+      if (useTemperature && isTemperatureError(err)) {
+        useTemperature = false;
+        try {
+          const response = await callJudge(client, model, messages, false);
+          return parseJudgeResponse(response, model);
+        } catch (retryErr) {
+          return {
+            name: "llm-judge-correctness",
+            value: -1,
+            details: { reason: `judge call failed: ${retryErr instanceof Error ? retryErr.message : String(retryErr)}` }
+          };
+        }
+      }
       return {
         name: "llm-judge-correctness",
         value: -1,
@@ -1692,18 +1725,89 @@ function createLlmJudgeScorer(judgeModel, timeoutMs = REQUEST_TIMEOUT_MS) {
     }
   };
 }
+async function callJudge(client, model, messages, withTemperature) {
+  return client.chat.completions.create({
+    model,
+    messages,
+    max_completion_tokens: 2048,
+    ...withTemperature ? { temperature: 0 } : {}
+  });
+}
+function parseJudgeResponse(response, model) {
+  const content = response.choices[0]?.message?.content?.trim() ?? "";
+  const parsed = {};
+  for (const line of content.split("\n")) {
+    const match = line.match(/^(accuracy|completeness|conciseness)\s*:\s*([\d.]+)/i);
+    if (match) parsed[match[1].toLowerCase()] = parseFloat(match[2]);
+  }
+  const accuracy = parsed.accuracy;
+  const completeness = parsed.completeness;
+  const conciseness = parsed.conciseness;
+  if (accuracy == null || completeness == null || conciseness == null || [accuracy, completeness, conciseness].some((s) => isNaN(s) || s < 0 || s > 1)) {
+    return {
+      name: "llm-judge-correctness",
+      value: -1,
+      details: { reason: `judge returned unparseable scores: "${content}"`, model }
+    };
+  }
+  const composite = Math.round((accuracy + completeness + conciseness) / 3 * 100) / 100;
+  return {
+    name: "llm-judge-correctness",
+    value: composite,
+    details: { model, accuracy, completeness, conciseness }
+  };
+}
 
 // src/scorers/tool-usage.ts
 var toolUsageScorer = ({ task, result }) => {
-  const expectedToolName = task.tools?.[0]?.name;
-  if (!expectedToolName) {
+  if (!task.tools?.length) {
     return { name: "tool-usage", value: -1, details: { reason: "no tools configured on task" } };
   }
-  const usedTool = result.toolCalls?.some((c) => c.name === expectedToolName) ?? false;
+  const calls = result.toolCalls ?? [];
+  const expectedIsObject = task.expected !== void 0 && typeof task.expected === "object" && task.expected !== null && !Array.isArray(task.expected);
+  if (expectedIsObject) {
+    const matchingCall = calls.find((c) => {
+      const toolDef = task.tools.find((t) => t.name === c.name);
+      if (!toolDef) return false;
+      return deepEqual(task.expected, c.arguments);
+    });
+    if (matchingCall) {
+      return {
+        name: "tool-usage",
+        value: 1,
+        details: { matchedTool: matchingCall.name, arguments: matchingCall.arguments, toolCalls: calls }
+      };
+    }
+    const expectedKeys = Object.keys(task.expected);
+    const partialMatch = calls.find((c) => {
+      if (typeof c.arguments !== "object" || c.arguments === null) return false;
+      const argKeys = Object.keys(c.arguments);
+      return expectedKeys.some((k) => argKeys.includes(k));
+    });
+    if (partialMatch) {
+      return {
+        name: "tool-usage",
+        value: 0.5,
+        details: {
+          reason: "correct tool but wrong arguments",
+          expected: task.expected,
+          actual: partialMatch.arguments,
+          toolCalls: calls
+        }
+      };
+    }
+    return {
+      name: "tool-usage",
+      value: 0,
+      details: { reason: "no matching tool call", expected: task.expected, toolCalls: calls }
+    };
+  }
+  const expectedToolName = task.tools[0].name;
+  const usedTool = calls.some((c) => c.name === expectedToolName);
   return {
     name: "tool-usage",
     value: usedTool ? 1 : 0,
-    details: { expectedToolName, usedTool, toolCalls: result.toolCalls ?? [] }
+    details: { expectedToolName, usedTool, toolCalls: calls }
   };
 };
 
@@ -1764,7 +1868,8 @@ async function runBenchmarks(options) {
               prompt: task.prompt,
               schema: task.schema,
               tools: task.tools,
-              signal
+              signal,
+              timeout
             }), timeout);
             const scores = await Promise.all(
               scorers.map((scorer) => scorer({ task, result: taskResult }, provider.id))
@@ -1973,37 +2078,76 @@ function computeColumnStats(providerData, scorerNames) {
   }
   return stats;
 }
+var QUALITY_SCORERS = /* @__PURE__ */ new Set([
+  "correctness",
+  "schema-correctness",
+  "fuzzy-similarity",
+  "llm-judge-correctness",
+  "tool-usage"
+]);
+function passesQualityGate(providerId, columnStats) {
+  const qualityColumns = [...columnStats.keys()].filter((k) => QUALITY_SCORERS.has(k));
+  if (qualityColumns.length === 0) return true;
+  return qualityColumns.some((col) => {
+    const val = columnStats.get(col)?.values.get(providerId);
+    return val !== void 0 && val > 0;
+  });
+}
 function computeMedals(columnStats, providerIds) {
   const medals = /* @__PURE__ */ new Map();
   if (providerIds.length < 2) {
     for (const id of providerIds) medals.set(id, "none");
     return medals;
   }
-  const wins = /* @__PURE__ */ new Map();
-  for (const id of providerIds) wins.set(id, 0);
-  for (const [, colStats] of columnStats) {
+  const eligible = new Set(providerIds.filter((id) => passesQualityGate(id, columnStats)));
+  const qualityWins = /* @__PURE__ */ new Map();
+  const efficiencyWins = /* @__PURE__ */ new Map();
+  for (const id of providerIds) {
+    qualityWins.set(id, 0);
+    efficiencyWins.set(id, 0);
+  }
+  for (const [colName, colStats] of columnStats) {
     if (colStats.best === void 0) continue;
     const bestProviders = [...colStats.values.entries()].filter(([, v]) => v !== void 0 && v === colStats.best);
     if (bestProviders.length === 1) {
-      wins.set(bestProviders[0][0], (wins.get(bestProviders[0][0]) ?? 0) + 1);
+      const winnerId = bestProviders[0][0];
+      if (QUALITY_SCORERS.has(colName)) {
+        qualityWins.set(winnerId, (qualityWins.get(winnerId) ?? 0) + 1);
+      } else {
+        efficiencyWins.set(winnerId, (efficiencyWins.get(winnerId) ?? 0) + 1);
+      }
     }
   }
-  const totalWins = [...wins.values()].reduce((a, b) => a + b, 0);
+  const totalWins = [...qualityWins.values()].reduce((a, b) => a + b, 0) + [...efficiencyWins.values()].reduce((a, b) => a + b, 0);
   if (totalWins === 0) {
     for (const id of providerIds) medals.set(id, "none");
     return medals;
   }
-  const sorted = [...wins.entries()].sort(
-    (a, b) => b[1] - a[1] || a[0].localeCompare(b[0])
-  );
+  const eligibleSorted = providerIds.filter((id) => eligible.has(id)).sort((a, b) => {
+    const qDiff = (qualityWins.get(b) ?? 0) - (qualityWins.get(a) ?? 0);
+    if (qDiff !== 0) return qDiff;
+    const eDiff = (efficiencyWins.get(b) ?? 0) - (efficiencyWins.get(a) ?? 0);
+    if (eDiff !== 0) return eDiff;
+    return a.localeCompare(b);
+  });
   const medalList = ["gold", "silver", "bronze"];
   let rank = 0;
-  for (let i = 0; i < sorted.length; i++) {
-    if (i > 0 && sorted[i][1] < sorted[i - 1][1]) {
-      rank = i;
+  for (let i = 0; i < eligibleSorted.length; i++) {
+    if (i > 0) {
+      const prevQ = qualityWins.get(eligibleSorted[i - 1]) ?? 0;
+      const currQ = qualityWins.get(eligibleSorted[i]) ?? 0;
+      if (currQ < prevQ) {
+        rank = i;
+      } else if (currQ === prevQ) {
+        const prevE = efficiencyWins.get(eligibleSorted[i - 1]) ?? 0;
+        const currE = efficiencyWins.get(eligibleSorted[i]) ?? 0;
+        if (currE < prevE) rank = i;
+      }
     }
-    const hasWins = sorted[i][1] > 0;
-    medals.set(sorted[i][0], hasWins && rank < medalList.length ? medalList[rank] : "none");
+    medals.set(eligibleSorted[i], rank < medalList.length ? medalList[rank] : "none");
+  }
+  for (const id of providerIds) {
+    if (!eligible.has(id)) medals.set(id, "none");
   }
   return medals;
 }
@@ -2400,24 +2544,10 @@ function printSummary(results, providers, byProvider) {
       console.log(`  ${medal} Cheapest:      ${bold(byCost.id)} ${dim(providerLabel(byCost.id))}  ${brightGreen}${boldCode}${costStr}${reset}`);
     }
   }
-  if (!single) {
-    const wins = /* @__PURE__ */ new Map();
-    for (const id of providers) wins.set(id, 0);
-    if (byCorrectness) wins.set(byCorrectness.id, (wins.get(byCorrectness.id) ?? 0) + 1);
-    if (byLatency && byLatency.avg !== Infinity) wins.set(byLatency.id, (wins.get(byLatency.id) ?? 0) + 1);
-    if (byCost?.avg !== void 0) wins.set(byCost.id, (wins.get(byCost.id) ?? 0) + 1);
-    const maxWins = Math.max(...wins.values());
-    if (maxWins > 0) {
-      const topProviders = [...wins.entries()].filter(([, w]) => w === maxWins);
-      console.log("");
-      if (topProviders.length === 1) {
-        const [winnerId, winCount] = topProviders[0];
-        console.log(`  \u{1F3C6} Overall:      ${brightGreen}${boldCode}${winnerId}${reset} ${dim(providerLabel(winnerId))}  ${dim(`(${winCount}/3 categories)`)}`);
-      } else {
-        const names = topProviders.map(([id]) => bold(id)).join(dim(", "));
-        console.log(`  \u{1F3C6} Overall:      ${names}  ${dim(`(tied at ${maxWins}/3)`)}`);
-      }
-    }
+  if (!single && byCorrectness && byCorrectness.avg > 0) {
+    console.log("");
+    const pct = `${Math.round(byCorrectness.avg * 100)}%`;
+    console.log(`  \u{1F3C6} Overall:      ${brightGreen}${boldCode}${byCorrectness.id}${reset} ${dim(providerLabel(byCorrectness.id))}  ${dim(`(${pct} avg correctness)`)}`);
   }
   console.log("");
 }
@@ -2451,15 +2581,15 @@ function defineArena(config) {
   if (config.providers.length === 0) {
     throw new Error("At least one provider is required");
   }
-  if (config.tasks.length === 0) {
-    throw new Error("At least one task is required");
-  }
   const scorerNames = config.scorers ?? ["latency", "cost", "correctness"];
   const scorerFns = resolveScorers(scorerNames, config.judgeModel, config.timeout);
   const runs = config.runs ?? 1;
   return {
     config,
     async run(options) {
+      if (config.tasks.length === 0) {
+        throw new Error("At least one task is required");
+      }
       return runBenchmarks({
         providers: config.providers,
         tasks: config.tasks,
@@ -2485,13 +2615,15 @@ function anthropic(model, options) {
     model,
     async run(input) {
       const start = Date.now();
-      const systemMessage = input.schema ? SCHEMA_SYSTEM_MESSAGE : void 0;
+      const systemMessage = input.schema ? buildSchemaSystemMessage(input.schema) : void 0;
+      const reqOpts = { signal: input.signal };
+      if (input.timeout) reqOpts.timeout = input.timeout;
       const response = await client.messages.create({
         model,
         max_tokens: maxTokens,
         system: systemMessage,
         messages: [{ role: "user", content: input.prompt }]
-      }, { signal: input.signal });
+      }, reqOpts);
       const latencyMs = Date.now() - start;
       const textBlock = response.content.find((b) => b.type === "text");
       const rawContent = textBlock?.type === "text" ? textBlock.text : "";
@@ -2636,17 +2768,8 @@ function htmlReporter(results) {
     return { id, avg };
   }).filter((p) => p.avg !== void 0).sort((a, b) => a.avg - b.avg)[0];
   let overallWinner;
-  if (multi) {
-    const wins = /* @__PURE__ */ new Map();
-    for (const id of providers) wins.set(id, 0);
-    if (byCorrectness) wins.set(byCorrectness.id, (wins.get(byCorrectness.id) ?? 0) + 1);
-    if (byLatency && byLatency.avg !== Infinity) wins.set(byLatency.id, (wins.get(byLatency.id) ?? 0) + 1);
-    if (byCost?.avg !== void 0) wins.set(byCost.id, (wins.get(byCost.id) ?? 0) + 1);
-    const maxWins = Math.max(...wins.values());
-    if (maxWins > 0) {
-      const tops = [...wins.entries()].filter(([, w]) => w === maxWins);
-      if (tops.length === 1) overallWinner = tops[0][0];
-    }
+  if (multi && byCorrectness && byCorrectness.avg > 0) {
+    overallWinner = byCorrectness.id;
   }
   const errorResults = results.filter((r) => r.error);
   const deduped = dedupeErrors(errorResults);
@@ -3167,7 +3290,7 @@ function renderErrors(errors) {
     </div>`;
   }).join("\n");
   return `<section class="errors-section">
-  <h2 class="errors-title" onclick="this.nextElementSibling.style.display=this.nextElementSibling.style.display==='none'?'block':'block'">Errors</h2>
+  <h2 class="errors-title" onclick="this.nextElementSibling.style.display=this.nextElementSibling.style.display==='none'?'block':'none'">Errors</h2>
   <div class="errors-list">
     ${items}
   </div>
@@ -3231,6 +3354,384 @@ function renderScript(taskCount) {
 </script>`;
 }
 
+// src/packs/structured-output.ts
+import { z } from "zod";
+var structuredOutputPack = {
+  name: "structured-output",
+  label: "Structured Output",
+  description: "Zod schema stress test \u2014 flat objects, nesting, arrays, enums, empty arrays, and adversarial input",
+  tasks: [
+    {
+      name: "so:flat-entity",
+      prompt: "Extract the person's details from this text: 'Maria Garcia, age 34, works as a software architect in Barcelona, Spain. Her employee ID is EMP-2847.' Return as JSON.",
+      expected: {
+        name: "Maria Garcia",
+        age: 34,
+        role: "software architect",
+        city: "Barcelona",
+        country: "Spain",
+        employeeId: "EMP-2847"
+      },
+      schema: z.object({
+        name: z.string(),
+        age: z.number(),
+        role: z.string(),
+        city: z.string(),
+        country: z.string(),
+        employeeId: z.string()
+      })
+    },
+    {
+      name: "so:nested-object",
+      prompt: "Parse this shipping label into structured JSON: 'Ship to: Acme Corp, Attn: John Lee, 4th Floor, 742 Evergreen Terrace, Springfield, IL 62704, USA. Order #ORD-9912, 3 items, 2.4kg, express shipping.' Use shippingMethod values: standard, express, or overnight. Return as JSON.",
+      expected: {
+        recipient: { company: "Acme Corp", contact: "John Lee", floor: "4th Floor" },
+        address: { street: "742 Evergreen Terrace", city: "Springfield", state: "IL", zip: "62704", country: "USA" },
+        order: { id: "ORD-9912", itemCount: 3, weightKg: 2.4, shippingMethod: "express" }
+      },
+      schema: z.object({
+        recipient: z.object({ company: z.string(), contact: z.string(), floor: z.string() }),
+        address: z.object({
+          street: z.string(),
+          city: z.string(),
+          state: z.string(),
+          zip: z.string(),
+          country: z.string()
+        }),
+        order: z.object({
+          id: z.string(),
+          itemCount: z.number(),
+          weightKg: z.number(),
+          shippingMethod: z.enum(["standard", "express", "overnight"])
+        })
+      })
+    },
+    {
+      name: "so:array-of-objects",
+      prompt: "Extract all mentioned products with their prices and categories from this text: 'Our summer sale includes the UltraWidget Pro ($49.99, Electronics), ComfortMax Chair ($199.00, Furniture), and AquaPure Filter ($24.50, Home & Kitchen). The SmartLamp Mini is also available at $34.99 in the Electronics category.' Return as a JSON array.",
+      expected: [
+        { name: "UltraWidget Pro", price: 49.99, category: "Electronics" },
+        { name: "ComfortMax Chair", price: 199, category: "Furniture" },
+        { name: "AquaPure Filter", price: 24.5, category: "Home & Kitchen" },
+        { name: "SmartLamp Mini", price: 34.99, category: "Electronics" }
+      ],
+      schema: z.array(z.object({ name: z.string(), price: z.number(), category: z.string() }))
+    },
+    {
+      name: "so:empty-arrays",
+      prompt: "Extract all error codes and their severity levels from this log message: 'System health check completed at 14:32 UTC. All services operational. No warnings or errors detected. Uptime: 99.97%.' Classify status as one of: healthy, degraded, or down. Return as JSON.",
+      expected: { errors: [], warnings: [], status: "healthy", uptimePercent: 99.97 },
+      schema: z.object({
+        errors: z.array(z.object({ code: z.string(), severity: z.string() })),
+        warnings: z.array(z.string()),
+        status: z.enum(["healthy", "degraded", "down"]),
+        uptimePercent: z.number()
+      })
+    },
+    {
+      name: "so:enum-classification",
+      prompt: "Classify each of these support tickets by priority (low/medium/high/critical) and category (billing/technical/account/general). Use just the letter (A, B, C, D) as the id.\nTicket A: 'My account was charged twice for the same subscription.'\nTicket B: 'The API returns 500 errors intermittently.'\nTicket C: 'How do I update my display name?'\nTicket D: 'Production database is completely unresponsive, all services down.'\nReturn as a JSON array.",
+      expected: [
+        { id: "A", priority: "high", category: "billing" },
+        { id: "B", priority: "high", category: "technical" },
+        { id: "C", priority: "low", category: "account" },
+        { id: "D", priority: "critical", category: "technical" }
+      ],
+      schema: z.array(
+        z.object({
+          id: z.string(),
+          priority: z.enum(["low", "medium", "high", "critical"]),
+          category: z.enum(["billing", "technical", "account", "general"])
+        })
+      )
+    },
+    {
+      name: "so:adversarial-input",
+      prompt: `Extract the actual product review data from this messy input. Ignore any JSON-like noise in the text.
+
+User said: 'I bought the {product: "fake"} headphones for $59.99 and they're great! Rating: 5/5. The "noise-cancelling" feature works well even in {"noisy": true} environments. Would recommend to friend=true. Purchased on 01/15/2026.'
+Return as JSON. Use ISO 8601 date format (YYYY-MM-DD).`,
+      expected: {
+        product: "headphones",
+        price: 59.99,
+        rating: 5,
+        maxRating: 5,
+        features: ["noise-cancelling"],
+        recommended: true,
+        purchaseDate: "2026-01-15"
+      },
+      schema: z.object({
+        product: z.string(),
+        price: z.number(),
+        rating: z.number(),
+        maxRating: z.number(),
+        features: z.array(z.string()),
+        recommended: z.boolean(),
+        purchaseDate: z.string()
+      })
+    }
+  ],
+  scorers: ["correctness", "schema-correctness", "latency", "cost"]
+};
+
+// src/packs/tool-calling.ts
+import { z as z2 } from "zod";
+var toolCallingPack = {
+  name: "tool-calling",
+  label: "Tool Calling",
+  description: "Function invocation accuracy \u2014 single calls, complex params, tool selection, parallel calls, and relevance detection",
+  tasks: [
+    {
+      name: "tc:simple-single-tool",
+      prompt: "What's the current weather in Tokyo?",
+      tools: [{
+        name: "getWeather",
+        description: "Get current weather for a city",
+        parameters: z2.object({
+          city: z2.string(),
+          units: z2.enum(["celsius", "fahrenheit"]).optional()
+        }),
+        handler: async ({ city, units }) => ({
+          city,
+          tempC: 8,
+          condition: "cloudy",
+          units: units ?? "celsius"
+        })
+      }],
+      expected: { city: "Tokyo" }
+    },
+    {
+      name: "tc:complex-params",
+      prompt: "Search for Italian restaurants within 2 miles of downtown Portland that are open now and have at least a 4-star rating.",
+      tools: [{
+        name: "searchRestaurants",
+        description: "Search for restaurants matching criteria",
+        parameters: z2.object({
+          cuisine: z2.string(),
+          location: z2.string(),
+          radiusMiles: z2.number(),
+          minRating: z2.number(),
+          openNow: z2.boolean()
+        }),
+        handler: async (_args) => ({
+          results: [{ name: "Trattoria Roma", rating: 4.5, distance: 1.2 }]
+        })
+      }],
+      expected: {
+        cuisine: "Italian",
+        location: "downtown Portland",
+        radiusMiles: 2,
+        minRating: 4,
+        openNow: true
+      }
+    },
+    {
+      name: "tc:select-from-many",
+      prompt: "Convert 150 USD to Euros.",
+      tools: [
+        {
+          name: "getWeather",
+          description: "Get current weather for a city",
+          parameters: z2.object({ city: z2.string() }),
+          handler: async () => ({ tempC: 20 })
+        },
+        {
+          name: "convertCurrency",
+          description: "Convert an amount between currencies",
+          parameters: z2.object({
+            amount: z2.number(),
+            from: z2.string(),
+            to: z2.string()
+          }),
+          handler: async ({ amount, from, to }) => ({
+            amount,
+            from,
+            to,
+            result: 138.75,
+            rate: 0.925
+          })
+        },
+        {
+          name: "translateText",
+          description: "Translate text between languages",
+          parameters: z2.object({ text: z2.string(), targetLang: z2.string() }),
+          handler: async () => ({ translated: "" })
+        },
+        {
+          name: "calculateTip",
+          description: "Calculate tip amount for a bill",
+          parameters: z2.object({ billAmount: z2.number(), tipPercent: z2.number() }),
+          handler: async () => ({ tip: 0 })
+        }
+      ],
+      expected: { amount: 150, from: "USD", to: "EUR" }
+    },
+    {
+      name: "tc:parallel-calls",
+      prompt: "I'm planning a trip. What's the weather like in both Paris and London right now?",
+      tools: [{
+        name: "getWeather",
+        description: "Get current weather for a city",
+        parameters: z2.object({ city: z2.string() }),
+        handler: async ({ city }) => {
+          const data = {
+            Paris: { tempC: 12, condition: "partly cloudy" },
+            London: { tempC: 9, condition: "rainy" }
+          };
+          return data[city] ?? { tempC: 15, condition: "unknown" };
+        }
+      }],
+      expected: "weather data for Paris and London"
+    }
+  ],
+  scorers: ["tool-usage", "latency", "cost"]
+};
+
+// src/packs/reasoning.ts
+import { z as z3 } from "zod";
+var reasoningPack = {
+  name: "reasoning",
+  label: "Reasoning",
+  description: "Logic, math, and multi-step thinking \u2014 arithmetic, deduction, data interpretation, critical path, and business rules",
+  tasks: [
+    {
+      name: "rs:saas-mrr-calc",
+      prompt: `A SaaS company charges $49/month for the basic plan and $149/month for pro.
+In Q1 they had 200 basic subscribers and 85 pro subscribers.
+In Q2, 15% of basic users upgraded to pro and they gained 40 new basic subscribers.
+No one churned. What is the Q2 monthly recurring revenue (MRR)?
+Return as JSON with your reasoning and the final MRR number.`,
+      expected: { mrr: 27425 },
+      schema: z3.object({
+        reasoning: z3.string().optional(),
+        mrr: z3.number()
+      })
+    },
+    {
+      name: "rs:logical-deduction",
+      prompt: `Five developers \u2014 Alice, Bob, Carol, Dave, and Eve \u2014 each use a different
+primary language: Rust, TypeScript, Python, Go, and Java. Given:
+1. Alice does not use Python, Java, or Go.
+2. Bob uses TypeScript.
+3. Carol uses neither Rust nor Go.
+4. Dave does not use Java.
+5. Eve uses neither Rust, Go, nor Java.
+What language does each developer use? Return as JSON.`,
+      expected: {
+        Alice: "Rust",
+        Bob: "TypeScript",
+        Carol: "Java",
+        Dave: "Go",
+        Eve: "Python"
+      },
+      schema: z3.object({
+        Alice: z3.string(),
+        Bob: z3.string(),
+        Carol: z3.string(),
+        Dave: z3.string(),
+        Eve: z3.string()
+      })
+    },
+    {
+      name: "rs:data-interpretation",
+      prompt: `Given this quarterly revenue data:
+| Quarter | Revenue | Growth |
+|---------|---------|--------|
+| Q1 2025 | $2.1M   | -      |
+| Q2 2025 | $2.4M   | 14.3%  |
+| Q3 2025 | $2.2M   | -8.3%  |
+| Q4 2025 | $2.8M   | 27.3%  |
+
+Which quarter had the highest absolute revenue increase compared to the previous
+quarter? What was the full-year total revenue in millions? Return as JSON.`,
+      expected: {
+        highestGrowthQuarter: "Q4 2025",
+        absoluteIncrease: 0.6,
+        fullYearRevenue: 9.5
+      },
+      schema: z3.object({
+        highestGrowthQuarter: z3.string(),
+        absoluteIncrease: z3.number(),
+        fullYearRevenue: z3.number()
+      })
+    },
+    {
+      name: "rs:critical-path",
+      prompt: `A deployment pipeline has these stages with dependencies:
+- Build (3 min, no dependency)
+- Unit tests (5 min, depends on Build)
+- Integration tests (8 min, depends on Build)
+- Security scan (4 min, depends on Build)
+- Staging deploy (2 min, depends on Unit tests AND Integration tests AND Security scan)
+- Smoke tests (3 min, depends on Staging deploy)
+
+Assuming stages run in parallel where possible, what is the total pipeline
+duration in minutes? Which stages are on the critical path? Return as JSON.`,
+      expected: {
+        totalMinutes: 16,
+        criticalPath: ["Build", "Integration tests", "Staging deploy", "Smoke tests"]
+      },
+      schema: z3.object({
+        totalMinutes: z3.number(),
+        criticalPath: z3.array(z3.string())
+      })
+    },
+    {
+      name: "rs:pricing-rules",
+      prompt: `Apply these pricing rules to each customer and return the final price:
+Rules:
+- Base price: $100
+- Enterprise customers (>100 seats): 30% discount
+- Annual billing: additional 15% off the discounted price
+- Non-profit organizations: flat $50 regardless of other rules
+
+Customers:
+A: 50 seats, monthly billing, for-profit
+B: 200 seats, annual billing, for-profit
+C: 75 seats, annual billing, non-profit
+D: 150 seats, monthly billing, for-profit
+
+Return as a JSON array with customer id and finalPrice.`,
+      expected: [
+        { id: "A", finalPrice: 100 },
+        { id: "B", finalPrice: 59.5 },
+        { id: "C", finalPrice: 50 },
+        { id: "D", finalPrice: 70 }
+      ],
+      schema: z3.array(z3.object({
+        id: z3.string(),
+        finalPrice: z3.number()
+      }))
+    }
+  ],
+  scorers: ["correctness", "latency", "cost"]
+};
+
+// src/packs/index.ts
+var registry = /* @__PURE__ */ new Map();
+function register(pack) {
+  registry.set(pack.name, pack);
+}
+register(structuredOutputPack);
+register(toolCallingPack);
+register(reasoningPack);
+function loadPack(name) {
+  const pack = registry.get(name);
+  if (!pack) {
+    const available = [...registry.keys()].join(", ");
+    throw new Error(`Unknown pack "${name}". Available packs: ${available}`);
+  }
+  return pack;
+}
+function listPacks() {
+  return [...registry.values()].map((p) => ({
+    name: p.name,
+    label: p.label,
+    description: p.description,
+    taskCount: p.tasks.length
+  }));
+}
+
 // src/ci.ts
 import { readFileSync, writeFileSync, mkdirSync } from "fs";
 import { dirname } from "path";
@@ -3521,7 +4022,9 @@ export {
   gemini,
   htmlReporter,
   jsonReporter,
+  listPacks,
   loadBaseline,
+  loadPack,
   markdownReporter,
   openai,
   openaiCompatible,