agent-duelist 0.3.0 → 0.3.1

package/dist/index.js

@@ -1344,6 +1344,27 @@ var costScorer = ({ result }, providerId) => {
   };
 };
 
+// src/utils/deep-equal.ts
+function deepEqual(expected, actual) {
+  if (expected === actual) return true;
+  if (typeof expected === "string" && typeof actual === "string") {
+    return expected.trim().toLowerCase() === actual.trim().toLowerCase();
+  }
+  if (typeof expected !== typeof actual) return false;
+  if (expected === null || actual === null) return expected === actual;
+  if (Array.isArray(expected) && Array.isArray(actual)) {
+    if (expected.length !== actual.length) return false;
+    return expected.every((val, i) => deepEqual(val, actual[i]));
+  }
+  if (typeof expected === "object" && typeof actual === "object") {
+    const objExpected = expected;
+    const objActual = actual;
+    const keysExpected = Object.keys(objExpected);
+    return keysExpected.every((key) => key in objActual && deepEqual(objExpected[key], objActual[key]));
+  }
+  return expected === actual;
+}
+
 // src/scorers/correctness.ts
 var correctnessScorer = ({ task, result }) => {
   if (task.expected === void 0) {
@@ -1367,25 +1388,6 @@ function normalizeOutput(expected, actual) {
   }
   return actual;
 }
-function deepEqual(expected, actual) {
-  if (expected === actual) return true;
-  if (typeof expected === "string" && typeof actual === "string") {
-    return expected.trim().toLowerCase() === actual.trim().toLowerCase();
-  }
-  if (typeof expected !== typeof actual) return false;
-  if (expected === null || actual === null) return expected === actual;
-  if (Array.isArray(expected) && Array.isArray(actual)) {
-    if (expected.length !== actual.length) return false;
-    return expected.every((val, i) => deepEqual(val, actual[i]));
-  }
-  if (typeof expected === "object" && typeof actual === "object") {
-    const objExpected = expected;
-    const objActual = actual;
-    const keysExpected = Object.keys(objExpected);
-    return keysExpected.every((key) => key in objActual && deepEqual(objExpected[key], objActual[key]));
-  }
-  return expected === actual;
-}
 
 // src/scorers/schema-correctness.ts
 var schemaCorrectnessScorer = ({ task, result }) => {
@@ -1758,15 +1760,54 @@ function parseJudgeResponse(response, model) {
 
 // src/scorers/tool-usage.ts
 var toolUsageScorer = ({ task, result }) => {
-  const expectedToolName = task.tools?.[0]?.name;
-  if (!expectedToolName) {
+  if (!task.tools?.length) {
     return { name: "tool-usage", value: -1, details: { reason: "no tools configured on task" } };
   }
-  const usedTool = result.toolCalls?.some((c) => c.name === expectedToolName) ?? false;
+  const calls = result.toolCalls ?? [];
+  const expectedIsObject = task.expected !== void 0 && typeof task.expected === "object" && task.expected !== null && !Array.isArray(task.expected);
+  if (expectedIsObject) {
+    const matchingCall = calls.find((c) => {
+      const toolDef = task.tools.find((t) => t.name === c.name);
+      if (!toolDef) return false;
+      return deepEqual(task.expected, c.arguments);
+    });
+    if (matchingCall) {
+      return {
+        name: "tool-usage",
+        value: 1,
+        details: { matchedTool: matchingCall.name, arguments: matchingCall.arguments, toolCalls: calls }
+      };
+    }
+    const expectedKeys = Object.keys(task.expected);
+    const partialMatch = calls.find((c) => {
+      if (typeof c.arguments !== "object" || c.arguments === null) return false;
+      const argKeys = Object.keys(c.arguments);
+      return expectedKeys.some((k) => argKeys.includes(k));
+    });
+    if (partialMatch) {
+      return {
+        name: "tool-usage",
+        value: 0.5,
+        details: {
+          reason: "correct tool but wrong arguments",
+          expected: task.expected,
+          actual: partialMatch.arguments,
+          toolCalls: calls
+        }
+      };
+    }
+    return {
+      name: "tool-usage",
+      value: 0,
+      details: { reason: "no matching tool call", expected: task.expected, toolCalls: calls }
+    };
+  }
+  const expectedToolName = task.tools[0].name;
+  const usedTool = calls.some((c) => c.name === expectedToolName);
   return {
     name: "tool-usage",
     value: usedTool ? 1 : 0,
-    details: { expectedToolName, usedTool, toolCalls: result.toolCalls ?? [] }
+    details: { expectedToolName, usedTool, toolCalls: calls }
   };
 };
 
@@ -3433,12 +3474,247 @@ Return as JSON. Use ISO 8601 date format (YYYY-MM-DD).`,
   scorers: ["correctness", "schema-correctness", "latency", "cost"]
 };
 
+// src/packs/tool-calling.ts
+import { z as z2 } from "zod";
+var toolCallingPack = {
+  name: "tool-calling",
+  label: "Tool Calling",
+  description: "Function invocation accuracy \u2014 single calls, complex params, tool selection, parallel calls, and relevance detection",
+  tasks: [
+    {
+      name: "tc:simple-single-tool",
+      prompt: "What's the current weather in Tokyo?",
+      tools: [{
+        name: "getWeather",
+        description: "Get current weather for a city",
+        parameters: z2.object({
+          city: z2.string(),
+          units: z2.enum(["celsius", "fahrenheit"]).optional()
+        }),
+        handler: async ({ city, units }) => ({
+          city,
+          tempC: 8,
+          condition: "cloudy",
+          units: units ?? "celsius"
+        })
+      }],
+      expected: { city: "Tokyo" }
+    },
+    {
+      name: "tc:complex-params",
+      prompt: "Search for Italian restaurants within 2 miles of downtown Portland that are open now and have at least a 4-star rating.",
+      tools: [{
+        name: "searchRestaurants",
+        description: "Search for restaurants matching criteria",
+        parameters: z2.object({
+          cuisine: z2.string(),
+          location: z2.string(),
+          radiusMiles: z2.number(),
+          minRating: z2.number(),
+          openNow: z2.boolean()
+        }),
+        handler: async (_args) => ({
+          results: [{ name: "Trattoria Roma", rating: 4.5, distance: 1.2 }]
+        })
+      }],
+      expected: {
+        cuisine: "Italian",
+        location: "downtown Portland",
+        radiusMiles: 2,
+        minRating: 4,
+        openNow: true
+      }
+    },
+    {
+      name: "tc:select-from-many",
+      prompt: "Convert 150 USD to Euros.",
+      tools: [
+        {
+          name: "getWeather",
+          description: "Get current weather for a city",
+          parameters: z2.object({ city: z2.string() }),
+          handler: async () => ({ tempC: 20 })
+        },
+        {
+          name: "convertCurrency",
+          description: "Convert an amount between currencies",
+          parameters: z2.object({
+            amount: z2.number(),
+            from: z2.string(),
+            to: z2.string()
+          }),
+          handler: async ({ amount, from, to }) => ({
+            amount,
+            from,
+            to,
+            result: 138.75,
+            rate: 0.925
+          })
+        },
+        {
+          name: "translateText",
+          description: "Translate text between languages",
+          parameters: z2.object({ text: z2.string(), targetLang: z2.string() }),
+          handler: async () => ({ translated: "" })
+        },
+        {
+          name: "calculateTip",
+          description: "Calculate tip amount for a bill",
+          parameters: z2.object({ billAmount: z2.number(), tipPercent: z2.number() }),
+          handler: async () => ({ tip: 0 })
+        }
+      ],
+      expected: { amount: 150, from: "USD", to: "EUR" }
+    },
+    {
+      name: "tc:parallel-calls",
+      prompt: "I'm planning a trip. What's the weather like in both Paris and London right now?",
+      tools: [{
+        name: "getWeather",
+        description: "Get current weather for a city",
+        parameters: z2.object({ city: z2.string() }),
+        handler: async ({ city }) => {
+          const data = {
+            Paris: { tempC: 12, condition: "partly cloudy" },
+            London: { tempC: 9, condition: "rainy" }
+          };
+          return data[city] ?? { tempC: 15, condition: "unknown" };
+        }
+      }],
+      expected: "weather data for Paris and London"
+    }
+  ],
+  scorers: ["tool-usage", "latency", "cost"]
+};
+
+// src/packs/reasoning.ts
+import { z as z3 } from "zod";
+var reasoningPack = {
+  name: "reasoning",
+  label: "Reasoning",
+  description: "Logic, math, and multi-step thinking \u2014 arithmetic, deduction, data interpretation, critical path, and business rules",
+  tasks: [
+    {
+      name: "rs:saas-mrr-calc",
+      prompt: `A SaaS company charges $49/month for the basic plan and $149/month for pro.
+In Q1 they had 200 basic subscribers and 85 pro subscribers.
+In Q2, 15% of basic users upgraded to pro and they gained 40 new basic subscribers.
+No one churned. What is the Q2 monthly recurring revenue (MRR)?
+Return as JSON with your reasoning and the final MRR number.`,
+      expected: { mrr: 27425 },
+      schema: z3.object({
+        reasoning: z3.string().optional(),
+        mrr: z3.number()
+      })
+    },
+    {
+      name: "rs:logical-deduction",
+      prompt: `Five developers \u2014 Alice, Bob, Carol, Dave, and Eve \u2014 each use a different
+primary language: Rust, TypeScript, Python, Go, and Java. Given:
+1. Alice does not use Python, Java, or Go.
+2. Bob uses TypeScript.
+3. Carol uses neither Rust nor Go.
+4. Dave does not use Java.
+5. Eve uses neither Rust, Go, nor Java.
+What language does each developer use? Return as JSON.`,
+      expected: {
+        Alice: "Rust",
+        Bob: "TypeScript",
+        Carol: "Java",
+        Dave: "Go",
+        Eve: "Python"
+      },
+      schema: z3.object({
+        Alice: z3.string(),
+        Bob: z3.string(),
+        Carol: z3.string(),
+        Dave: z3.string(),
+        Eve: z3.string()
+      })
+    },
+    {
+      name: "rs:data-interpretation",
+      prompt: `Given this quarterly revenue data:
+| Quarter | Revenue | Growth |
+|---------|---------|--------|
+| Q1 2025 | $2.1M   | -      |
+| Q2 2025 | $2.4M   | 14.3%  |
+| Q3 2025 | $2.2M   | -8.3%  |
+| Q4 2025 | $2.8M   | 27.3%  |
+
+Which quarter had the highest absolute revenue increase compared to the previous
+quarter? What was the full-year total revenue in millions? Return as JSON.`,
+      expected: {
+        highestGrowthQuarter: "Q4 2025",
+        absoluteIncrease: 0.6,
+        fullYearRevenue: 9.5
+      },
+      schema: z3.object({
+        highestGrowthQuarter: z3.string(),
+        absoluteIncrease: z3.number(),
+        fullYearRevenue: z3.number()
+      })
+    },
+    {
+      name: "rs:critical-path",
+      prompt: `A deployment pipeline has these stages with dependencies:
+- Build (3 min, no dependency)
+- Unit tests (5 min, depends on Build)
+- Integration tests (8 min, depends on Build)
+- Security scan (4 min, depends on Build)
+- Staging deploy (2 min, depends on Unit tests AND Integration tests AND Security scan)
+- Smoke tests (3 min, depends on Staging deploy)
+
+Assuming stages run in parallel where possible, what is the total pipeline
+duration in minutes? Which stages are on the critical path? Return as JSON.`,
+      expected: {
+        totalMinutes: 16,
+        criticalPath: ["Build", "Integration tests", "Staging deploy", "Smoke tests"]
+      },
+      schema: z3.object({
+        totalMinutes: z3.number(),
+        criticalPath: z3.array(z3.string())
+      })
+    },
+    {
+      name: "rs:pricing-rules",
+      prompt: `Apply these pricing rules to each customer and return the final price:
+Rules:
+- Base price: $100
+- Enterprise customers (>100 seats): 30% discount
+- Annual billing: additional 15% off the discounted price
+- Non-profit organizations: flat $50 regardless of other rules
+
+Customers:
+A: 50 seats, monthly billing, for-profit
+B: 200 seats, annual billing, for-profit
+C: 75 seats, annual billing, non-profit
+D: 150 seats, monthly billing, for-profit
+
+Return as a JSON array with customer id and finalPrice.`,
+      expected: [
+        { id: "A", finalPrice: 100 },
+        { id: "B", finalPrice: 59.5 },
+        { id: "C", finalPrice: 50 },
+        { id: "D", finalPrice: 70 }
+      ],
+      schema: z3.array(z3.object({
+        id: z3.string(),
+        finalPrice: z3.number()
+      }))
+    }
+  ],
+  scorers: ["correctness", "latency", "cost"]
+};
+
 // src/packs/index.ts
 var registry = /* @__PURE__ */ new Map();
 function register(pack) {
   registry.set(pack.name, pack);
 }
 register(structuredOutputPack);
+register(toolCallingPack);
+register(reasoningPack);
 function loadPack(name) {
   const pack = registry.get(name);
   if (!pack) {