agent-duelist 0.3.0 → 0.3.1

package/dist/index.cjs

@@ -1398,6 +1398,27 @@ var costScorer = ({ result }, providerId) => {
   };
 };
 
+// src/utils/deep-equal.ts
+function deepEqual(expected, actual) {
+  if (expected === actual) return true;
+  if (typeof expected === "string" && typeof actual === "string") {
+    return expected.trim().toLowerCase() === actual.trim().toLowerCase();
+  }
+  if (typeof expected !== typeof actual) return false;
+  if (expected === null || actual === null) return expected === actual;
+  if (Array.isArray(expected) && Array.isArray(actual)) {
+    if (expected.length !== actual.length) return false;
+    return expected.every((val, i) => deepEqual(val, actual[i]));
+  }
+  if (typeof expected === "object" && typeof actual === "object") {
+    const objExpected = expected;
+    const objActual = actual;
+    const keysExpected = Object.keys(objExpected);
+    return keysExpected.every((key) => key in objActual && deepEqual(objExpected[key], objActual[key]));
+  }
+  return expected === actual;
+}
+
 // src/scorers/correctness.ts
 var correctnessScorer = ({ task, result }) => {
   if (task.expected === void 0) {
@@ -1421,25 +1442,6 @@ function normalizeOutput(expected, actual) {
   }
   return actual;
 }
-function deepEqual(expected, actual) {
-  if (expected === actual) return true;
-  if (typeof expected === "string" && typeof actual === "string") {
-    return expected.trim().toLowerCase() === actual.trim().toLowerCase();
-  }
-  if (typeof expected !== typeof actual) return false;
-  if (expected === null || actual === null) return expected === actual;
-  if (Array.isArray(expected) && Array.isArray(actual)) {
-    if (expected.length !== actual.length) return false;
-    return expected.every((val, i) => deepEqual(val, actual[i]));
-  }
-  if (typeof expected === "object" && typeof actual === "object") {
-    const objExpected = expected;
-    const objActual = actual;
-    const keysExpected = Object.keys(objExpected);
-    return keysExpected.every((key) => key in objActual && deepEqual(objExpected[key], objActual[key]));
-  }
-  return expected === actual;
-}
 
 // src/scorers/schema-correctness.ts
 var schemaCorrectnessScorer = ({ task, result }) => {
@@ -1812,15 +1814,54 @@ function parseJudgeResponse(response, model) {
 
 // src/scorers/tool-usage.ts
 var toolUsageScorer = ({ task, result }) => {
-  const expectedToolName = task.tools?.[0]?.name;
-  if (!expectedToolName) {
+  if (!task.tools?.length) {
     return { name: "tool-usage", value: -1, details: { reason: "no tools configured on task" } };
   }
-  const usedTool = result.toolCalls?.some((c) => c.name === expectedToolName) ?? false;
+  const calls = result.toolCalls ?? [];
+  const expectedIsObject = task.expected !== void 0 && typeof task.expected === "object" && task.expected !== null && !Array.isArray(task.expected);
+  if (expectedIsObject) {
+    const matchingCall = calls.find((c) => {
+      const toolDef = task.tools.find((t) => t.name === c.name);
+      if (!toolDef) return false;
+      return deepEqual(task.expected, c.arguments);
+    });
+    if (matchingCall) {
+      return {
+        name: "tool-usage",
+        value: 1,
+        details: { matchedTool: matchingCall.name, arguments: matchingCall.arguments, toolCalls: calls }
+      };
+    }
+    const expectedKeys = Object.keys(task.expected);
+    const partialMatch = calls.find((c) => {
+      if (typeof c.arguments !== "object" || c.arguments === null) return false;
+      const argKeys = Object.keys(c.arguments);
+      return expectedKeys.some((k) => argKeys.includes(k));
+    });
+    if (partialMatch) {
+      return {
+        name: "tool-usage",
+        value: 0.5,
+        details: {
+          reason: "correct tool but wrong arguments",
+          expected: task.expected,
+          actual: partialMatch.arguments,
+          toolCalls: calls
+        }
+      };
+    }
+    return {
+      name: "tool-usage",
+      value: 0,
+      details: { reason: "no matching tool call", expected: task.expected, toolCalls: calls }
+    };
+  }
+  const expectedToolName = task.tools[0].name;
+  const usedTool = calls.some((c) => c.name === expectedToolName);
   return {
     name: "tool-usage",
     value: usedTool ? 1 : 0,
-    details: { expectedToolName, usedTool, toolCalls: result.toolCalls ?? [] }
+    details: { expectedToolName, usedTool, toolCalls: calls }
   };
 };
 
@@ -3487,12 +3528,247 @@ Return as JSON. Use ISO 8601 date format (YYYY-MM-DD).`,
   scorers: ["correctness", "schema-correctness", "latency", "cost"]
 };
 
+// src/packs/tool-calling.ts
+var import_zod2 = require("zod");
+var toolCallingPack = {
+  name: "tool-calling",
+  label: "Tool Calling",
+  description: "Function invocation accuracy \u2014 single calls, complex params, tool selection, parallel calls, and relevance detection",
+  tasks: [
+    {
+      name: "tc:simple-single-tool",
+      prompt: "What's the current weather in Tokyo?",
+      tools: [{
+        name: "getWeather",
+        description: "Get current weather for a city",
+        parameters: import_zod2.z.object({
+          city: import_zod2.z.string(),
+          units: import_zod2.z.enum(["celsius", "fahrenheit"]).optional()
+        }),
+        handler: async ({ city, units }) => ({
+          city,
+          tempC: 8,
+          condition: "cloudy",
+          units: units ?? "celsius"
+        })
+      }],
+      expected: { city: "Tokyo" }
+    },
+    {
+      name: "tc:complex-params",
+      prompt: "Search for Italian restaurants within 2 miles of downtown Portland that are open now and have at least a 4-star rating.",
+      tools: [{
+        name: "searchRestaurants",
+        description: "Search for restaurants matching criteria",
+        parameters: import_zod2.z.object({
+          cuisine: import_zod2.z.string(),
+          location: import_zod2.z.string(),
+          radiusMiles: import_zod2.z.number(),
+          minRating: import_zod2.z.number(),
+          openNow: import_zod2.z.boolean()
+        }),
+        handler: async (_args) => ({
+          results: [{ name: "Trattoria Roma", rating: 4.5, distance: 1.2 }]
+        })
+      }],
+      expected: {
+        cuisine: "Italian",
+        location: "downtown Portland",
+        radiusMiles: 2,
+        minRating: 4,
+        openNow: true
+      }
+    },
+    {
+      name: "tc:select-from-many",
+      prompt: "Convert 150 USD to Euros.",
+      tools: [
+        {
+          name: "getWeather",
+          description: "Get current weather for a city",
+          parameters: import_zod2.z.object({ city: import_zod2.z.string() }),
+          handler: async () => ({ tempC: 20 })
+        },
+        {
+          name: "convertCurrency",
+          description: "Convert an amount between currencies",
+          parameters: import_zod2.z.object({
+            amount: import_zod2.z.number(),
+            from: import_zod2.z.string(),
+            to: import_zod2.z.string()
+          }),
+          handler: async ({ amount, from, to }) => ({
+            amount,
+            from,
+            to,
+            result: 138.75,
+            rate: 0.925
+          })
+        },
+        {
+          name: "translateText",
+          description: "Translate text between languages",
+          parameters: import_zod2.z.object({ text: import_zod2.z.string(), targetLang: import_zod2.z.string() }),
+          handler: async () => ({ translated: "" })
+        },
+        {
+          name: "calculateTip",
+          description: "Calculate tip amount for a bill",
+          parameters: import_zod2.z.object({ billAmount: import_zod2.z.number(), tipPercent: import_zod2.z.number() }),
+          handler: async () => ({ tip: 0 })
+        }
+      ],
+      expected: { amount: 150, from: "USD", to: "EUR" }
+    },
+    {
+      name: "tc:parallel-calls",
+      prompt: "I'm planning a trip. What's the weather like in both Paris and London right now?",
+      tools: [{
+        name: "getWeather",
+        description: "Get current weather for a city",
+        parameters: import_zod2.z.object({ city: import_zod2.z.string() }),
+        handler: async ({ city }) => {
+          const data = {
+            Paris: { tempC: 12, condition: "partly cloudy" },
+            London: { tempC: 9, condition: "rainy" }
+          };
+          return data[city] ?? { tempC: 15, condition: "unknown" };
+        }
+      }],
+      expected: "weather data for Paris and London"
+    }
+  ],
+  scorers: ["tool-usage", "latency", "cost"]
+};
+
+// src/packs/reasoning.ts
+var import_zod3 = require("zod");
+var reasoningPack = {
+  name: "reasoning",
+  label: "Reasoning",
+  description: "Logic, math, and multi-step thinking \u2014 arithmetic, deduction, data interpretation, critical path, and business rules",
+  tasks: [
+    {
+      name: "rs:saas-mrr-calc",
+      prompt: `A SaaS company charges $49/month for the basic plan and $149/month for pro.
+In Q1 they had 200 basic subscribers and 85 pro subscribers.
+In Q2, 15% of basic users upgraded to pro and they gained 40 new basic subscribers.
+No one churned. What is the Q2 monthly recurring revenue (MRR)?
+Return as JSON with your reasoning and the final MRR number.`,
+      expected: { mrr: 27425 },
+      schema: import_zod3.z.object({
+        reasoning: import_zod3.z.string().optional(),
+        mrr: import_zod3.z.number()
+      })
+    },
+    {
+      name: "rs:logical-deduction",
+      prompt: `Five developers \u2014 Alice, Bob, Carol, Dave, and Eve \u2014 each use a different
+primary language: Rust, TypeScript, Python, Go, and Java. Given:
+1. Alice does not use Python, Java, or Go.
+2. Bob uses TypeScript.
+3. Carol uses neither Rust nor Go.
+4. Dave does not use Java.
+5. Eve uses neither Rust, Go, nor Java.
+What language does each developer use? Return as JSON.`,
+      expected: {
+        Alice: "Rust",
+        Bob: "TypeScript",
+        Carol: "Java",
+        Dave: "Go",
+        Eve: "Python"
+      },
+      schema: import_zod3.z.object({
+        Alice: import_zod3.z.string(),
+        Bob: import_zod3.z.string(),
+        Carol: import_zod3.z.string(),
+        Dave: import_zod3.z.string(),
+        Eve: import_zod3.z.string()
+      })
+    },
+    {
+      name: "rs:data-interpretation",
+      prompt: `Given this quarterly revenue data:
+| Quarter | Revenue | Growth |
+|---------|---------|--------|
+| Q1 2025 | $2.1M   | -      |
+| Q2 2025 | $2.4M   | 14.3%  |
+| Q3 2025 | $2.2M   | -8.3%  |
+| Q4 2025 | $2.8M   | 27.3%  |
+
+Which quarter had the highest absolute revenue increase compared to the previous
+quarter? What was the full-year total revenue in millions? Return as JSON.`,
+      expected: {
+        highestGrowthQuarter: "Q4 2025",
+        absoluteIncrease: 0.6,
+        fullYearRevenue: 9.5
+      },
+      schema: import_zod3.z.object({
+        highestGrowthQuarter: import_zod3.z.string(),
+        absoluteIncrease: import_zod3.z.number(),
+        fullYearRevenue: import_zod3.z.number()
+      })
+    },
+    {
+      name: "rs:critical-path",
+      prompt: `A deployment pipeline has these stages with dependencies:
+- Build (3 min, no dependency)
+- Unit tests (5 min, depends on Build)
+- Integration tests (8 min, depends on Build)
+- Security scan (4 min, depends on Build)
+- Staging deploy (2 min, depends on Unit tests AND Integration tests AND Security scan)
+- Smoke tests (3 min, depends on Staging deploy)
+
+Assuming stages run in parallel where possible, what is the total pipeline
+duration in minutes? Which stages are on the critical path? Return as JSON.`,
+      expected: {
+        totalMinutes: 16,
+        criticalPath: ["Build", "Integration tests", "Staging deploy", "Smoke tests"]
+      },
+      schema: import_zod3.z.object({
+        totalMinutes: import_zod3.z.number(),
+        criticalPath: import_zod3.z.array(import_zod3.z.string())
+      })
+    },
+    {
+      name: "rs:pricing-rules",
+      prompt: `Apply these pricing rules to each customer and return the final price:
+Rules:
+- Base price: $100
+- Enterprise customers (>100 seats): 30% discount
+- Annual billing: additional 15% off the discounted price
+- Non-profit organizations: flat $50 regardless of other rules
+
+Customers:
+A: 50 seats, monthly billing, for-profit
+B: 200 seats, annual billing, for-profit
+C: 75 seats, annual billing, non-profit
+D: 150 seats, monthly billing, for-profit
+
+Return as a JSON array with customer id and finalPrice.`,
+      expected: [
+        { id: "A", finalPrice: 100 },
+        { id: "B", finalPrice: 59.5 },
+        { id: "C", finalPrice: 50 },
+        { id: "D", finalPrice: 70 }
+      ],
+      schema: import_zod3.z.array(import_zod3.z.object({
+        id: import_zod3.z.string(),
+        finalPrice: import_zod3.z.number()
+      }))
+    }
+  ],
+  scorers: ["correctness", "latency", "cost"]
+};
+
 // src/packs/index.ts
 var registry = /* @__PURE__ */ new Map();
 function register(pack) {
   registry.set(pack.name, pack);
 }
 register(structuredOutputPack);
+register(toolCallingPack);
+register(reasoningPack);
 function loadPack(name) {
   const pack = registry.get(name);
   if (!pack) {