@skilljack/evals 1.0.0

package/dist/src/runner/skill-setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill-setup.js","sourceRoot":"","sources":["../../../src/runner/skill-setup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,eAAuB,EACvB,GAAW;IAEX,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IACtD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3E,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC3D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAEtD,MAAM,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;gBACzC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBACrC,kCAAkC;gBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;gBACjD,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACrE,MAAM,EAAE,CAAC,QAAQ,CACf,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,IAAI,CAAC,EACtC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAC5C,CAAC;gBACF,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACb,qCAAqC,eAAe,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC5G,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,GAAW;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,OAAO,CAAC,GAAW,EAAE,IAAY;IAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE/D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/src/scorer/deterministic.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Deterministic scorer for skill evaluations.
+ *
+ * Performs fast, free checks based on tool call analysis and output markers.
+ * No LLM calls required — checks are purely based on the session data.
+ */
+import type { EvalTask, TaskResult, DeterministicResult } from '../types.js';
+/**
+ * Run deterministic checks on a task result.
+ */
+export declare function scoreDeterministic(task: EvalTask, result: TaskResult): DeterministicResult | null;
+//# sourceMappingURL=deterministic.d.ts.map

package/dist/src/scorer/deterministic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../../src/scorer/deterministic.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,QAAQ,EACR,UAAU,EACV,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAqBrB;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,GACjB,mBAAmB,GAAG,IAAI,CAuH5B"}

package/dist/src/scorer/deterministic.js

@@ -0,0 +1,149 @@
+/**
+ * Deterministic scorer for skill evaluations.
+ *
+ * Performs fast, free checks based on tool call analysis and output markers.
+ * No LLM calls required — checks are purely based on the session data.
+ */
+/**
+ * Check if a tool name is a skill activation tool.
+ */
+function isSkillTool(toolName) {
+    // Local mode uses "Skill", MCP mode uses mcp__*__skill
+    return toolName === 'Skill' ||
+        (toolName.includes('skill') && !toolName.includes('skill-resource'));
+}
+/**
+ * Extract skill name from a tool call input.
+ */
+function extractSkillName(input) {
+    if (typeof input !== 'object' || input === null)
+        return undefined;
+    const obj = input;
+    // Local Skill tool uses 'skill', MCP uses 'name'
+    return obj.skill || obj.skill_name || obj.name || undefined;
+}
+/**
+ * Run deterministic checks on a task result.
+ */
+export function scoreDeterministic(task, result) {
+    const check = task.deterministic;
+    if (!check)
+        return null;
+    const details = [];
+    // 1. Check skill activation
+    let skillActivated = false;
+    let activatedSkillName;
+    if (result.isError) {
+        details.push('Task errored — treating as no activation');
+    }
+    else {
+        // Check tool calls for skill invocations
+        for (const call of result.toolCalls) {
+            if (isSkillTool(call.tool)) {
+                const name = extractSkillName(call.input);
+                if (name) {
+                    skillActivated = true;
+                    activatedSkillName = name;
+                    break;
+                }
+            }
+        }
+        // Also check skillLoads array (may be populated by runner)
+        if (!skillActivated && result.skillLoads.length > 0) {
+            skillActivated = true;
+            activatedSkillName = result.skillLoads[0];
+        }
+    }
+    // Verify activation matches expectations
+    if (check.expectSkillActivation) {
+        if (skillActivated) {
+            // Check if the correct skill was activated
+            if (task.expectedSkillLoad && task.expectedSkillLoad !== 'none') {
+                if (activatedSkillName === task.expectedSkillLoad) {
+                    details.push(`Skill activated correctly: ${activatedSkillName}`);
+                }
+                else {
+                    details.push(`Wrong skill activated: expected '${task.expectedSkillLoad}', got '${activatedSkillName}'`);
+                    skillActivated = false; // Wrong skill doesn't count
+                }
+            }
+            else {
+                details.push(`Skill activated: ${activatedSkillName}`);
+            }
+        }
+        else {
+            details.push(`Expected skill activation but no skill was loaded`);
+        }
+    }
+    else {
+        // Expect NO activation (false positive test)
+        if (skillActivated) {
+            details.push(`Unexpected skill activation: ${activatedSkillName} (false positive)`);
+        }
+        else {
+            details.push('Correctly did not activate any skill');
+        }
+    }
+    // 2. Check marker in output
+    let markerFound = null;
+    if (check.expectMarker) {
+        const output = result.output.toLowerCase();
+        const marker = check.expectMarker.toLowerCase();
+        markerFound = output.includes(marker);
+        details.push(markerFound
+            ? `Marker found: "${check.expectMarker}"`
+            : `Marker not found: "${check.expectMarker}"`);
+    }
+    // 3. Check expected tool calls
+    let expectedToolsCalled = null;
+    if (check.expectToolCalls && check.expectToolCalls.length > 0) {
+        const calledTools = new Set(result.toolCalls.map((c) => c.tool));
+        const missing = check.expectToolCalls.filter((t) => !calledTools.has(t));
+        expectedToolsCalled = missing.length === 0;
+        if (expectedToolsCalled) {
+            details.push(`All expected tools called: ${check.expectToolCalls.join(', ')}`);
+        }
+        else {
+            details.push(`Missing expected tool calls: ${missing.join(', ')}`);
+        }
+    }
+    // 4. Check forbidden tool calls
+    let unexpectedToolsCalled = null;
+    if (check.expectNoToolCalls && check.expectNoToolCalls.length > 0) {
+        const calledTools = new Set(result.toolCalls.map((c) => c.tool));
+        const forbidden = check.expectNoToolCalls.filter((t) => calledTools.has(t));
+        unexpectedToolsCalled = forbidden.length > 0;
+        if (unexpectedToolsCalled) {
+            details.push(`Forbidden tools were called: ${forbidden.join(', ')}`);
+        }
+        else {
+            details.push(`No forbidden tools called`);
+        }
+    }
+    // Compute overall pass/fail
+    let passed;
+    if (check.expectSkillActivation) {
+        // For positive tests: skill must be activated
+        passed = skillActivated;
+        if (markerFound !== null)
+            passed = passed && markerFound;
+        if (expectedToolsCalled !== null)
+            passed = passed && expectedToolsCalled;
+        if (unexpectedToolsCalled !== null)
+            passed = passed && !unexpectedToolsCalled;
+    }
+    else {
+        // For negative tests (false positive): skill must NOT be activated
+        passed = !skillActivated;
+    }
+    return {
+        skillActivated,
+        skillName: activatedSkillName,
+        markerFound,
+        expectedToolsCalled,
+        unexpectedToolsCalled,
+        passed,
+        details,
+    };
+}
+//# sourceMappingURL=deterministic.js.map

package/dist/src/scorer/deterministic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../../src/scorer/deterministic.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,uDAAuD;IACvD,OAAO,QAAQ,KAAK,OAAO;QACzB,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,iDAAiD;IACjD,OAAQ,GAAG,CAAC,KAAgB,IAAK,GAAG,CAAC,UAAqB,IAAK,GAAG,CAAC,IAAe,IAAI,SAAS,CAAC;AAClG,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAc,EACd,MAAkB;IAElB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;IACjC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,4BAA4B;IAC5B,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,kBAAsC,CAAC;IAE3C,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IAC3D,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACpC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC1C,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;oBACtB,kBAAkB,GAAG,IAAI,CAAC;oBAC1B,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,IAAI,CAAC,cAAc,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,cAAc,GAAG,IAAI,CAAC;YACtB,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC;QAChC,IAAI,cAAc,EAAE,CAAC;YACnB,2CAA2C;YAC3C,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB,KAAK,MAAM,EAAE,CAAC;gBAChE,IAAI,kBAAkB,KAAK,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBAClD,OAAO,CAAC,IAAI,CAAC,8BAA8B,kBAAkB,EAAE,CAAC,CAAC;gBACnE,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC,oCAAoC,IAAI,CAAC,iBAAiB,WAAW,kBAAkB,GAAG,CAAC,CAAC;oBACzG,cAAc,GAAG,KAAK,CAAC,CAAC,4BAA4B;gBACtD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,oBAAoB,kBAAkB,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,6CAA6C;QAC7C,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,gCAAgC,kBAAkB,mBAAmB,CAAC,CAAC;QACtF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,WAAW,GAAmB,IAAI,CAAC;IACvC,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAChD,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CACV,WAAW;YACT,CAAC,CAAC,kBAAkB,KAAK,CAAC,YAAY,GAAG;YACzC,CAAC,CAAC,sBAAsB,KAAK,CAAC,YAAY,GAAG,CAChD,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,IAAI,mBAAmB,GAAmB,IAAI,CAAC;IAC/C,IAAI,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9D,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,mBAAmB,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;QAC3C,IAAI,mBAAmB,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,gCAAgC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,IAAI,qBAAqB,GAAmB,IAAI,CAAC;IACjD,IAAI,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,qBAAqB,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7C,IAAI,qBAAqB,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,gCAAgC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,MAAe,CAAC;IACpB,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC;QAChC,8CAA8C;QAC9C,MAAM,GAAG,cAAc,CAAC;QACxB,IAAI,WAAW,KAAK,IAAI;YAAE,MAAM,GAAG,MAAM,IAAI,WAAW,CAAC;QACzD,IAAI,mBAAmB,KAAK,IAAI;YAAE,MAAM,GAAG,MAAM,IAAI,mBAAmB,CAAC;QACzE,IAAI,qBAAqB,KAAK,IAAI;YAAE,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;IAChF,CAAC;SAAM,CAAC;QACN,mEAAmE;QACnE,MAAM,GAAG,CAAC,cAAc,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,cAAc;QACd,SAAS,EAAE,kBAAkB;QAC7B,WAAW;QACX,mBAAmB;QACnB,qBAAqB;QACrB,MAAM;QACN,OAAO;KACR,CAAC;AACJ,CAAC"}

package/dist/src/scorer/judge.d.ts

@@ -0,0 +1,34 @@
+/**
+ * LLM-as-judge for scoring skill evaluation results.
+ *
+ * Uses Claude (via Agent SDK) to evaluate agent performance on three dimensions:
+ * - Discovery (0/1): Did agent load the expected skill?
+ * - Adherence (1-5): How well did agent follow skill instructions?
+ * - Output Quality (1-5): Does output meet task requirements?
+ */
+import type { EvalTask, TaskResult, JudgeScore, JudgeOptions } from '../types.js';
+/**
+ * LLM-as-judge for scoring skill evaluation results.
+ */
+export declare class SkillJudge {
+    private options;
+    constructor(options?: JudgeOptions);
+    /**
+     * Build the prompt for the judge.
+     */
+    private buildJudgePrompt;
+    /**
+     * Parse the judge's JSON response into a JudgeScore.
+     */
+    private parseJudgeResponse;
+    private createErrorScore;
+    /**
+     * Score a single evaluation result.
+     */
+    judgeResult(task: EvalTask, result: TaskResult): Promise<JudgeScore>;
+    /**
+     * Score all evaluation results.
+     */
+    judgeAll(tasks: EvalTask[], results: TaskResult[]): Promise<JudgeScore[]>;
+}
+//# sourceMappingURL=judge.d.ts.map

package/dist/src/scorer/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/scorer/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EACV,QAAQ,EACR,UAAU,EACV,UAAU,EACV,YAAY,EAEb,MAAM,aAAa,CAAC;AAsErB;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAyB;gBAE5B,OAAO,GAAE,YAAiB;IAStC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAyBxB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAuC1B,OAAO,CAAC,gBAAgB;IAYxB;;OAEG;IACG,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IA+D1E;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;CAShF"}

package/dist/src/scorer/judge.js

@@ -0,0 +1,226 @@
+/**
+ * LLM-as-judge for scoring skill evaluation results.
+ *
+ * Uses Claude (via Agent SDK) to evaluate agent performance on three dimensions:
+ * - Discovery (0/1): Did agent load the expected skill?
+ * - Adherence (1-5): How well did agent follow skill instructions?
+ * - Output Quality (1-5): Does output meet task requirements?
+ */
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { isAssistantMessage, isResultMessage, isTextBlock, } from '../types.js';
+import { loadConfigSync } from '../config.js';
+const JUDGE_PROMPT_TEMPLATE = `You are an expert evaluator for AI agent skills. Score this skill evaluation result.
+
+## Task Information
+**Prompt given to agent:** {prompt}
+
+**Expected skill to load:** {expectedSkill}
+
+**Criteria:**
+{criteriaText}
+
+**Golden checklist (expected behaviors):**
+{checklistText}
+
+## Agent Result
+**Skills that were loaded:** {skillLoads}
+
+**Agent output:**
+{output}
+
+## Scoring Instructions
+
+Score the agent's performance on three dimensions:
+
+1. **Discovery (0 or 1)**: Did the agent load the expected skill "{expectedSkill}"?
+   - Score 1 if the expected skill was loaded
+   - Score 0 if it was not loaded
+   - If expected skill is "none", score 1 if NO skill was loaded, 0 if a skill was incorrectly loaded
+
+2. **Adherence (1-5)**: How well did the agent follow the skill's instructions?
+   - 5 = Perfectly followed all instructions
+   - 4 = Followed most instructions with minor deviations
+   - 3 = Followed core instructions but missed some details
+   - 2 = Partially followed instructions with significant gaps
+   - 1 = Did not follow the skill's instructions
+
+3. **Output Quality (1-5)**: Does the output meet the task requirements?
+   - 5 = Excellent output, meets all requirements
+   - 4 = Good output with minor issues
+   - 3 = Acceptable output, meets basic requirements
+   - 2 = Poor output, missing key requirements
+   - 1 = Unacceptable output
+
+4. **Failure Category** (if score < 4 on any dimension):
+   - "discovery_failure": Agent didn't load the skill when it should have
+   - "false_positive": Agent loaded a skill when it should NOT have
+   - "instruction_ambiguity": Agent misinterpreted skill instructions
+   - "missing_guidance": Skill didn't cover a needed case
+   - "agent_error": Agent made a mistake despite clear guidance
+   - "none": No significant failure
+
+Respond with a JSON object:
+\`\`\`json
+{
+  "discovery": <0 or 1>,
+  "adherence": <1-5>,
+  "output_quality": <1-5>,
+  "failure_category": "<category or none>",
+  "reasoning": "<brief explanation of scores>"
+}
+\`\`\`
+`;
+/**
+ * LLM-as-judge for scoring skill evaluation results.
+ */
+export class SkillJudge {
+    options;
+    constructor(options = {}) {
+        const config = loadConfigSync();
+        this.options = {
+            model: options.model ?? config.defaultJudgeModel,
+            outputTruncation: options.outputTruncation ?? config.judgeOutputTruncation,
+        };
+    }
+    /**
+     * Build the prompt for the judge.
+     */
+    buildJudgePrompt(task, result) {
+        const criteriaLines = task.criteria.map((c) => `- **${capitalize(c.dimension)}** (weight ${c.weight}): ${c.description}`);
+        const criteriaText = criteriaLines.length > 0
+            ? criteriaLines.join('\n')
+            : '- No specific criteria defined';
+        const checklistText = task.goldenChecklist.length > 0
+            ? task.goldenChecklist.map((item) => `- ${item}`).join('\n')
+            : '- No checklist defined';
+        const skillLoads = result.skillLoads.length > 0
+            ? result.skillLoads.join(', ')
+            : 'None';
+        return JUDGE_PROMPT_TEMPLATE
+            .replace('{prompt}', task.prompt)
+            .replace(/{expectedSkill}/g, task.expectedSkillLoad)
+            .replace('{criteriaText}', criteriaText)
+            .replace('{checklistText}', checklistText)
+            .replace('{skillLoads}', skillLoads)
+            .replace('{output}', result.output.slice(0, this.options.outputTruncation) || '(no output)');
+    }
+    /**
+     * Parse the judge's JSON response into a JudgeScore.
+     */
+    parseJudgeResponse(response, taskId, weights) {
+        const jsonMatch = response.match(/\{[\s\S]*?\}/);
+        if (!jsonMatch) {
+            return this.createErrorScore(taskId, 'Failed to parse judge response');
+        }
+        try {
+            const data = JSON.parse(jsonMatch[0]);
+            const discovery = Number(data.discovery) || 0;
+            const adherence = Number(data.adherence) || 1;
+            const outputQuality = Number(data.output_quality) || 1;
+            const adherenceNorm = (adherence - 1) / 4;
+            const outputNorm = (outputQuality - 1) / 4;
+            const weightedScore = (weights.get('discovery') ?? 0.3) * discovery +
+                (weights.get('adherence') ?? 0.4) * adherenceNorm +
+                (weights.get('output') ?? 0.3) * outputNorm;
+            return {
+                taskId,
+                discovery,
+                adherence,
+                outputQuality,
+                weightedScore,
+                failureCategory: (data.failure_category || 'none'),
+                reasoning: data.reasoning || '',
+            };
+        }
+        catch {
+            return this.createErrorScore(taskId, 'Invalid JSON in judge response');
+        }
+    }
+    createErrorScore(taskId, reason) {
+        return {
+            taskId,
+            discovery: 0,
+            adherence: 1,
+            outputQuality: 1,
+            weightedScore: 0,
+            failureCategory: 'agent_error',
+            reasoning: reason,
+        };
+    }
+    /**
+     * Score a single evaluation result.
+     */
+    async judgeResult(task, result) {
+        if (result.isError) {
+            return {
+                taskId: task.id,
+                discovery: 0,
+                adherence: 1,
+                outputQuality: 1,
+                weightedScore: 0,
+                failureCategory: 'agent_error',
+                reasoning: `Task failed with error: ${result.errorMessage}`,
+            };
+        }
+        const weights = new Map();
+        for (const c of task.criteria) {
+            weights.set(c.dimension, c.weight);
+        }
+        const prompt = this.buildJudgePrompt(task, result);
+        try {
+            let responseText = '';
+            for await (const message of query({
+                prompt,
+                options: {
+                    model: this.options.model,
+                    allowedTools: [],
+                    permissionMode: 'bypassPermissions',
+                },
+            })) {
+                if (isAssistantMessage(message)) {
+                    const content = message.message.content;
+                    for (const block of content) {
+                        if (isTextBlock(block)) {
+                            responseText += block.text;
+                        }
+                    }
+                }
+                if (isResultMessage(message)) {
+                    if (message.result) {
+                        responseText = message.result;
+                    }
+                }
+            }
+            return this.parseJudgeResponse(responseText, task.id, weights);
+        }
+        catch (error) {
+            // Fallback: heuristic scoring
+            const discovery = result.skillLoads.includes(task.expectedSkillLoad) ? 1 : 0;
+            return {
+                taskId: task.id,
+                discovery,
+                adherence: 3,
+                outputQuality: 3,
+                weightedScore: 0.5,
+                failureCategory: discovery === 0 ? 'discovery_failure' : 'none',
+                reasoning: `Heuristic scoring (judge error: ${error instanceof Error ? error.message : 'unknown'})`,
+            };
+        }
+    }
+    /**
+     * Score all evaluation results.
+     */
+    async judgeAll(tasks, results) {
+        const scores = [];
+        for (let i = 0; i < tasks.length; i++) {
+            console.log(`Judging task ${tasks[i].id}...`);
+            const score = await this.judgeResult(tasks[i], results[i]);
+            scores.push(score);
+        }
+        return scores;
+    }
+}
+function capitalize(s) {
+    return s.charAt(0).toUpperCase() + s.slice(1);
+}
+//# sourceMappingURL=judge.js.map

package/dist/src/scorer/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../src/scorer/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAQvD,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,WAAW,GACZ,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,qBAAqB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4D7B,CAAC;AAEF;;GAEG;AACH,MAAM,OAAO,UAAU;IACb,OAAO,CAAyB;IAExC,YAAY,UAAwB,EAAE;QACpC,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;QAEhC,IAAI,CAAC,OAAO,GAAG;YACb,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,iBAAiB;YAChD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CAAC,qBAAqB;SAC3E,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,IAAc,EAAE,MAAkB;QACzD,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,WAAW,EAAE,CACjF,CAAC;QACF,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC;YAC3C,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAC1B,CAAC,CAAC,gCAAgC,CAAC;QAErC,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;YACnD,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC5D,CAAC,CAAC,wBAAwB,CAAC;QAE7B,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC7C,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9B,CAAC,CAAC,MAAM,CAAC;QAEX,OAAO,qBAAqB;aACzB,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC;aAChC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,CAAC;aACnD,OAAO,CAAC,gBAAgB,EAAE,YAAY,CAAC;aACvC,OAAO,CAAC,iBAAiB,EAAE,aAAa,CAAC;aACzC,OAAO,CAAC,cAAc,EAAE,UAAU,CAAC;aACnC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,aAAa,CAAC,CAAC;IACjG,CAAC;IAED;;OAEG;IACK,kBAAkB,CACxB,QAAgB,EAChB,MAAc,EACd,OAA4B;QAE5B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,gCAAgC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YAEtC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEvD,MAAM,aAAa,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,UAAU,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YAE3C,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,SAAS;gBAC7C,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,aAAa;gBACjD,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC;YAE9C,OAAO;gBACL,MAAM;gBACN,SAAS;gBACT,SAAS;gBACT,aAAa;gBACb,aAAa;gBACb,eAAe,EAAE,CAAC,IAAI,CAAC,gBAAgB,IAAI,MAAM,CAAoB;gBACrE,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,EAAE;aAChC,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,gCAAgC,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,MAAc,EAAE,MAAc;QACrD,OAAO;YACL,MAAM;YACN,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,CAAC;YAChB,eAAe,EAAE,aAAa;YAC9B,SAAS,EAAE,MAAM;SAClB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc,EAAE,MAAkB;QAClD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,CAAC;gBAChB,eAAe,EAAE,aAAa;gBAC9B,SAAS,EAAE,2BAA2B,MAAM,CAAC,YAAY,EAAE;aAC5D,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAEnD,IAAI,CAAC;YACH,IAAI,YAAY,GAAG,EAAE,CAAC;YAEtB,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,KAAK,CAAC;gBAChC,MAAM;gBACN,OAAO,EAAE;oBACP,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;oBACzB,YAAY,EAAE,EAAE;oBAChB,cAAc,EAAE,mBAAmB;iBACpC;aACF,CAAC,EAAE,CAAC;gBACH,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;oBACxC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;wBAC5B,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;4BACvB,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC;wBAC7B,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnB,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;oBAChC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8BAA8B;YAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7E,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,SAAS;gBACT,SAAS,EAAE,CAAC;gBACZ,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,GAAG;gBAClB,eAAe,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM;gBAC/D,SAAS,EAAE,mCAAmC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,GAAG;aACpG,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAiB,EAAE,OAAqB;QACrD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC"}

package/dist/src/scorer/scorer.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Scoring orchestrator that combines deterministic and LLM-as-judge scoring.
+ *
+ * Deterministic scoring runs first (free, fast), then LLM judge if configured.
+ * Results are merged with deterministic taking precedence for discovery.
+ */
+import type { EvalTask, TaskResult, CombinedScore } from '../types.js';
+import type { JudgeOptions } from '../types.js';
+export interface ScorerOptions {
+    /** Skip deterministic scoring */
+    noDeterministic?: boolean;
+    /** Skip LLM judge scoring */
+    noJudge?: boolean;
+    /** Judge options */
+    judgeOptions?: JudgeOptions;
+}
+/**
+ * Score a single task result using both deterministic and LLM judge methods.
+ */
+export declare function scoreTask(task: EvalTask, result: TaskResult, options?: ScorerOptions): Promise<CombinedScore>;
+/**
+ * Score all task results.
+ */
+export declare function scoreAll(tasks: EvalTask[], results: TaskResult[], options?: ScorerOptions): Promise<CombinedScore[]>;
+//# sourceMappingURL=scorer.d.ts.map

package/dist/src/scorer/scorer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../../src/scorer/scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,QAAQ,EACR,UAAU,EACV,aAAa,EAId,MAAM,aAAa,CAAC;AAGrB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAGhD,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,oBAAoB;IACpB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,QAAQ,EACd,MAAM,EAAE,UAAU,EAClB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAmBxB;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,EAAE,UAAU,EAAE,EACrB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,aAAa,EAAE,CAAC,CAY1B"}

package/dist/src/scorer/scorer.js

@@ -0,0 +1,149 @@
+/**
+ * Scoring orchestrator that combines deterministic and LLM-as-judge scoring.
+ *
+ * Deterministic scoring runs first (free, fast), then LLM judge if configured.
+ * Results are merged with deterministic taking precedence for discovery.
+ */
+import { scoreDeterministic } from './deterministic.js';
+import { SkillJudge } from './judge.js';
+import { loadConfigSync, getDefaultWeights } from '../config.js';
+/**
+ * Score a single task result using both deterministic and LLM judge methods.
+ */
+export async function scoreTask(task, result, options = {}) {
+    const config = loadConfigSync();
+    const weights = getDefaultWeights(config);
+    // Run deterministic scoring
+    let deterministicResult = null;
+    if (!options.noDeterministic && task.deterministic) {
+        deterministicResult = scoreDeterministic(task, result);
+    }
+    // Run LLM judge scoring
+    let judgeResult = null;
+    if (!options.noJudge && task.criteria.length > 0) {
+        const judge = new SkillJudge(options.judgeOptions);
+        judgeResult = await judge.judgeResult(task, result);
+    }
+    const isNegativeTest = task.expectedSkillLoad === 'none';
+    return mergeScores(task.id, deterministicResult, judgeResult, weights, isNegativeTest);
+}
+/**
+ * Score all task results.
+ */
+export async function scoreAll(tasks, results, options = {}) {
+    const scores = [];
+    for (let i = 0; i < tasks.length; i++) {
+        const task = tasks[i];
+        const result = results[i];
+        console.log(`Scoring task ${task.id}...`);
+        const score = await scoreTask(task, result, options);
+        scores.push(score);
+    }
+    return scores;
+}
+/**
+ * Merge deterministic and judge scores into a combined score.
+ *
+ * Merge rules:
+ * - Discovery: deterministic is authoritative (checks actual tool calls)
+ * - Adherence/output: from judge; if no judge, map deterministic pass→5, fail→1
+ * - Failure category: determined from available evidence
+ */
+function mergeScores(taskId, det, judge, weights, isNegativeTest = false) {
+    // For negative tests (expectedSkillLoad === 'none'):
+    // discovery = 1 means correctly did NOT activate (good)
+    // discovery = 0 means incorrectly activated (false positive)
+    const computeDiscovery = (activated) => isNegativeTest ? (activated ? 0 : 1) : (activated ? 1 : 0);
+    // Case 1: Both available — merge
+    if (det && judge) {
+        const discovery = computeDiscovery(det.skillActivated);
+        const adherence = judge.adherence;
+        const outputQuality = judge.outputQuality;
+        const adherenceNorm = (adherence - 1) / 4;
+        const outputNorm = (outputQuality - 1) / 4;
+        const weightedScore = (weights.get('discovery') ?? 0.3) * discovery +
+            (weights.get('adherence') ?? 0.4) * adherenceNorm +
+            (weights.get('output') ?? 0.3) * outputNorm;
+        // Determine failure category
+        let failureCategory = judge.failureCategory;
+        if (!det.passed && det.skillActivated === false) {
+            failureCategory = 'discovery_failure';
+        }
+        // Check for false positive via deterministic
+        if (det.skillActivated && det.details.some((d) => d.includes('false positive'))) {
+            failureCategory = 'false_positive';
+        }
+        const reasons = [];
+        if (det.details.length > 0)
+            reasons.push(`Deterministic: ${det.details.join('; ')}`);
+        if (judge.reasoning)
+            reasons.push(`Judge: ${judge.reasoning}`);
+        return {
+            taskId,
+            deterministic: det,
+            judge,
+            discovery,
+            adherence,
+            outputQuality,
+            weightedScore,
+            failureCategory,
+            reasoning: reasons.join(' | '),
+        };
+    }
+    // Case 2: Deterministic only
+    if (det) {
+        const discovery = computeDiscovery(det.skillActivated);
+        const adherence = det.passed ? 5 : 1;
+        const outputQuality = det.passed ? 5 : 1;
+        const adherenceNorm = (adherence - 1) / 4;
+        const outputNorm = (outputQuality - 1) / 4;
+        const weightedScore = (weights.get('discovery') ?? 0.3) * discovery +
+            (weights.get('adherence') ?? 0.4) * adherenceNorm +
+            (weights.get('output') ?? 0.3) * outputNorm;
+        let failureCategory = 'none';
+        if (!det.skillActivated && det.details.some((d) => d.includes('Expected skill activation'))) {
+            failureCategory = 'discovery_failure';
+        }
+        if (det.details.some((d) => d.includes('false positive'))) {
+            failureCategory = 'false_positive';
+        }
+        return {
+            taskId,
+            deterministic: det,
+            judge: null,
+            discovery,
+            adherence,
+            outputQuality,
+            weightedScore,
+            failureCategory,
+            reasoning: `Deterministic only: ${det.details.join('; ')}`,
+        };
+    }
+    // Case 3: Judge only
+    if (judge) {
+        return {
+            taskId,
+            deterministic: null,
+            judge,
+            discovery: judge.discovery,
+            adherence: judge.adherence,
+            outputQuality: judge.outputQuality,
+            weightedScore: judge.weightedScore,
+            failureCategory: judge.failureCategory,
+            reasoning: judge.reasoning,
+        };
+    }
+    // Case 4: No scoring available
+    return {
+        taskId,
+        deterministic: null,
+        judge: null,
+        discovery: 0,
+        adherence: 1,
+        outputQuality: 1,
+        weightedScore: 0,
+        failureCategory: 'agent_error',
+        reasoning: 'No scoring method available (no deterministic check or LLM judge criteria defined)',
+    };
+}
+//# sourceMappingURL=scorer.js.map

package/dist/src/scorer/scorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../../../src/scorer/scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAWjE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,MAAkB,EAClB,UAAyB,EAAE;IAE3B,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE1C,4BAA4B;IAC5B,IAAI,mBAAmB,GAA+B,IAAI,CAAC;IAC3D,IAAI,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACnD,mBAAmB,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,wBAAwB;IACxB,IAAI,WAAW,GAAsB,IAAI,CAAC;IAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACnD,WAAW,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,KAAK,MAAM,CAAC;IACzD,OAAO,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,mBAAmB,EAAE,WAAW,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC;AACzF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAiB,EACjB,OAAqB,EACrB,UAAyB,EAAE;IAE3B,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAClB,MAAc,EACd,GAA+B,EAC/B,KAAwB,EACxB,OAA4B,EAC5B,cAAc,GAAG,KAAK;IAEtB,qDAAqD;IACrD,wDAAwD;IACxD,6DAA6D;IAC7D,MAAM,gBAAgB,GAAG,CAAC,SAAkB,EAAE,EAAE,CAC9C,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,iCAAiC;IACjC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QAClC,MAAM,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;QAE1C,MAAM,aAAa,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,SAAS;YAC7C,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,aAAa;YACjD,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC;QAE9C,6BAA6B;QAC7B,IAAI,eAAe,GAAG,KAAK,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;YAChD,eAAe,GAAG,mBAAmB,CAAC;QACxC,CAAC;QACD,6CAA6C;QAC7C,IAAI,GAAG,CAAC,cAAc,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;YAChF,eAAe,GAAG,gBAAgB,CAAC;QACrC,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrF,IAAI,KAAK,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAE/D,OAAO;YACL,MAAM;YACN,aAAa,EAAE,GAAG;YAClB,KAAK;YACL,SAAS;YACT,SAAS;YACT,aAAa;YACb,aAAa;YACb,eAAe;YACf,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;SAC/B,CAAC;IACJ,CAAC;IAED,6BAA6B;IAC7B,IAAI,GAAG,EAAE,CAAC;QACR,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzC,MAAM,aAAa,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,SAAS;YAC7C,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,aAAa;YACjD,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC;QAE9C,IAAI,eAAe,GAAoB,MAAM,CAAC;QAC9C,IAAI,CAAC,GAAG,CAAC,cAAc,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,2BAA2B,CAAC,CAAC,EAAE,CAAC;YAC5F,eAAe,GAAG,mBAAmB,CAAC;QACxC,CAAC;QACD,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC;YAC1D,eAAe,GAAG,gBAAgB,CAAC;QACrC,CAAC;QAED,OAAO;YACL,MAAM;YACN,aAAa,EAAE,GAAG;YAClB,KAAK,EAAE,IAAI;YACX,SAAS;YACT,SAAS;YACT,aAAa;YACb,aAAa;YACb,eAAe;YACf,SAAS,EAAE,uBAAuB,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC3D,CAAC;IACJ,CAAC;IAED,qBAAqB;IACrB,IAAI,KAAK,EAAE,CAAC;QACV,OAAO;YACL,MAAM;YACN,aAAa,EAAE,IAAI;YACnB,KAAK;YACL,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,aAAa,EAAE,KAAK,CAAC,aAAa;YAClC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,OAAO;QACL,MAAM;QACN,aAAa,EAAE,IAAI;QACnB,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,aAAa;QAC9B,SAAS,EAAE,oFAAoF;KAChG,CAAC;AACJ,CAAC"}