@netlify/axis 0.2.0

package/dist/scoring/deep-eval.js

@@ -0,0 +1,382 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { getAdapter } from "../adapters/registry.js";
+import { DEFAULT_AUDIT_SCORES } from "./category-score.js";
+import { parseJsonFromText } from "./parse-json.js";
+/** Max characters of full content to include per interaction. */
+const MAX_CONTENT_PER_INTERACTION = 3_000;
+/** Max total content characters to send to the judge. */
+const MAX_TOTAL_CONTENT = 40_000;
+/** Max characters for the sparse index in the evaluation prompt. */
+const MAX_SPARSE_INDEX_CHARS = 60_000;
+/**
+ * Run the deep evaluation LLM pass.
+ *
+ * Speed is always computed heuristically from interaction timing data (no LLM needed).
+ * The LLM evaluates ALL interactions for success, weight, contextRelevance,
+ * and necessity per category.
+ */
+export async function runDeepEval(result, sparseIndex, triage, normalized) {
+    // If there are no interactions at all, return defaults
+    if (sparseIndex.interactions.length === 0) {
+        return buildDefaultResult(sparseIndex);
+    }
+    // Always call LLM to evaluate all interactions
+    const prompt = buildDeepEvalPrompt(result, sparseIndex, triage, normalized);
+    const responseText = await callJudge(result, prompt);
+    const deepResult = parseDeepEvalResponse(responseText, sparseIndex);
+    // Inject heuristic speed into ALL audits — speed is always deterministic
+    for (const audit of deepResult.audits) {
+        const interaction = sparseIndex.interactions.find((i) => i.id === audit.id);
+        if (interaction) {
+            audit.speed = computeHeuristicSpeed(interaction);
+        }
+    }
+    return deepResult;
+}
+/**
+ * Compute a heuristic speed score (0-1) for an interaction based on
+ * duration and category. Deterministic — no LLM needed.
+ *
+ * Thresholds are generous to account for system overhead
+ * (SDK roundtrips, sandbox setup, process spawning).
+ */
+export function computeHeuristicSpeed(interaction) {
+    const { durationMs, categories } = interaction;
+    // No timing data — assume efficient
+    if (durationMs === null || durationMs <= 0)
+        return 1.0;
+    const seconds = durationMs / 1000;
+    // Service interactions (API calls, web fetches): network latency expected
+    if (categories.includes("service")) {
+        if (seconds <= 2)
+            return 1.0;
+        if (seconds <= 5)
+            return 0.9;
+        if (seconds <= 10)
+            return 0.8;
+        if (seconds <= 25)
+            return 0.6;
+        return 0.4;
+    }
+    // Environment interactions (file ops, shell commands): local, should be near-instant
+    if (categories.includes("environment")) {
+        if (seconds <= 0.5)
+            return 1.0;
+        if (seconds <= 2)
+            return 0.9;
+        if (seconds <= 5)
+            return 0.8;
+        if (seconds <= 10)
+            return 0.6;
+        return 0.4;
+    }
+    // Agent thinking: reasoning latency
+    if (seconds <= 2)
+        return 1.0;
+    if (seconds <= 5)
+        return 0.9;
+    if (seconds <= 15)
+        return 0.8;
+    if (seconds <= 30)
+        return 0.6;
+    return 0.4;
+}
+function buildDeepEvalPrompt(result, sparseIndex, triage, normalized) {
+    const { stats } = sparseIndex;
+    // Always include the full sparse index
+    const sparseLines = truncateSparseLines(sparseIndex.lines);
+    // Include full content for ALL interactions (within budget)
+    const interactionContent = buildInteractionContent(sparseIndex, triage, normalized);
+    // Include triage context when available
+    let triageSection = "";
+    if (triage.patterns.length > 0 || Object.values(triage.categoryNotes).some((n) => n)) {
+        const patternsText = triage.patterns
+            .map((p) => `- ${p.description} (severity: ${p.severity}, interactions: #${p.interactionIds.join(", #")})`)
+            .join("\n");
+        const categories = ["environment", "service", "agent"];
+        const categoryNotesText = categories
+            .filter((c) => triage.categoryNotes[c])
+            .map((c) => `${c}: ${triage.categoryNotes[c]}`)
+            .join("\n");
+        triageSection = `
+TRIAGE ANALYSIS:
+${categoryNotesText || "(none)"}
+${patternsText ? `\nPATTERNS:\n${patternsText}` : ""}
+`;
+    }
+    return `You are an expert evaluator for AXIS, an AI agent testing framework.
+
+You are performing a comprehensive evaluation of ALL interactions from an agent execution.
+
+SCENARIO: ${result.scenarioName}
+
+TASK GIVEN TO AGENT:
+${result.prompt}
+${triageSection}
+COMPLETE SPARSE INDEX (${stats.totalInteractions} interactions):
+${sparseLines}
+
+STATS:
+- Environment interactions: ${stats.byCategory.environment}
+- Service interactions: ${stats.byCategory.service}
+- Agent interactions: ${stats.byCategory.agent}
+- Errors: ${stats.totalErrors}
+- Total duration: ${stats.totalDurationMs}ms
+
+FULL INTERACTION CONTENT:
+${interactionContent}
+
+NOTE: Content shown above may be truncated for evaluation purposes. This does NOT mean the agent's actual tool results were truncated — evaluate based on the quality and structure of what is shown, not on apparent truncation boundaries.
+
+EVALUATION DIMENSIONS (score each 0.0 to 1.0):
+- success: Did the interaction complete without errors? Were the results correct and usable? Evaluate based on the actual content returned, not assumptions about what a "complete" result should look like. For service calls (API requests, web fetches), if the call returned structured, usable content and the agent used it successfully, score success high — do not speculate about content that might be missing or hypothesize about JS-gated pages or truncation.
+- weight: Was the tool invocation right-sized for the operation? Evaluate whether the agent sent an appropriate amount of data to the tool and received a proportionate response. For environment tools (file writes, edits, shell commands), judge the tool operation — not the semantic quality of the content the agent chose to write. A 2KB file write is right-sized if the agent intended to write 2KB of content. For service calls, if the call returned the data the agent needed, it is right-sized — do not penalize because a page returned fewer bytes than expected. (1.0 = right-sized, 0.3 = bloated/wasteful)
+- contextRelevance: Was the tool's output relevant and usable for the task? If the tool succeeded and the agent used the output to make progress, score 1.0. Only reduce this score if the output was genuinely irrelevant noise that the agent could not use. Do NOT reduce this score for content quality judgments (e.g., whether a summary was condensed enough, whether fetched content was comprehensive enough) — those are evaluated by goal achievement, not here. Agent-internal operations (tool discovery, planning) are necessary infrastructure — score based on whether they were needed. (1.0 = all useful/necessary, 0.0 = all noise)
+
+For each CATEGORY present, also evaluate necessity:
+- necessity (0.0 to 1.0): Were the interactions that the agent performed in this category necessary? Evaluate only what the agent actually did — do not penalize for hypothetical steps it could have taken. 1.0 = all interactions were necessary, 0.0 = all were unnecessary.
+- List any interaction IDs that were unnecessary.
+
+CONTEXT FOR EVALUATION:
+- Tool discovery (e.g., ToolSearch, ListTools) and agent configuration reads are required infrastructure — do not flag as unnecessary unless genuinely redundant (same query repeated).
+- Byte counts in sparse lines show total I/O transferred, not file content size. Small results are normal for write/edit confirmations.
+- If a service call (API request, web fetch) returned structured, usable content and the agent used it to complete the task, do not flag it for concerns about hypothetical missing content or page size.
+
+Respond with ONLY valid JSON:
+{
+  "audits": [
+    {"id": 1, "category": "environment", "success": 0.9, "weight": 0.8, "contextRelevance": 0.6, "rationale": "brief explanation"},
+    ...
+  ],
+  "necessity": [
+    {"category": "environment", "score": 0.85, "unnecessaryIds": [4], "rationale": "brief explanation"},
+    {"category": "service", "score": 0.7, "unnecessaryIds": [5, 6], "rationale": "brief explanation"},
+    {"category": "agent", "score": 0.95, "unnecessaryIds": [], "rationale": "brief explanation"}
+  ]
+}
+
+Include an audit for EVERY interaction listed above.`;
+}
+/**
+ * Build the full content section for ALL interactions.
+ * Includes as much content as fits within the total budget.
+ */
+function buildInteractionContent(sparseIndex, triage, normalized) {
+    const sections = [];
+    let totalChars = 0;
+    const flagMap = new Map(triage.flaggedInteractions.map((f) => [f.id, f]));
+    for (let idx = 0; idx < sparseIndex.interactions.length; idx++) {
+        const interaction = sparseIndex.interactions[idx];
+        // Build full content from the normalized entries
+        const fullContent = interaction.entryIndices.map((i) => formatFullEntry(normalized.entries[i])).join("\n");
+        const truncatedContent = fullContent.length > MAX_CONTENT_PER_INTERACTION
+            ? fullContent.slice(0, MAX_CONTENT_PER_INTERACTION) + "\n... (truncated)"
+            : fullContent;
+        if (totalChars + truncatedContent.length > MAX_TOTAL_CONTENT) {
+            sections.push(`\n... (remaining ${sparseIndex.interactions.length - idx} interactions shown only in sparse index above)`);
+            break;
+        }
+        const flag = flagMap.get(interaction.id);
+        const triageNote = flag ? ` | Triage: ${flag.reason}` : "";
+        sections.push(`---
+#${interaction.id} | Category: ${interaction.categories.join(", ")}${triageNote}
+${truncatedContent}
+---`);
+        totalChars += truncatedContent.length;
+    }
+    return sections.join("\n\n");
+}
+function formatFullEntry(entry) {
+    const parts = [];
+    switch (entry.type) {
+        case "assistant":
+            parts.push(`[ASSISTANT] ${entry.text ?? "(no text)"}`);
+            break;
+        case "tool_use":
+            parts.push(`[TOOL_USE] ${entry.toolName ?? "unknown"}`);
+            if (entry.toolInputSummary)
+                parts.push(`  Input: ${entry.toolInputSummary}`);
+            if (entry.toolInput) {
+                const inputStr = JSON.stringify(entry.toolInput);
+                parts.push(`  Full input: ${inputStr.length > 1000 ? inputStr.slice(0, 1000) + "..." : inputStr}`);
+            }
+            break;
+        case "tool_result":
+            parts.push(`[TOOL_RESULT]`);
+            if (entry.toolResultText) {
+                const result = entry.toolResultText.length > 2000 ? entry.toolResultText.slice(0, 2000) + "..." : entry.toolResultText;
+                parts.push(`  Result: ${result}`);
+            }
+            break;
+        case "error":
+            parts.push(`[ERROR] ${entry.errorMessage ?? entry.text ?? "(unknown error)"}`);
+            break;
+        default:
+            parts.push(`[${entry.type.toUpperCase()}] ${entry.text ?? "(no content)"}`);
+    }
+    return parts.join("\n");
+}
+async function callJudge(runResult, prompt) {
+    const adapter = getAdapter(runResult.agentConfig.adapter);
+    const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "axis-deep-eval-"));
+    try {
+        const output = await adapter.run({
+            prompt,
+            config: runResult.agentConfig,
+            scenario: {
+                key: "__deep_eval__",
+                name: "AXIS Deep Evaluation",
+                prompt,
+                rubric: [],
+            },
+            workingDirectory: workspace,
+        });
+        return output.result ?? "";
+    }
+    finally {
+        try {
+            fs.rmSync(workspace, { recursive: true, force: true });
+        }
+        catch {
+            /* ignore */
+        }
+    }
+}
+// --- Response parsing ---
+/**
+ * Parse the deep eval LLM response.
+ * Fills in default audits for interactions the LLM missed and default necessity for missing categories.
+ */
+export function parseDeepEvalResponse(responseText, sparseIndex) {
+    const parsed = parseJsonFromText(responseText);
+    let llmAudits = [];
+    let llmNecessity = [];
+    if (parsed) {
+        llmAudits = parseAudits(parsed.audits, sparseIndex);
+        llmNecessity = parseNecessity(parsed.necessity);
+    }
+    // Build complete audit list: LLM-scored where available, defaults for any the LLM missed
+    const auditMap = new Map(llmAudits.map((a) => [a.id, a]));
+    const allAudits = [];
+    for (const interaction of sparseIndex.interactions) {
+        const existing = auditMap.get(interaction.id);
+        if (existing) {
+            allAudits.push(existing);
+        }
+        else {
+            allAudits.push({
+                id: interaction.id,
+                categories: interaction.categories,
+                success: interaction.hasError ? 0.3 : DEFAULT_AUDIT_SCORES.success,
+                speed: DEFAULT_AUDIT_SCORES.speed,
+                weight: DEFAULT_AUDIT_SCORES.weight,
+                contextRelevance: DEFAULT_AUDIT_SCORES.contextRelevance,
+                rationale: "default",
+            });
+        }
+    }
+    // Ensure all three categories have necessity judgments
+    const categories = ["environment", "service", "agent"];
+    const necessityMap = new Map(llmNecessity.map((n) => [n.category, n]));
+    const allNecessity = categories.map((cat) => {
+        const existing = necessityMap.get(cat);
+        if (existing)
+            return existing;
+        return {
+            category: cat,
+            score: 1.0,
+            unnecessaryIds: [],
+            rationale: "default",
+        };
+    });
+    return { audits: allAudits, necessity: allNecessity };
+}
+function parseAudits(raw, sparseIndex) {
+    if (!Array.isArray(raw))
+        return [];
+    const interactionMap = new Map(sparseIndex.interactions.map((i) => [i.id, i]));
+    const audits = [];
+    for (const item of raw) {
+        if (!item || typeof item !== "object")
+            continue;
+        const obj = item;
+        if (typeof obj.id !== "number")
+            continue;
+        const interaction = interactionMap.get(obj.id);
+        if (!interaction)
+            continue;
+        audits.push({
+            id: obj.id,
+            categories: interaction.categories,
+            success: clamp01(obj.success),
+            speed: DEFAULT_AUDIT_SCORES.speed, // placeholder — overridden by heuristic
+            weight: clamp01(obj.weight),
+            contextRelevance: clamp01(obj.contextRelevance),
+            rationale: typeof obj.rationale === "string" ? obj.rationale : "",
+        });
+    }
+    return audits;
+}
+function parseNecessity(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const validCategories = new Set(["environment", "service", "agent"]);
+    const judgments = [];
+    for (const item of raw) {
+        if (!item || typeof item !== "object")
+            continue;
+        const obj = item;
+        if (typeof obj.category !== "string" || !validCategories.has(obj.category))
+            continue;
+        const unnecessaryIds = Array.isArray(obj.unnecessaryIds)
+            ? obj.unnecessaryIds.filter((id) => typeof id === "number")
+            : [];
+        judgments.push({
+            category: obj.category,
+            score: clamp01(obj.score),
+            unnecessaryIds,
+            rationale: typeof obj.rationale === "string" ? obj.rationale : "",
+        });
+    }
+    return judgments;
+}
+function buildDefaultResult(sparseIndex) {
+    const audits = sparseIndex.interactions.map((interaction) => ({
+        id: interaction.id,
+        categories: interaction.categories,
+        success: interaction.hasError ? 0.3 : DEFAULT_AUDIT_SCORES.success,
+        speed: DEFAULT_AUDIT_SCORES.speed,
+        weight: DEFAULT_AUDIT_SCORES.weight,
+        contextRelevance: DEFAULT_AUDIT_SCORES.contextRelevance,
+        rationale: "default",
+    }));
+    const categories = ["environment", "service", "agent"];
+    const necessity = categories.map((cat) => ({
+        category: cat,
+        score: 0.8,
+        unnecessaryIds: [],
+        rationale: "default",
+    }));
+    return { audits, necessity };
+}
+function truncateSparseLines(lines) {
+    let totalChars = 0;
+    const included = [];
+    for (const line of lines) {
+        if (totalChars + line.length > MAX_SPARSE_INDEX_CHARS) {
+            included.push(`... (${lines.length - included.length} more interactions omitted)`);
+            break;
+        }
+        included.push(line);
+        totalChars += line.length;
+    }
+    return included.join("\n");
+}
+function clamp01(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return 0.5;
+    return Math.max(0, Math.min(1, value));
+}
+//# sourceMappingURL=deep-eval.js.map

package/dist/scoring/deep-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deep-eval.js","sourceRoot":"","sources":["../../src/scoring/deep-eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAYrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,iEAAiE;AACjE,MAAM,2BAA2B,GAAG,KAAK,CAAC;AAE1C,yDAAyD;AACzD,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC,oEAAoE;AACpE,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAEtC;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAiB,EACjB,WAAwB,EACxB,MAAoB,EACpB,UAAgC;IAEhC,uDAAuD;IACvD,IAAI,WAAW,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1C,OAAO,kBAAkB,CAAC,WAAW,CAAC,CAAC;IACzC,CAAC;IAED,+CAA+C;IAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAEpE,yEAAyE;IACzE,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5E,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,KAAK,GAAG,qBAAqB,CAAC,WAAW,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,WAAwB;IAC5D,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC;IAE/C,oCAAoC;IACpC,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IAEvD,MAAM,OAAO,GAAG,UAAU,GAAG,IAAI,CAAC;IAElC,0EAA0E;IAC1E,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC7B,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC7B,IAAI,OAAO,IAAI,EAAE;YAAE,OAAO,GAAG,CAAC;QAC9B,IAAI,OAAO,IAAI,EAAE;YAAE,OAAO,GAAG,CAAC;QAC9B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,qFAAqF;IACrF,IAAI,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACvC,IAAI,OAAO,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QAC/B,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC7B,IAAI,OAAO,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC7B,IAAI,OAAO,IAAI,EAAE;YAAE,OAAO,GAAG,CAAC;QAC9B,OAAO,GAAG,CAAC;IACb,CAAC;IAED,oCAAoC;IACpC,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IAC7B,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IAC7B,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC;IAC9B,IAAI,OAAO,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC;IAC9B,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAiB,EACjB,WAAwB,EACxB,MAAoB,EACpB,UAAgC;IAEhC,MAAM,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC;IAE9B,uCAAuC;IACvC,MAAM,WAAW,GAAG,mBAAmB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAE3D,4DAA4D;IAC5D,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEpF,wCAAwC;IACxC,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ;aACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,WAAW,eAAe,CAAC,CAAC,QAAQ,oBAAoB,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;aAC1G,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,UAAU,GAA0B,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9E,MAAM,iBAAiB,GAAG,UAAU;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;aACtC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9C,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,aAAa,GAAG;;EAElB,iBAAiB,IAAI,QAAQ;EAC7B,YAAY,CAAC,CAAC,CAAC,gBAAgB,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE;CACnD,CAAC;IACA,CAAC;IAED,OAAO;;;;YAIG,MAAM,CAAC,YAAY;;;EAG7B,MAAM,CAAC,MAAM;EACb,aAAa;yBACU,KAAK,CAAC,iBAAiB;EAC9C,WAAW;;;8BAGiB,KAAK,CAAC,UAAU,CAAC,WAAW;0BAChC,KAAK,CAAC,UAAU,CAAC,OAAO;wBAC1B,KAAK,CAAC,UAAU,CAAC,KAAK;YAClC,KAAK,CAAC,WAAW;oBACT,KAAK,CAAC,eAAe;;;EAGvC,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qDA+BiC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAC9B,WAAwB,EACxB,MAAoB,EACpB,UAAgC;IAEhC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,WAAW,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC/D,MAAM,WAAW,GAAG,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAElD,iDAAiD;QACjD,MAAM,WAAW,GAAG,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE3G,MAAM,gBAAgB,GACpB,WAAW,CAAC,MAAM,GAAG,2BAA2B;YAC9C,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,GAAG,mBAAmB;YACzE,CAAC,CAAC,WAAW,CAAC;QAElB,IAAI,UAAU,GAAG,gBAAgB,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;YAC7D,QAAQ,CAAC,IAAI,CACX,oBAAoB,WAAW,CAAC,YAAY,CAAC,MAAM,GAAG,GAAG,iDAAiD,CAC3G,CAAC;YACF,MAAM;QACR,CAAC;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3D,QAAQ,CAAC,IAAI,CAAC;GACf,WAAW,CAAC,EAAE,gBAAgB,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,UAAU;EAC7E,gBAAgB;IACd,CAAC,CAAC;QAEF,UAAU,IAAI,gBAAgB,CAAC,MAAM,CAAC;IACxC,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,eAAe,CAAC,KAAsB;IAC7C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;YACvD,MAAM;QACR,KAAK,UAAU;YACb,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;YACxD,IAAI,KAAK,CAAC,gBAAgB;gBAAE,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC;YAC7E,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBACjD,KAAK,CAAC,IAAI,CAAC,iBAAiB,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrG,CAAC;YACD,MAAM;QACR,KAAK,aAAa;YAChB,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;gBACzB,MAAM,MAAM,GACV,KAAK,CAAC,cAAc,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC;gBAC1G,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;YACpC,CAAC;YACD,MAAM;QACR,KAAK,OAAO;YACV,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,IAAI,IAAI,iBAAiB,EAAE,CAAC,CAAC;YAC/E,MAAM;QACR;YACE,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAoB,EAAE,MAAc;IAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/B,MAAM;YACN,MAAM,EAAE,SAAS,CAAC,WAAW;YAC7B,QAAQ,EAAE;gBACR,GAAG,EAAE,eAAe;gBACpB,IAAI,EAAE,sBAAsB;gBAC5B,MAAM;gBACN,MAAM,EAAE,EAAE;aACX;YACD,gBAAgB,EAAE,SAAS;SAC5B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAC7B,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,2BAA2B;AAE3B;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,YAAoB,EAAE,WAAwB;IAClF,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAE/C,IAAI,SAAS,GAAuB,EAAE,CAAC;IACvC,IAAI,YAAY,GAAwB,EAAE,CAAC;IAE3C,IAAI,MAAM,EAAE,CAAC;QACX,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACpD,YAAY,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;IAED,yFAAyF;IACzF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAuB,EAAE,CAAC;IAEzC,KAAK,MAAM,WAAW,IAAI,WAAW,CAAC,YAAY,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,OAAO,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO;gBAClE,KAAK,EAAE,oBAAoB,CAAC,KAAK;gBACjC,MAAM,EAAE,oBAAoB,CAAC,MAAM;gBACnC,gBAAgB,EAAE,oBAAoB,CAAC,gBAAgB;gBACvD,SAAS,EAAE,SAAS;aACrB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,MAAM,UAAU,GAA0B,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,YAAY,GAAwB,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/D,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC9B,OAAO;YACL,QAAQ,EAAE,GAAG;YACb,KAAK,EAAE,GAAG;YACV,cAAc,EAAE,EAAE;YAClB,SAAS,EAAE,SAAS;SACrB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,WAAW,CAAC,GAAY,EAAE,WAAwB;IACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAuB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,SAAS;QAChD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAE5C,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ;YAAE,SAAS;QAEzC,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,WAAW;YAAE,SAAS;QAE3B,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,UAAU,EAAE,WAAW,CAAC,UAAU;YAClC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;YAC7B,KAAK,EAAE,oBAAoB,CAAC,KAAK,EAAE,wCAAwC;YAC3E,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAC3B,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;YAC/C,SAAS,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SAClE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,GAAY;IAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAsB,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1F,MAAM,SAAS,GAAwB,EAAE,CAAC;IAE1C,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,SAAS;QAChD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAE5C,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,QAA+B,CAAC;YAAE,SAAS;QAE5G,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YACtD,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,EAAgB,EAAE,CAAC,OAAO,EAAE,KAAK,QAAQ,CAAC;YACzE,CAAC,CAAC,EAAE,CAAC;QAEP,SAAS,CAAC,IAAI,CAAC;YACb,QAAQ,EAAE,GAAG,CAAC,QAA+B;YAC7C,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;YACzB,cAAc;YACd,SAAS,EAAE,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SAClE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,WAAwB;IAClD,MAAM,MAAM,GAAuB,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAChF,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,WAAW,CAAC,UAAU;QAClC,OAAO,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO;QAClE,KAAK,EAAE,oBAAoB,CAAC,KAAK;QACjC,MAAM,EAAE,oBAAoB,CAAC,MAAM;QACnC,gBAAgB,EAAE,oBAAoB,CAAC,gBAAgB;QACvD,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,UAAU,GAA0B,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAwB,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9D,QAAQ,EAAE,GAAG;QACb,KAAK,EAAE,GAAG;QACV,cAAc,EAAE,EAAE;QAClB,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAe;IAC1C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,6BAA6B,CAAC,CAAC;YACnF,MAAM;QACR,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACrE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC"}

package/dist/scoring/goal-achievement.d.ts

@@ -0,0 +1,5 @@
+import type { NormalizedEntry } from "../transcript/types.js";
+import type { RunResult } from "../types/output.js";
+import type { GoalAchievementScore } from "../types/scoring.js";
+export declare function scoreGoalAchievement(result: RunResult, normalizedEntries: NormalizedEntry[]): Promise<GoalAchievementScore>;
+//# sourceMappingURL=goal-achievement.d.ts.map

package/dist/scoring/goal-achievement.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"goal-achievement.d.ts","sourceRoot":"","sources":["../../src/scoring/goal-achievement.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,EAAE,oBAAoB,EAAkB,MAAM,qBAAqB,CAAC;AAGhF,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,SAAS,EACjB,iBAAiB,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,oBAAoB,CAAC,CAa/B"}

package/dist/scoring/goal-achievement.js

@@ -0,0 +1,241 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { getAdapter } from "../adapters/registry.js";
+import { parseJsonFromText } from "./parse-json.js";
+export async function scoreGoalAchievement(result, normalizedEntries) {
+    const { rubric } = result;
+    const { result: finalResult } = result.output;
+    if (typeof rubric === "string") {
+        return scoreStringRubric(result, rubric, normalizedEntries, finalResult);
+    }
+    if (!rubric || rubric.length === 0) {
+        return { score: 0, criteria: [] };
+    }
+    return scoreArrayRubric(result, rubric, normalizedEntries, finalResult);
+}
+async function scoreStringRubric(runResult, rubric, entries, finalResult) {
+    const prompt = buildStringRubricPrompt(runResult, entries, finalResult, rubric);
+    const responseText = await callJudge(runResult, prompt);
+    const parsed = parseJsonFromText(responseText);
+    if (!parsed || typeof parsed.score !== "number") {
+        return {
+            score: 0,
+            criteria: [
+                {
+                    check: rubric,
+                    weight: 1.0,
+                    score: 0,
+                    rationale: "Failed to parse judge response",
+                },
+            ],
+        };
+    }
+    const score = Math.max(0, Math.min(10, Math.round(parsed.score)));
+    return {
+        score: Math.round((score / 10) * 100),
+        criteria: [
+            {
+                check: rubric,
+                weight: 1.0,
+                score,
+                rationale: parsed.rationale ?? "",
+            },
+        ],
+    };
+}
+async function scoreArrayRubric(runResult, rubric, entries, finalResult) {
+    const prompt = buildArrayRubricPrompt(runResult, entries, finalResult, rubric);
+    const responseText = await callJudge(runResult, prompt);
+    const criteria = parseArrayJudgeResponse(responseText, rubric);
+    const score = computeWeightedScore(criteria);
+    return { score, criteria };
+}
+async function callJudge(runResult, prompt) {
+    const adapter = getAdapter(runResult.agentConfig.adapter);
+    const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "axis-judge-"));
+    try {
+        const output = await adapter.run({
+            prompt,
+            config: runResult.agentConfig,
+            scenario: {
+                key: "__judge__",
+                name: "AXIS Judge",
+                prompt,
+                rubric: [],
+            },
+            workingDirectory: workspace,
+        });
+        return output.result ?? "";
+    }
+    finally {
+        try {
+            fs.rmSync(workspace, { recursive: true, force: true });
+        }
+        catch {
+            /* ignore */
+        }
+    }
+}
+/** Max characters for the condensed transcript section. */
+const MAX_TRANSCRIPT_CHARS = 50_000;
+/** Max characters per individual transcript entry. */
+const MAX_ENTRY_CHARS = 2_000;
+function buildStringRubricPrompt(result, entries, finalResult, rubric) {
+    return `You are an expert evaluator for an AI agent testing framework called AXIS.
+
+An AI agent was given a task. You must evaluate how well it performed by reviewing its transcript AND by independently verifying the results yourself.
+
+SCENARIO: ${result.scenarioName}
+
+TASK GIVEN TO AGENT:
+${getOriginalPrompt(result)}
+
+---
+
+AGENT TRANSCRIPT (condensed):
+${formatTranscriptForJudge(entries)}
+
+---
+
+AGENT'S FINAL RESULT:
+${finalResult ?? "(no final result)"}
+
+---
+
+RUBRIC:
+${rubric}
+
+---
+
+INSTRUCTIONS:
+1. Review the transcript to understand what the agent did.
+2. Where possible, independently verify the results — visit URLs, check endpoints, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
+3. Score based on what you can verify, not just what the agent claims.
+
+When done, respond with ONLY valid JSON on its own line:
+{"score": <0-10>, "rationale": "<1-2 sentence explanation>"}
+
+Score guide: 0 = not met at all, 5 = partially met, 10 = fully met.`;
+}
+function buildArrayRubricPrompt(result, entries, finalResult, rubric) {
+    const rubricText = rubric.map((r, i) => `${i}. "${r.check}" (weight: ${r.weight})`).join("\n");
+    return `You are an expert evaluator for an AI agent testing framework called AXIS.
+
+An AI agent was given a task. You must evaluate how well it performed by reviewing its transcript AND by independently verifying the results yourself.
+
+SCENARIO: ${result.scenarioName}
+
+TASK GIVEN TO AGENT:
+${getOriginalPrompt(result)}
+
+---
+
+AGENT TRANSCRIPT (condensed):
+${formatTranscriptForJudge(entries)}
+
+---
+
+AGENT'S FINAL RESULT:
+${finalResult ?? "(no final result)"}
+
+---
+
+RUBRIC CRITERIA:
+${rubricText}
+
+---
+
+INSTRUCTIONS:
+1. Review the transcript to understand what the agent did.
+2. Where possible, independently verify the results — visit URLs, check endpoints, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
+3. For each criterion, provide a score from 0 to 10 and a brief rationale.
+
+Score guide: 0 = not met at all, 5 = partially met, 10 = fully met.
+
+When done, respond with ONLY valid JSON on its own line:
+{"grades": [{"criterion_index": 0, "score": <0-10>, "rationale": "<string>"}, ...]}`;
+}
+function getOriginalPrompt(result) {
+    return result.prompt;
+}
+/**
+ * Condense normalized entries into a human-readable summary for the judge.
+ */
+function formatTranscriptForJudge(entries) {
+    if (entries.length === 0)
+        return "(empty transcript)";
+    const lines = [];
+    let totalChars = 0;
+    for (let i = 0; i < entries.length; i++) {
+        const condensed = condenseEntry(entries[i], i + 1);
+        if (totalChars + condensed.length > MAX_TRANSCRIPT_CHARS) {
+            const remaining = entries.length - i;
+            lines.push(`\n... (${remaining} more entries truncated for brevity)`);
+            break;
+        }
+        lines.push(condensed);
+        totalChars += condensed.length;
+    }
+    return lines.join("\n");
+}
+/**
+ * Condense a single normalized entry into a readable line.
+ */
+function condenseEntry(entry, index) {
+    switch (entry.type) {
+        case "assistant":
+            return `[${index}] ASSISTANT: ${truncate(entry.text ?? "(no text)", MAX_ENTRY_CHARS)}`;
+        case "tool_use": {
+            const name = entry.toolName ?? "unknown";
+            const input = entry.toolInputSummary ? `(${truncate(entry.toolInputSummary, 500)})` : "";
+            return `[${index}] TOOL_USE: ${name}${input}`;
+        }
+        case "tool_result":
+            return `[${index}] TOOL_RESULT: ${truncate(entry.toolResultText ?? "(no result)", MAX_ENTRY_CHARS)}`;
+        case "error":
+            return `[${index}] ERROR: ${truncate(entry.errorMessage ?? entry.text ?? "(unknown error)", MAX_ENTRY_CHARS)}`;
+        case "system":
+            return `[${index}] SYSTEM: ${truncate(entry.text ?? "(no content)", 500)}`;
+        case "user":
+            return `[${index}] USER: ${truncate(entry.text ?? "(no content)", MAX_ENTRY_CHARS)}`;
+        default:
+            return `[${index}] ${entry.type}: ${truncate(entry.text ?? "(no content)", 500)}`;
+    }
+}
+function truncate(text, maxLen) {
+    if (text.length <= maxLen)
+        return text;
+    return text.slice(0, maxLen) + "...";
+}
+function parseArrayJudgeResponse(responseText, rubric) {
+    const parsed = parseJsonFromText(responseText);
+    if (!parsed || !Array.isArray(parsed.grades)) {
+        return rubric.map((r) => ({
+            check: r.check,
+            weight: r.weight,
+            score: 0,
+            rationale: "Failed to parse judge response",
+        }));
+    }
+    const grades = parsed.grades;
+    return rubric.map((r, i) => {
+        const grade = grades.find((g) => g.criterion_index === i);
+        return {
+            check: r.check,
+            weight: r.weight,
+            score: grade ? Math.max(0, Math.min(10, Math.round(grade.score))) : 0,
+            rationale: grade?.rationale ?? "No grade provided",
+        };
+    });
+}
+function computeWeightedScore(criteria) {
+    if (criteria.length === 0)
+        return 0;
+    const totalWeight = criteria.reduce((sum, c) => sum + c.weight, 0);
+    if (totalWeight === 0)
+        return 0;
+    const weightedSum = criteria.reduce((sum, c) => sum + (c.score / 10) * c.weight, 0);
+    return Math.round((weightedSum / totalWeight) * 100);
+}
+//# sourceMappingURL=goal-achievement.js.map

package/dist/scoring/goal-achievement.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"goal-achievement.js","sourceRoot":"","sources":["../../src/scoring/goal-achievement.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAKrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAiB,EACjB,iBAAoC;IAEpC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IAE9C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACpC,CAAC;IAED,OAAO,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAAoB,EACpB,MAAc,EACd,OAA0B,EAC1B,WAA0B;IAE1B,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAChF,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO;YACL,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR;oBACE,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,GAAG;oBACX,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,gCAAgC;iBAC5C;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC;QACrC,QAAQ,EAAE;YACR;gBACE,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,GAAG;gBACX,KAAK;gBACL,SAAS,EAAG,MAAM,CAAC,SAAoB,IAAI,EAAE;aAC9C;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAoB,EACpB,MAAyB,EACzB,OAA0B,EAC1B,WAA0B;IAE1B,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAE7C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAoB,EAAE,MAAc;IAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IACxE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/B,MAAM;YACN,MAAM,EAAE,SAAS,CAAC,WAAW;YAC7B,QAAQ,EAAE;gBACR,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,YAAY;gBAClB,MAAM;gBACN,MAAM,EAAE,EAAE;aACX;YACD,gBAAgB,EAAE,SAAS;SAC5B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAC7B,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAEpC,sDAAsD;AACtD,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,SAAS,uBAAuB,CAC9B,MAAiB,EACjB,OAA0B,EAC1B,WAA0B,EAC1B,MAAc;IAEd,OAAO;;;;YAIG,MAAM,CAAC,YAAY;;;EAG7B,iBAAiB,CAAC,MAAM,CAAC;;;;;EAKzB,wBAAwB,CAAC,OAAO,CAAC;;;;;EAKjC,WAAW,IAAI,mBAAmB;;;;;EAKlC,MAAM;;;;;;;;;;;;oEAY4D,CAAC;AACrE,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiB,EACjB,OAA0B,EAC1B,WAA0B,EAC1B,MAAyB;IAEzB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,MAAO,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhG,OAAO;;;;YAIG,MAAM,CAAC,YAAY;;;EAG7B,iBAAiB,CAAC,MAAM,CAAC;;;;;EAKzB,wBAAwB,CAAC,OAAO,CAAC;;;;;EAKjC,WAAW,IAAI,mBAAmB;;;;;EAKlC,UAAU;;;;;;;;;;;;oFAYwE,CAAC;AACrF,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAiB;IAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,wBAAwB,CAAC,OAA0B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,oBAAoB,CAAC;IAEtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAEnD,IAAI,UAAU,GAAG,SAAS,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;YACzD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,sCAAsC,CAAC,CAAC;YACtE,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,KAAsB,EAAE,KAAa;IAC1D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,IAAI,KAAK,gBAAgB,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,WAAW,EAAE,eAAe,CAAC,EAAE,CAAC;QACzF,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,IAAI,SAAS,CAAC;YACzC,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,IAAI,KAAK,eAAe,IAAI,GAAG,KAAK,EAAE,CAAC;QAChD,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,IAAI,KAAK,kBAAkB,QAAQ,CAAC,KAAK,CAAC,cAAc,IAAI,aAAa,EAAE,eAAe,CAAC,EAAE,CAAC;QACvG,KAAK,OAAO;YACV,OAAO,IAAI,KAAK,YAAY,QAAQ,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,IAAI,IAAI,iBAAiB,EAAE,eAAe,CAAC,EAAE,CAAC;QACjH,KAAK,QAAQ;YACX,OAAO,IAAI,KAAK,aAAa,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM;YACT,OAAO,IAAI,KAAK,WAAW,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,eAAe,CAAC,EAAE,CAAC;QACvF;YACE,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC;IACtF,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAc;IAC5C,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAoB,EAAE,MAAyB;IAC9E,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAO;YACjB,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,gCAAgC;SAC5C,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAIpB,CAAC;IAEH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAO;YACjB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrE,SAAS,EAAE,KAAK,EAAE,SAAS,IAAI,mBAAmB;SACnD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAAC,QAA0B;IACtD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEpC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnE,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEpF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC;AACvD,CAAC"}