@browserbasehq/orca 3.5.1-preview.0 → 3.6.0-gemini-cua

package/dist/esm/lib/v3/verifier/prompts/firstPointOfFailure.js

@@ -0,0 +1,162 @@
+/**
+ * First-point-of-failure prompt — Step 9a of the rubric verifier pipeline.
+ *
+ * Identifies ALL distinct failure points in a trajectory and pinpoints the
+ * earliest one (the "first" failure). Diagnostic signal only — does NOT
+ * affect scoring. Surfaced in EvaluationResult.firstPointOfFailure.
+ *
+ * Uses error-taxonomy categories 1–6 (the agent-controllable error space):
+ * Selection, Hallucination, Execution & Strategy, Critical Point,
+ * Side-Effect, Tool Interaction. Categories 7–8 (task ambiguity / invalid
+ * task) are handled by Steps 10 / 9b.
+ *
+ * The taxonomy is pre-rendered into the prompt body at module load time
+ * (the data is static). Variables substituted at call time:
+ *   - task_definition
+ *   - init_url_context
+ *   - action_history
+ *   - predicted_output
+ *   - rubric_summary
+ *   - evidence_summary
+ *   - outcome_verification
+ */
+import { CALIBRATION_NOTE, getSummaryTable, getTaxonomyText, } from "../errorTaxonomy.js";
+// Pre-render the taxonomy + summary table (categories 1–6) once at load.
+const TAXONOMY_TEXT = getTaxonomyText(1, 6, 3);
+const SUMMARY_TABLE = getSummaryTable(1, 6);
+export const FIRST_POINT_OF_FAILURE_PROMPT = `You are an expert failure analyst for computer-use web agents. You will analyze a single task trajectory to identify all failure points and pinpoint the first (earliest) point of failure.
+
+You are given:
+- The task the agent was asked to complete
+- The agent's full step-by-step action history (each step has a step number, the agent's reasoning, the action taken, the URL, and a human-readable description)
+- The agent's predicted output (final answer)
+- A scored rubric with multimodal screenshot evidence showing how the agent performed on each criterion
+- The outcome verification result (whether the task was deemed successful overall)
+
+Your job is to identify **every distinct failure point** in the trajectory, pinpoint the **exact step number(s)** where it occurred, classify it using the error taxonomy below, and determine which failure occurred **first** (earliest step number).
+
+**Calibration:** ${CALIBRATION_NOTE}
+
+## Error Taxonomy
+
+${TAXONOMY_TEXT}
+
+${SUMMARY_TABLE}
+
+## Context
+
+Task: "$task_definition"$init_url_context
+
+Action History: >>>
+$action_history
+<<<
+
+Predicted Output: >>>
+$predicted_output
+<<<
+
+Scored Rubric (post-multimodal verification): >>>
+$rubric_summary
+<<<
+
+Screenshot Evidence by Criterion: >>>
+$evidence_summary
+<<<
+
+Outcome Verification Result: >>>
+$outcome_verification
+<<<
+
+## Instructions
+
+Analyze the trajectory and identify ALL distinct failure points. For each failure point:
+1. Identify the exact step number(s) in the action history where the failure occurred.
+2. Classify it using the error taxonomy above (use exact category and error type names).
+3. Ground your classification in concrete evidence (screenshot index, action-history quote, or tool output).
+
+**IMPORTANT**
+Output your answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:
+
+{{
+    "reasoning": str,
+    "has_failure": bool,
+    "failure_points": [
+        {{
+            "step_numbers": str,
+            "error_code": str,
+            "error_category": str,
+            "error_type": str,
+            "what_happened": str,
+            "agent_reasoning": str,
+            "evidence": str,
+            "impact": str
+        }}
+    ]
+}}
+`;
+/**
+ * Parse the model's flexible step-numbers field into a sorted array of step
+ * indices.
+ *
+ * Accepts:
+ *   "5"        → [5]
+ *   "5-7"      → [5, 6, 7]
+ *   "5,8,12"   → [5, 8, 12]
+ *   "5,7-9,12" → [5, 7, 8, 9, 12]
+ *
+ * Returns an empty array for unparseable input rather than throwing — failure
+ * analysis is best-effort and a malformed step-numbers field shouldn't tank
+ * the whole EvaluationResult.
+ */
+const DEFAULT_MAX_EXPANDED_STEPS = 1000;
+export function parseFailureStepNumbers(raw, opts = {}) {
+    if (typeof raw !== "string" || raw.length === 0)
+        return [];
+    const maxExpandedSteps = sanitizeNonNegativeInt(opts.maxExpandedSteps, DEFAULT_MAX_EXPANDED_STEPS);
+    if (maxExpandedSteps === 0)
+        return [];
+    const maxStep = opts.maxStep === undefined
+        ? undefined
+        : sanitizeNonNegativeInt(opts.maxStep, 0);
+    const out = new Set();
+    const addStep = (n) => {
+        if (!Number.isFinite(n) || n < 0)
+            return out.size < maxExpandedSteps;
+        if (maxStep !== undefined && n > maxStep)
+            return out.size < maxExpandedSteps;
+        out.add(n);
+        return out.size < maxExpandedSteps;
+    };
+    for (const segment of raw.split(",")) {
+        const seg = segment.trim();
+        if (!seg)
+            continue;
+        const dashIdx = seg.indexOf("-");
+        if (dashIdx > 0) {
+            const lo = Number.parseInt(seg.slice(0, dashIdx), 10);
+            const hi = Number.parseInt(seg.slice(dashIdx + 1), 10);
+            if (Number.isFinite(lo) && Number.isFinite(hi) && lo <= hi) {
+                const cappedHi = Math.min(hi, maxStep ?? hi, lo + (maxExpandedSteps - out.size) - 1);
+                for (let i = lo; i <= cappedHi; i++) {
+                    if (!addStep(i))
+                        break;
+                }
+            }
+        }
+        else {
+            const n = Number.parseInt(seg, 10);
+            if (!addStep(n))
+                break;
+        }
+        if (out.size >= maxExpandedSteps)
+            break;
+    }
+    // De-dup + sort ascending.
+    return Array.from(out).sort((a, b) => a - b);
+}
+function sanitizeNonNegativeInt(value, fallback) {
+    if (value === undefined || !Number.isFinite(value))
+        return fallback;
+    return Math.max(0, Math.floor(value));
+}
+//# sourceMappingURL=firstPointOfFailure.js.map

package/dist/esm/lib/v3/verifier/prompts/firstPointOfFailure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"firstPointOfFailure.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/firstPointOfFailure.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAG7B,yEAAyE;AACzE,MAAM,aAAa,GAAG,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/C,MAAM,aAAa,GAAG,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE5C,MAAM,CAAC,MAAM,6BAA6B,GAAG;;;;;;;;;;;mBAW1B,gBAAgB;;;;EAIjC,aAAa;;EAEb,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoDd,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,0BAA0B,GAAG,IAAI,CAAC;AAExC,MAAM,UAAU,uBAAuB,CACrC,GAAW,EACX,OAAuC,EAAE;IAEzC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC3D,MAAM,gBAAgB,GAAG,sBAAsB,CAC7C,IAAI,CAAC,gBAAgB,EACrB,0BAA0B,CAC3B,CAAC;IACF,IAAI,gBAAgB,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,KAAK,SAAS;QACxB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,MAAM,OAAO,GAAG,CAAC,CAAS,EAAW,EAAE;QACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,GAAG,gBAAgB,CAAC;QACrE,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,GAAG,OAAO;YACtC,OAAO,GAAG,CAAC,IAAI,GAAG,gBAAgB,CAAC;QACrC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACX,OAAO,GAAG,CAAC,IAAI,GAAG,gBAAgB,CAAC;IACrC,CAAC,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACtD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvD,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,EAAE,EACF,OAAO,IAAI,EAAE,EACb,EAAE,GAAG,CAAC,gBAAgB,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CACvC,CAAC;gBACF,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;wBAAE,MAAM;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAAE,MAAM;QACzB,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,IAAI,gBAAgB;YAAE,MAAM;IAC1C,CAAC;IACD,2BAA2B;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,sBAAsB,CAC7B,KAAyB,EACzB,QAAgB;IAEhB,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AACxC,CAAC","sourcesContent":["/**\n * First-point-of-failure prompt — Step 9a of the rubric verifier pipeline.\n *\n * Identifies ALL distinct failure points in a trajectory and pinpoints the\n * earliest one (the \"first\" failure). Diagnostic signal only — does NOT\n * affect scoring. Surfaced in EvaluationResult.firstPointOfFailure.\n *\n * Uses error-taxonomy categories 1–6 (the agent-controllable error space):\n * Selection, Hallucination, Execution & Strategy, Critical Point,\n * Side-Effect, Tool Interaction. Categories 7–8 (task ambiguity / invalid\n * task) are handled by Steps 10 / 9b.\n *\n * The taxonomy is pre-rendered into the prompt body at module load time\n * (the data is static). Variables substituted at call time:\n *   - task_definition\n *   - init_url_context\n *   - action_history\n *   - predicted_output\n *   - rubric_summary\n *   - evidence_summary\n *   - outcome_verification\n */\nimport {\n  CALIBRATION_NOTE,\n  getSummaryTable,\n  getTaxonomyText,\n} from \"../errorTaxonomy.js\";\nimport type { ParseFailureStepNumbersOptions } from \"../types.js\";\n\n// Pre-render the taxonomy + summary table (categories 1–6) once at load.\nconst TAXONOMY_TEXT = getTaxonomyText(1, 6, 3);\nconst SUMMARY_TABLE = getSummaryTable(1, 6);\n\nexport const FIRST_POINT_OF_FAILURE_PROMPT = `You are an expert failure analyst for computer-use web agents. You will analyze a single task trajectory to identify all failure points and pinpoint the first (earliest) point of failure.\n\nYou are given:\n- The task the agent was asked to complete\n- The agent's full step-by-step action history (each step has a step number, the agent's reasoning, the action taken, the URL, and a human-readable description)\n- The agent's predicted output (final answer)\n- A scored rubric with multimodal screenshot evidence showing how the agent performed on each criterion\n- The outcome verification result (whether the task was deemed successful overall)\n\nYour job is to identify **every distinct failure point** in the trajectory, pinpoint the **exact step number(s)** where it occurred, classify it using the error taxonomy below, and determine which failure occurred **first** (earliest step number).\n\n**Calibration:** ${CALIBRATION_NOTE}\n\n## Error Taxonomy\n\n${TAXONOMY_TEXT}\n\n${SUMMARY_TABLE}\n\n## Context\n\nTask: \"$task_definition\"$init_url_context\n\nAction History: >>>\n$action_history\n<<<\n\nPredicted Output: >>>\n$predicted_output\n<<<\n\nScored Rubric (post-multimodal verification): >>>\n$rubric_summary\n<<<\n\nScreenshot Evidence by Criterion: >>>\n$evidence_summary\n<<<\n\nOutcome Verification Result: >>>\n$outcome_verification\n<<<\n\n## Instructions\n\nAnalyze the trajectory and identify ALL distinct failure points. For each failure point:\n1. Identify the exact step number(s) in the action history where the failure occurred.\n2. Classify it using the error taxonomy above (use exact category and error type names).\n3. Ground your classification in concrete evidence (screenshot index, action-history quote, or tool output).\n\n**IMPORTANT**\nOutput your answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THIS SCHEMA:\n\n{{\n    \"reasoning\": str,\n    \"has_failure\": bool,\n    \"failure_points\": [\n        {{\n            \"step_numbers\": str,\n            \"error_code\": str,\n            \"error_category\": str,\n            \"error_type\": str,\n            \"what_happened\": str,\n            \"agent_reasoning\": str,\n            \"evidence\": str,\n            \"impact\": str\n        }}\n    ]\n}}\n`;\n\n/**\n * Parse the model's flexible step-numbers field into a sorted array of step\n * indices.\n *\n * Accepts:\n *   \"5\"        → [5]\n *   \"5-7\"      → [5, 6, 7]\n *   \"5,8,12\"   → [5, 8, 12]\n *   \"5,7-9,12\" → [5, 7, 8, 9, 12]\n *\n * Returns an empty array for unparseable input rather than throwing — failure\n * analysis is best-effort and a malformed step-numbers field shouldn't tank\n * the whole EvaluationResult.\n */\nconst DEFAULT_MAX_EXPANDED_STEPS = 1000;\n\nexport function parseFailureStepNumbers(\n  raw: string,\n  opts: ParseFailureStepNumbersOptions = {},\n): number[] {\n  if (typeof raw !== \"string\" || raw.length === 0) return [];\n  const maxExpandedSteps = sanitizeNonNegativeInt(\n    opts.maxExpandedSteps,\n    DEFAULT_MAX_EXPANDED_STEPS,\n  );\n  if (maxExpandedSteps === 0) return [];\n  const maxStep =\n    opts.maxStep === undefined\n      ? undefined\n      : sanitizeNonNegativeInt(opts.maxStep, 0);\n  const out = new Set<number>();\n  const addStep = (n: number): boolean => {\n    if (!Number.isFinite(n) || n < 0) return out.size < maxExpandedSteps;\n    if (maxStep !== undefined && n > maxStep)\n      return out.size < maxExpandedSteps;\n    out.add(n);\n    return out.size < maxExpandedSteps;\n  };\n  for (const segment of raw.split(\",\")) {\n    const seg = segment.trim();\n    if (!seg) continue;\n    const dashIdx = seg.indexOf(\"-\");\n    if (dashIdx > 0) {\n      const lo = Number.parseInt(seg.slice(0, dashIdx), 10);\n      const hi = Number.parseInt(seg.slice(dashIdx + 1), 10);\n      if (Number.isFinite(lo) && Number.isFinite(hi) && lo <= hi) {\n        const cappedHi = Math.min(\n          hi,\n          maxStep ?? hi,\n          lo + (maxExpandedSteps - out.size) - 1,\n        );\n        for (let i = lo; i <= cappedHi; i++) {\n          if (!addStep(i)) break;\n        }\n      }\n    } else {\n      const n = Number.parseInt(seg, 10);\n      if (!addStep(n)) break;\n    }\n    if (out.size >= maxExpandedSteps) break;\n  }\n  // De-dup + sort ascending.\n  return Array.from(out).sort((a, b) => a - b);\n}\n\nfunction sanitizeNonNegativeInt(\n  value: number | undefined,\n  fallback: number,\n): number {\n  if (value === undefined || !Number.isFinite(value)) return fallback;\n  return Math.max(0, Math.floor(value));\n}\n"]}

package/dist/esm/lib/v3/verifier/prompts/fusedJudgment.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Fused judgment prompt — Approach B's single-call replacement for
+ * Steps 4+6+8 (and optionally folded 9a + 10).
+ *
+ * One multimodal call grades every rubric criterion, emits an independent
+ * outcome verdict, surfaces findings, optionally identifies the first point
+ * of failure, and optionally classifies task validity. The structured
+ * response is rich enough to populate the full `EvaluationResult` object without
+ * additional LLM calls.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - rubric_block             — full rubric: index, criterion, description,
+ *                                max_points, optional condition
+ *   - evidence_block           — per-criterion top-K evidence manifest
+ *                                (image refs + ariaTree snippets). Each image
+ *                                in the message body is keyed by its label
+ *                                here (e.g., "Evidence #3 — step=12, probe").
+ *   - taxonomy_block           — error taxonomy text (only when
+ *                                fold_failure_analysis = true). Otherwise
+ *                                empty string.
+ *   - fold_failure_analysis    — "true" / "false" — whether to emit failure
+ *                                analysis in the response.
+ *   - fold_task_validity       — "true" / "false" — whether to emit task
+ *                                validity classification in the response.
+ *   - final_state_block        — always-attached final URL + ariaTree of the
+ *                                last step probe and finalObservation; this
+ *                                bypasses the per-criterion top-K selection
+ *                                so the judge always has the closing page
+ *                                content available.
+ */
+export declare const FUSED_JUDGMENT_PROMPT = "Task: $task_definition$init_url_context\n\n**Current Date:** $current_date\n\nYou are an expert evaluator of web-navigation agent trajectories. You will grade the agent's run against a rubric, decide whether the overall task succeeded, and surface diagnostics \u2014 all in one structured response.\n\nUse the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible \u2014 classify as task_validity.is_invalid with code 8.1).\n\n**Action History:**\n$action_history\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Rubric:**\n$rubric_block\n\n**Evidence (grouped by criterion):**\n$evidence_block\n\nEach evidence reference points to an image attached below or to a text snippet inline above. Screenshots are listed in chronological order across the trajectory; when two screenshots show the same element, **the LATER screenshot reflects the final state and takes precedence**.\n\n**Final trajectory state** (authoritative \u2014 this is the page content and screenshot captured at the very end of the run; treat it as ground truth for what the agent saw on its final page, even if no `extract`/`observe` step appears in the action history):\n$final_state_block\n\n**Optional sections to include in the response:**\n- Failure analysis: $fold_failure_analysis\n- Task validity classification: $fold_task_validity\n\nWhen failure analysis is requested and you judge `output_success: false`, you must populate `failure_point` using the error taxonomy below:\n\n$taxonomy_block\n\nWhen task validity is requested, you must populate `task_validity` with the booleans `is_ambiguous` / `is_invalid` and, when each is true, a single one-line free-form reason in `ambiguity_reason` / `invalid_reason` (e.g., \"Requested dates are in the past relative to the current date\"). Leave the reason field empty when the corresponding flag is false.\n\n---\n\n**Core Evaluation Principles** (these OVERRIDE the criterion descriptions when they conflict):\n\n1. **Best Effort Evaluation.** The agent should be evaluated on helpfulness and effort within constraints it cannot control.\n\n2. **Uncontrollable Blockers** (award full credit when these prevent task completion): platform issues (site down, CAPTCHA, login walls), entity non-existence, availability constraints (out of stock, sold out), platform limitations. If screenshots CONFIRM the blocker, award full credit even for downstream dependent criteria.\n\n3. **Controllable Failures** (penalize): wrong selections when correct options are available, poor execution (not using filters, not attempting specified platforms), hallucinations (claiming success without evidence), insufficient effort.\n\n4. **Tasks with Explicit Constraints.** Distinguish \"searched for the constraint\" from \"found results that actually satisfy it\". If the hard constraint is NOT met in the evidence, award only minimal partial credit for the search effort.\n\n5. **Ambiguous Wording \u2014 Don't Penalize for One Valid Interpretation.** If the task has multiple defensible readings, the agent picking one is fine.\n\n6. **Cascading Dependencies:**\n   - Scenario A: Blocker is uncontrollable \u2192 award full credit for downstream criteria that couldn't be attempted.\n   - Scenario B: Blocker is a controllable error \u2192 cascade partial/zero credit downstream.\n   - Scenario C: Don't re-penalize for the same deviation across multiple criteria.\n   - Scenario D: Shared platform blockers across sibling sub-tasks \u2192 award full credit for all affected siblings.\n\n7. **Conditional Criteria.** Some criteria have a \"condition\" field. Only score them when the condition is met; otherwise mark the criterion as not applicable (give it max_points so it doesn't drag down the process score).\n\n8. **Distinguish nitpicks from critical errors:**\n   - Only nitpicks \u2192 75\u2013100% of max\n   - Correct approach, wrong final answer \u2192 40\u201380%\n   - Critical error \u2192 penalize per severity\n   - Mix of nitpicks + a critical error \u2192 score based on the critical error\n\n**Outcome Judgment:**\n\n`output_success` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them \u2014 a task can have high process score and still fail (right approach, wrong final answer) or have lower process score and still succeed (the answer is right, intermediate steps were inelegant).\n\n**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.\n\n---\n\n**Output Format:**\n\nOutput one JSON object matching this schema. Include the optional sections only when requested above.\n\n{{\n  \"outcome\": {{\n    \"primary_intent\": \"<one-sentence restatement of what the task was asking for>\",\n    \"reasoning\": \"<your reasoning for the success / failure verdict>\",\n    \"output_success\": true,\n    \"findings\": [\n      {{\n        \"category\": \"agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other\",\n        \"severity\": \"info|warning|blocking\",\n        \"description\": \"...\",\n        \"suggestedAction\": \"...\",\n        \"relatedSteps\": [3, 4]\n      }}\n    ]\n  }},\n  \"per_criterion\": [\n    {{\n      \"criterion_idx\": 0,\n      \"applicable_evidence\": \"Which evidence is applicable; cite by 'Screenshot N \u2014 step=K' or aria-tree step number.\",\n      \"justification\": \"How the visual / textual evidence supports your score.\",\n      \"earned_points\": 4,\n      \"evidence_sufficient\": true\n    }}\n  ],\n  \"failure_point\": {{\n    \"step_index\": 17,\n    \"error_code\": \"1.4\",\n    \"error_category\": \"Selection\",\n    \"description\": \"<one-line description of what went wrong at this step>\"\n  }},\n  \"task_validity\": {{\n    \"is_ambiguous\": false,\n    \"ambiguity_reason\": \"\",\n    \"is_invalid\": false,\n    \"invalid_reason\": \"\"\n  }}\n}}\n\n- Omit `failure_point` when `output_success` is true or when failure analysis was not requested.\n- Omit `task_validity` when task-validity classification was not requested.\n- You MUST emit exactly one `per_criterion` entry per rubric item, in rubric order.\n- `earned_points` must be in [0, max_points] for that criterion.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n";

package/dist/esm/lib/v3/verifier/prompts/fusedJudgment.js

@@ -0,0 +1,155 @@
+/**
+ * Fused judgment prompt — Approach B's single-call replacement for
+ * Steps 4+6+8 (and optionally folded 9a + 10).
+ *
+ * One multimodal call grades every rubric criterion, emits an independent
+ * outcome verdict, surfaces findings, optionally identifies the first point
+ * of failure, and optionally classifies task validity. The structured
+ * response is rich enough to populate the full `EvaluationResult` object without
+ * additional LLM calls.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - rubric_block             — full rubric: index, criterion, description,
+ *                                max_points, optional condition
+ *   - evidence_block           — per-criterion top-K evidence manifest
+ *                                (image refs + ariaTree snippets). Each image
+ *                                in the message body is keyed by its label
+ *                                here (e.g., "Evidence #3 — step=12, probe").
+ *   - taxonomy_block           — error taxonomy text (only when
+ *                                fold_failure_analysis = true). Otherwise
+ *                                empty string.
+ *   - fold_failure_analysis    — "true" / "false" — whether to emit failure
+ *                                analysis in the response.
+ *   - fold_task_validity       — "true" / "false" — whether to emit task
+ *                                validity classification in the response.
+ *   - final_state_block        — always-attached final URL + ariaTree of the
+ *                                last step probe and finalObservation; this
+ *                                bypasses the per-criterion top-K selection
+ *                                so the judge always has the closing page
+ *                                content available.
+ */
+export const FUSED_JUDGMENT_PROMPT = `Task: $task_definition$init_url_context
+
+**Current Date:** $current_date
+
+You are an expert evaluator of web-navigation agent trajectories. You will grade the agent's run against a rubric, decide whether the overall task succeeded, and surface diagnostics — all in one structured response.
+
+Use the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible — classify as task_validity.is_invalid with code 8.1).
+
+**Action History:**
+$action_history
+
+**Agent's Predicted Output (Final Answer):**
+$agent_predicted_output
+
+**Rubric:**
+$rubric_block
+
+**Evidence (grouped by criterion):**
+$evidence_block
+
+Each evidence reference points to an image attached below or to a text snippet inline above. Screenshots are listed in chronological order across the trajectory; when two screenshots show the same element, **the LATER screenshot reflects the final state and takes precedence**.
+
+**Final trajectory state** (authoritative — this is the page content and screenshot captured at the very end of the run; treat it as ground truth for what the agent saw on its final page, even if no \`extract\`/\`observe\` step appears in the action history):
+$final_state_block
+
+**Optional sections to include in the response:**
+- Failure analysis: $fold_failure_analysis
+- Task validity classification: $fold_task_validity
+
+When failure analysis is requested and you judge \`output_success: false\`, you must populate \`failure_point\` using the error taxonomy below:
+
+$taxonomy_block
+
+When task validity is requested, you must populate \`task_validity\` with the booleans \`is_ambiguous\` / \`is_invalid\` and, when each is true, a single one-line free-form reason in \`ambiguity_reason\` / \`invalid_reason\` (e.g., "Requested dates are in the past relative to the current date"). Leave the reason field empty when the corresponding flag is false.
+
+---
+
+**Core Evaluation Principles** (these OVERRIDE the criterion descriptions when they conflict):
+
+1. **Best Effort Evaluation.** The agent should be evaluated on helpfulness and effort within constraints it cannot control.
+
+2. **Uncontrollable Blockers** (award full credit when these prevent task completion): platform issues (site down, CAPTCHA, login walls), entity non-existence, availability constraints (out of stock, sold out), platform limitations. If screenshots CONFIRM the blocker, award full credit even for downstream dependent criteria.
+
+3. **Controllable Failures** (penalize): wrong selections when correct options are available, poor execution (not using filters, not attempting specified platforms), hallucinations (claiming success without evidence), insufficient effort.
+
+4. **Tasks with Explicit Constraints.** Distinguish "searched for the constraint" from "found results that actually satisfy it". If the hard constraint is NOT met in the evidence, award only minimal partial credit for the search effort.
+
+5. **Ambiguous Wording — Don't Penalize for One Valid Interpretation.** If the task has multiple defensible readings, the agent picking one is fine.
+
+6. **Cascading Dependencies:**
+   - Scenario A: Blocker is uncontrollable → award full credit for downstream criteria that couldn't be attempted.
+   - Scenario B: Blocker is a controllable error → cascade partial/zero credit downstream.
+   - Scenario C: Don't re-penalize for the same deviation across multiple criteria.
+   - Scenario D: Shared platform blockers across sibling sub-tasks → award full credit for all affected siblings.
+
+7. **Conditional Criteria.** Some criteria have a "condition" field. Only score them when the condition is met; otherwise mark the criterion as not applicable (give it max_points so it doesn't drag down the process score).
+
+8. **Distinguish nitpicks from critical errors:**
+   - Only nitpicks → 75–100% of max
+   - Correct approach, wrong final answer → 40–80%
+   - Critical error → penalize per severity
+   - Mix of nitpicks + a critical error → score based on the critical error
+
+**Outcome Judgment:**
+
+\`output_success\` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them — a task can have high process score and still fail (right approach, wrong final answer) or have lower process score and still succeed (the answer is right, intermediate steps were inelegant).
+
+**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.
+
+---
+
+**Output Format:**
+
+Output one JSON object matching this schema. Include the optional sections only when requested above.
+
+{{
+  "outcome": {{
+    "primary_intent": "<one-sentence restatement of what the task was asking for>",
+    "reasoning": "<your reasoning for the success / failure verdict>",
+    "output_success": true,
+    "findings": [
+      {{
+        "category": "agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other",
+        "severity": "info|warning|blocking",
+        "description": "...",
+        "suggestedAction": "...",
+        "relatedSteps": [3, 4]
+      }}
+    ]
+  }},
+  "per_criterion": [
+    {{
+      "criterion_idx": 0,
+      "applicable_evidence": "Which evidence is applicable; cite by 'Screenshot N — step=K' or aria-tree step number.",
+      "justification": "How the visual / textual evidence supports your score.",
+      "earned_points": 4,
+      "evidence_sufficient": true
+    }}
+  ],
+  "failure_point": {{
+    "step_index": 17,
+    "error_code": "1.4",
+    "error_category": "Selection",
+    "description": "<one-line description of what went wrong at this step>"
+  }},
+  "task_validity": {{
+    "is_ambiguous": false,
+    "ambiguity_reason": "",
+    "is_invalid": false,
+    "invalid_reason": ""
+  }}
+}}
+
+- Omit \`failure_point\` when \`output_success\` is true or when failure analysis was not requested.
+- Omit \`task_validity\` when task-validity classification was not requested.
+- You MUST emit exactly one \`per_criterion\` entry per rubric item, in rubric order.
+- \`earned_points\` must be in [0, max_points] for that criterion.
+
+DO NOT OUTPUT ANYTHING OTHER THAN JSON.
+`;
+//# sourceMappingURL=fusedJudgment.js.map

package/dist/esm/lib/v3/verifier/prompts/fusedJudgment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fusedJudgment.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/fusedJudgment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuHpC,CAAC","sourcesContent":["/**\n * Fused judgment prompt — Approach B's single-call replacement for\n * Steps 4+6+8 (and optionally folded 9a + 10).\n *\n * One multimodal call grades every rubric criterion, emits an independent\n * outcome verdict, surfaces findings, optionally identifies the first point\n * of failure, and optionally classifies task validity. The structured\n * response is rich enough to populate the full `EvaluationResult` object without\n * additional LLM calls.\n *\n * Variables:\n *   - task_definition          — instruction string\n *   - init_url_context         — \"Starting URL: ...\" or empty\n *   - action_history           — compact textual action history\n *   - agent_predicted_output   — agent's final answer / message\n *   - rubric_block             — full rubric: index, criterion, description,\n *                                max_points, optional condition\n *   - evidence_block           — per-criterion top-K evidence manifest\n *                                (image refs + ariaTree snippets). Each image\n *                                in the message body is keyed by its label\n *                                here (e.g., \"Evidence #3 — step=12, probe\").\n *   - taxonomy_block           — error taxonomy text (only when\n *                                fold_failure_analysis = true). Otherwise\n *                                empty string.\n *   - fold_failure_analysis    — \"true\" / \"false\" — whether to emit failure\n *                                analysis in the response.\n *   - fold_task_validity       — \"true\" / \"false\" — whether to emit task\n *                                validity classification in the response.\n *   - final_state_block        — always-attached final URL + ariaTree of the\n *                                last step probe and finalObservation; this\n *                                bypasses the per-criterion top-K selection\n *                                so the judge always has the closing page\n *                                content available.\n */\n\nexport const FUSED_JUDGMENT_PROMPT = `Task: $task_definition$init_url_context\n\n**Current Date:** $current_date\n\nYou are an expert evaluator of web-navigation agent trajectories. You will grade the agent's run against a rubric, decide whether the overall task succeeded, and surface diagnostics — all in one structured response.\n\nUse the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible — classify as task_validity.is_invalid with code 8.1).\n\n**Action History:**\n$action_history\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Rubric:**\n$rubric_block\n\n**Evidence (grouped by criterion):**\n$evidence_block\n\nEach evidence reference points to an image attached below or to a text snippet inline above. Screenshots are listed in chronological order across the trajectory; when two screenshots show the same element, **the LATER screenshot reflects the final state and takes precedence**.\n\n**Final trajectory state** (authoritative — this is the page content and screenshot captured at the very end of the run; treat it as ground truth for what the agent saw on its final page, even if no \\`extract\\`/\\`observe\\` step appears in the action history):\n$final_state_block\n\n**Optional sections to include in the response:**\n- Failure analysis: $fold_failure_analysis\n- Task validity classification: $fold_task_validity\n\nWhen failure analysis is requested and you judge \\`output_success: false\\`, you must populate \\`failure_point\\` using the error taxonomy below:\n\n$taxonomy_block\n\nWhen task validity is requested, you must populate \\`task_validity\\` with the booleans \\`is_ambiguous\\` / \\`is_invalid\\` and, when each is true, a single one-line free-form reason in \\`ambiguity_reason\\` / \\`invalid_reason\\` (e.g., \"Requested dates are in the past relative to the current date\"). Leave the reason field empty when the corresponding flag is false.\n\n---\n\n**Core Evaluation Principles** (these OVERRIDE the criterion descriptions when they conflict):\n\n1. **Best Effort Evaluation.** The agent should be evaluated on helpfulness and effort within constraints it cannot control.\n\n2. **Uncontrollable Blockers** (award full credit when these prevent task completion): platform issues (site down, CAPTCHA, login walls), entity non-existence, availability constraints (out of stock, sold out), platform limitations. If screenshots CONFIRM the blocker, award full credit even for downstream dependent criteria.\n\n3. **Controllable Failures** (penalize): wrong selections when correct options are available, poor execution (not using filters, not attempting specified platforms), hallucinations (claiming success without evidence), insufficient effort.\n\n4. **Tasks with Explicit Constraints.** Distinguish \"searched for the constraint\" from \"found results that actually satisfy it\". If the hard constraint is NOT met in the evidence, award only minimal partial credit for the search effort.\n\n5. **Ambiguous Wording — Don't Penalize for One Valid Interpretation.** If the task has multiple defensible readings, the agent picking one is fine.\n\n6. **Cascading Dependencies:**\n   - Scenario A: Blocker is uncontrollable → award full credit for downstream criteria that couldn't be attempted.\n   - Scenario B: Blocker is a controllable error → cascade partial/zero credit downstream.\n   - Scenario C: Don't re-penalize for the same deviation across multiple criteria.\n   - Scenario D: Shared platform blockers across sibling sub-tasks → award full credit for all affected siblings.\n\n7. **Conditional Criteria.** Some criteria have a \"condition\" field. Only score them when the condition is met; otherwise mark the criterion as not applicable (give it max_points so it doesn't drag down the process score).\n\n8. **Distinguish nitpicks from critical errors:**\n   - Only nitpicks → 75–100% of max\n   - Correct approach, wrong final answer → 40–80%\n   - Critical error → penalize per severity\n   - Mix of nitpicks + a critical error → score based on the critical error\n\n**Outcome Judgment:**\n\n\\`output_success\\` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them — a task can have high process score and still fail (right approach, wrong final answer) or have lower process score and still succeed (the answer is right, intermediate steps were inelegant).\n\n**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.\n\n---\n\n**Output Format:**\n\nOutput one JSON object matching this schema. Include the optional sections only when requested above.\n\n{{\n  \"outcome\": {{\n    \"primary_intent\": \"<one-sentence restatement of what the task was asking for>\",\n    \"reasoning\": \"<your reasoning for the success / failure verdict>\",\n    \"output_success\": true,\n    \"findings\": [\n      {{\n        \"category\": \"agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other\",\n        \"severity\": \"info|warning|blocking\",\n        \"description\": \"...\",\n        \"suggestedAction\": \"...\",\n        \"relatedSteps\": [3, 4]\n      }}\n    ]\n  }},\n  \"per_criterion\": [\n    {{\n      \"criterion_idx\": 0,\n      \"applicable_evidence\": \"Which evidence is applicable; cite by 'Screenshot N — step=K' or aria-tree step number.\",\n      \"justification\": \"How the visual / textual evidence supports your score.\",\n      \"earned_points\": 4,\n      \"evidence_sufficient\": true\n    }}\n  ],\n  \"failure_point\": {{\n    \"step_index\": 17,\n    \"error_code\": \"1.4\",\n    \"error_category\": \"Selection\",\n    \"description\": \"<one-line description of what went wrong at this step>\"\n  }},\n  \"task_validity\": {{\n    \"is_ambiguous\": false,\n    \"ambiguity_reason\": \"\",\n    \"is_invalid\": false,\n    \"invalid_reason\": \"\"\n  }}\n}}\n\n- Omit \\`failure_point\\` when \\`output_success\\` is true or when failure analysis was not requested.\n- Omit \\`task_validity\\` when task-validity classification was not requested.\n- You MUST emit exactly one \\`per_criterion\\` entry per rubric item, in rubric order.\n- \\`earned_points\\` must be in [0, max_points] for that criterion.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n`;\n"]}

package/dist/esm/lib/v3/verifier/prompts/fusedOutcome.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Fused outcome prompt — Approach A's combined Step 8 + optional folded
+ * 9a/10 call. Consumes a pre-scored rubric (computed deterministically from
+ * Approach A's per-criterion analyses) and emits the outcome verdict,
+ * findings, and optionally the first point of failure + task validity.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - outcome_evidence_summary — selected text evidence snippets from the
+ *                                trajectory, ordered by step
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - rubric_summary           — pre-scored rubric: per-criterion earned/max
+ *                                + the justifications from per-criterion calls
+ *   - taxonomy_block           — error taxonomy text (only when
+ *                                fold_failure_analysis is true; "" otherwise)
+ *   - fold_failure_analysis    — "true" / "false"
+ *   - fold_task_validity       — "true" / "false"
+ *   - final_state_block        — always-attached final URL + ariaTree of the
+ *                                last step probe and finalObservation; this
+ *                                bypasses the keyword-based excerpt selection
+ *                                used by outcome_evidence_summary so the
+ *                                judge always has the closing page content.
+ */
+export declare const FUSED_OUTCOME_PROMPT = "Task: $task_definition$init_url_context\n\n**Current Date:** $current_date\n\nYou are an expert evaluator of web-navigation agent trajectories. The rubric has already been scored per criterion (results below). Your job is to produce the overall outcome verdict.\n\nUse the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible \u2014 classify as task_validity.is_invalid with code 8.1).\n\n**Action History:**\n$action_history\n\n**Selected Trajectory Evidence:**\n$outcome_evidence_summary\n\n**Final trajectory state** (authoritative \u2014 page content and screenshot captured at the very end of the run; treat as ground truth for what the agent saw on its final page, even when no `extract`/`observe` step appears in the action history):\n$final_state_block\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Pre-Scored Rubric (per-criterion earned points + justifications):**\n$rubric_summary\n\n**Optional sections in the response:**\n- Failure analysis: $fold_failure_analysis\n- Task validity classification: $fold_task_validity\n\nWhen failure analysis is requested and you judge `output_success: false`, populate `failure_point` using the error taxonomy below:\n\n$taxonomy_block\n\nWhen task validity is requested, populate `task_validity` with the booleans `is_ambiguous` / `is_invalid` and, when each is true, a single one-line free-form reason in `ambiguity_reason` / `invalid_reason` (e.g., \"Requested dates are in the past relative to the current date\"). Leave the reason field empty when the corresponding flag is false.\n\n---\n\n**Outcome judgment:**\n`output_success` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them \u2014 a task can have high process score and still fail (right approach, wrong final answer), or have lower process score and still succeed.\n\nApply these rules when making the outcome verdict:\n\n1. **Judge the delivered result, not the route.** If the final answer satisfies the user's requested deliverable and the trajectory does not contradict it, mark success even if the agent used an inefficient path, clicked around after finding the answer, or used a slightly different but reasonable search control. Do not fail solely because the process was imperfect.\n\n2. **Use the simulated trajectory as truth.** Do not use outside/current-world knowledge to override what the saved browser state shows. If search results, page text, URLs, or screenshots in the trajectory support a concrete answer, judge against that evidence even if the real web may differ. Do not introduce an alternate \"correct\" answer from your own knowledge unless that alternate answer is visible in the supplied trajectory evidence.\n\n3. **Treat attached screenshots as partial evidence.** Screenshots may be sampled from a longer trajectory. If the final answer is concrete and the action history shows the agent reached a relevant source (page, list, chart, search result, product page, map result), assume the answer may have been extracted from that source unless the attached screenshots or action history directly contradict it. Do not call a concrete answer fabricated merely because the exact supporting text is absent from the sampled screenshots.\n\n4. **Require the actual deliverable.** The final answer is the user-visible deliverable. For tasks asking to find, show, get, browse, list, summarize, report, compare, or provide values/content, the final answer must contain the requested content or a direct correct link to the requested document/page. Merely saying the agent \"found\", \"extracted\", \"provided\", \"displayed\", \"retrieved\", or \"located\" the information is not enough, even if the browser reached the right page. The selected trajectory evidence is only context; it cannot fill in missing values, lists, links, or summaries that are absent from the final answer. Only purely navigational tasks phrased as opening a page can succeed from navigation alone.\n\n5. **Report/document tasks need the report/document.** If the task asks to get, open, retrieve, download, or provide a report/document/page, the final answer must include a direct correct URL, the opened document/page itself must be the final browser state, or the final answer must include the requested document content. Merely naming the document title is not enough unless the user only asked for the title/name.\n\n6. **Do not invent equivalence.** If the final answer gives a URL, document, article id, product, entity, date, or value that differs from what the trajectory visibly reached, do not assume an unseen redirect, canonicalization, or equivalence. Treat the mismatch as a contradiction unless the supplied trajectory evidence itself shows they are equivalent.\n\n7. **Hard constraints still matter.** If the latest relevant attached screenshot, selected trajectory evidence, or action-history URL clearly shows that an explicit constraint was not met (for example wrong sort/filter/date/class, permanently closed location reported as active, visible product/list item outside the requested color/size/status, wrong URL/article id, or wrong entity), that is a critical issue.\n\n8. **Dynamic sources can disagree.** Search results, charts, rankings, prices, and listings can differ across pages or update times in the saved trajectory. If the final answer is concrete and matches at least one relevant trajectory source, do not fail solely because another relevant source in the trajectory shows a different dynamic value, unless the final answer clearly used the contradicted source or the task required that exact source.\n\n9. **Shopping/search constraints may be source-level.** For product-search tasks, a search query, filter, result page, or product title/snippet can satisfy a descriptive constraint such as \"hypoallergenic\" when the final answer provides concrete candidate products and no supplied evidence directly contradicts the constraint. Do not fail solely because a later product page does not repeat every search constraint as a formal specification.\n\n10. **Separate nitpicks from critical issues.** Minor wording, harmless method choices, unsupported extra details that are not part of the requested deliverable, caveats about imperfect verification, or small presentation differences should not flip a successful answer to failure when the final answer still provides concrete requested content and the trajectory does not directly contradict it. Wrong requested entities, unsupported fabricated requested facts, active-vs-closed mistakes, wrong sort/filter results, wrong dates/classes, and missing requested content are critical.\n\nUse this decision order:\n\n- First ask whether the final answer contains the requested user-facing artifact: values, list items, summary, comparison, report link, product candidates, or page/document URL. If it only narrates that the artifact was found/extracted/provided, mark failure.\n- Then check for direct contradictions in the supplied trajectory evidence. Contradictions must come from the provided action history, URLs, screenshots, or final answer, not from outside knowledge.\n- If the final answer is concrete, the trajectory reached a relevant source, and the supplied evidence does not directly contradict the final answer, mark success.\n\n**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.\n\n---\n\n**Output Format:**\n\nOutput one JSON object:\n\n{{\n  \"outcome\": {{\n    \"primary_intent\": \"<one-sentence restatement of what the task was asking for>\",\n    \"reasoning\": \"<your reasoning for the success / failure verdict>\",\n    \"output_success\": true,\n    \"findings\": [\n      {{\n        \"category\": \"agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other\",\n        \"severity\": \"info|warning|blocking\",\n        \"description\": \"...\",\n        \"suggestedAction\": \"...\",\n        \"relatedSteps\": [3, 4]\n      }}\n    ]\n  }},\n  \"failure_point\": {{\n    \"step_index\": 17,\n    \"error_code\": \"1.4\",\n    \"error_category\": \"Selection\",\n    \"description\": \"<one-line description of what went wrong at this step>\"\n  }},\n  \"task_validity\": {{\n    \"is_ambiguous\": false,\n    \"ambiguity_reason\": \"\",\n    \"is_invalid\": false,\n    \"invalid_reason\": \"\"\n  }}\n}}\n\n- Omit `failure_point` when `output_success` is true or failure analysis was not requested.\n- Omit `task_validity` when task-validity classification was not requested.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n";

package/dist/esm/lib/v3/verifier/prompts/fusedOutcome.js

@@ -0,0 +1,134 @@
+/**
+ * Fused outcome prompt — Approach A's combined Step 8 + optional folded
+ * 9a/10 call. Consumes a pre-scored rubric (computed deterministically from
+ * Approach A's per-criterion analyses) and emits the outcome verdict,
+ * findings, and optionally the first point of failure + task validity.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - outcome_evidence_summary — selected text evidence snippets from the
+ *                                trajectory, ordered by step
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - rubric_summary           — pre-scored rubric: per-criterion earned/max
+ *                                + the justifications from per-criterion calls
+ *   - taxonomy_block           — error taxonomy text (only when
+ *                                fold_failure_analysis is true; "" otherwise)
+ *   - fold_failure_analysis    — "true" / "false"
+ *   - fold_task_validity       — "true" / "false"
+ *   - final_state_block        — always-attached final URL + ariaTree of the
+ *                                last step probe and finalObservation; this
+ *                                bypasses the keyword-based excerpt selection
+ *                                used by outcome_evidence_summary so the
+ *                                judge always has the closing page content.
+ */
+export const FUSED_OUTCOME_PROMPT = `Task: $task_definition$init_url_context
+
+**Current Date:** $current_date
+
+You are an expert evaluator of web-navigation agent trajectories. The rubric has already been scored per criterion (results below). Your job is to produce the overall outcome verdict.
+
+Use the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible — classify as task_validity.is_invalid with code 8.1).
+
+**Action History:**
+$action_history
+
+**Selected Trajectory Evidence:**
+$outcome_evidence_summary
+
+**Final trajectory state** (authoritative — page content and screenshot captured at the very end of the run; treat as ground truth for what the agent saw on its final page, even when no \`extract\`/\`observe\` step appears in the action history):
+$final_state_block
+
+**Agent's Predicted Output (Final Answer):**
+$agent_predicted_output
+
+**Pre-Scored Rubric (per-criterion earned points + justifications):**
+$rubric_summary
+
+**Optional sections in the response:**
+- Failure analysis: $fold_failure_analysis
+- Task validity classification: $fold_task_validity
+
+When failure analysis is requested and you judge \`output_success: false\`, populate \`failure_point\` using the error taxonomy below:
+
+$taxonomy_block
+
+When task validity is requested, populate \`task_validity\` with the booleans \`is_ambiguous\` / \`is_invalid\` and, when each is true, a single one-line free-form reason in \`ambiguity_reason\` / \`invalid_reason\` (e.g., "Requested dates are in the past relative to the current date"). Leave the reason field empty when the corresponding flag is false.
+
+---
+
+**Outcome judgment:**
+\`output_success\` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them — a task can have high process score and still fail (right approach, wrong final answer), or have lower process score and still succeed.
+
+Apply these rules when making the outcome verdict:
+
+1. **Judge the delivered result, not the route.** If the final answer satisfies the user's requested deliverable and the trajectory does not contradict it, mark success even if the agent used an inefficient path, clicked around after finding the answer, or used a slightly different but reasonable search control. Do not fail solely because the process was imperfect.
+
+2. **Use the simulated trajectory as truth.** Do not use outside/current-world knowledge to override what the saved browser state shows. If search results, page text, URLs, or screenshots in the trajectory support a concrete answer, judge against that evidence even if the real web may differ. Do not introduce an alternate "correct" answer from your own knowledge unless that alternate answer is visible in the supplied trajectory evidence.
+
+3. **Treat attached screenshots as partial evidence.** Screenshots may be sampled from a longer trajectory. If the final answer is concrete and the action history shows the agent reached a relevant source (page, list, chart, search result, product page, map result), assume the answer may have been extracted from that source unless the attached screenshots or action history directly contradict it. Do not call a concrete answer fabricated merely because the exact supporting text is absent from the sampled screenshots.
+
+4. **Require the actual deliverable.** The final answer is the user-visible deliverable. For tasks asking to find, show, get, browse, list, summarize, report, compare, or provide values/content, the final answer must contain the requested content or a direct correct link to the requested document/page. Merely saying the agent "found", "extracted", "provided", "displayed", "retrieved", or "located" the information is not enough, even if the browser reached the right page. The selected trajectory evidence is only context; it cannot fill in missing values, lists, links, or summaries that are absent from the final answer. Only purely navigational tasks phrased as opening a page can succeed from navigation alone.
+
+5. **Report/document tasks need the report/document.** If the task asks to get, open, retrieve, download, or provide a report/document/page, the final answer must include a direct correct URL, the opened document/page itself must be the final browser state, or the final answer must include the requested document content. Merely naming the document title is not enough unless the user only asked for the title/name.
+
+6. **Do not invent equivalence.** If the final answer gives a URL, document, article id, product, entity, date, or value that differs from what the trajectory visibly reached, do not assume an unseen redirect, canonicalization, or equivalence. Treat the mismatch as a contradiction unless the supplied trajectory evidence itself shows they are equivalent.
+
+7. **Hard constraints still matter.** If the latest relevant attached screenshot, selected trajectory evidence, or action-history URL clearly shows that an explicit constraint was not met (for example wrong sort/filter/date/class, permanently closed location reported as active, visible product/list item outside the requested color/size/status, wrong URL/article id, or wrong entity), that is a critical issue.
+
+8. **Dynamic sources can disagree.** Search results, charts, rankings, prices, and listings can differ across pages or update times in the saved trajectory. If the final answer is concrete and matches at least one relevant trajectory source, do not fail solely because another relevant source in the trajectory shows a different dynamic value, unless the final answer clearly used the contradicted source or the task required that exact source.
+
+9. **Shopping/search constraints may be source-level.** For product-search tasks, a search query, filter, result page, or product title/snippet can satisfy a descriptive constraint such as "hypoallergenic" when the final answer provides concrete candidate products and no supplied evidence directly contradicts the constraint. Do not fail solely because a later product page does not repeat every search constraint as a formal specification.
+
+10. **Separate nitpicks from critical issues.** Minor wording, harmless method choices, unsupported extra details that are not part of the requested deliverable, caveats about imperfect verification, or small presentation differences should not flip a successful answer to failure when the final answer still provides concrete requested content and the trajectory does not directly contradict it. Wrong requested entities, unsupported fabricated requested facts, active-vs-closed mistakes, wrong sort/filter results, wrong dates/classes, and missing requested content are critical.
+
+Use this decision order:
+
+- First ask whether the final answer contains the requested user-facing artifact: values, list items, summary, comparison, report link, product candidates, or page/document URL. If it only narrates that the artifact was found/extracted/provided, mark failure.
+- Then check for direct contradictions in the supplied trajectory evidence. Contradictions must come from the provided action history, URLs, screenshots, or final answer, not from outside knowledge.
+- If the final answer is concrete, the trajectory reached a relevant source, and the supplied evidence does not directly contradict the final answer, mark success.
+
+**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.
+
+---
+
+**Output Format:**
+
+Output one JSON object:
+
+{{
+  "outcome": {{
+    "primary_intent": "<one-sentence restatement of what the task was asking for>",
+    "reasoning": "<your reasoning for the success / failure verdict>",
+    "output_success": true,
+    "findings": [
+      {{
+        "category": "agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other",
+        "severity": "info|warning|blocking",
+        "description": "...",
+        "suggestedAction": "...",
+        "relatedSteps": [3, 4]
+      }}
+    ]
+  }},
+  "failure_point": {{
+    "step_index": 17,
+    "error_code": "1.4",
+    "error_category": "Selection",
+    "description": "<one-line description of what went wrong at this step>"
+  }},
+  "task_validity": {{
+    "is_ambiguous": false,
+    "ambiguity_reason": "",
+    "is_invalid": false,
+    "invalid_reason": ""
+  }}
+}}
+
+- Omit \`failure_point\` when \`output_success\` is true or failure analysis was not requested.
+- Omit \`task_validity\` when task-validity classification was not requested.
+
+DO NOT OUTPUT ANYTHING OTHER THAN JSON.
+`;
+//# sourceMappingURL=fusedOutcome.js.map

package/dist/esm/lib/v3/verifier/prompts/fusedOutcome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fusedOutcome.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/fusedOutcome.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2GnC,CAAC","sourcesContent":["/**\n * Fused outcome prompt — Approach A's combined Step 8 + optional folded\n * 9a/10 call. Consumes a pre-scored rubric (computed deterministically from\n * Approach A's per-criterion analyses) and emits the outcome verdict,\n * findings, and optionally the first point of failure + task validity.\n *\n * Variables:\n *   - task_definition          — instruction string\n *   - init_url_context         — \"Starting URL: ...\" or empty\n *   - action_history           — compact textual action history\n *   - outcome_evidence_summary — selected text evidence snippets from the\n *                                trajectory, ordered by step\n *   - agent_predicted_output   — agent's final answer / message\n *   - rubric_summary           — pre-scored rubric: per-criterion earned/max\n *                                + the justifications from per-criterion calls\n *   - taxonomy_block           — error taxonomy text (only when\n *                                fold_failure_analysis is true; \"\" otherwise)\n *   - fold_failure_analysis    — \"true\" / \"false\"\n *   - fold_task_validity       — \"true\" / \"false\"\n *   - final_state_block        — always-attached final URL + ariaTree of the\n *                                last step probe and finalObservation; this\n *                                bypasses the keyword-based excerpt selection\n *                                used by outcome_evidence_summary so the\n *                                judge always has the closing page content.\n */\nexport const FUSED_OUTCOME_PROMPT = `Task: $task_definition$init_url_context\n\n**Current Date:** $current_date\n\nYou are an expert evaluator of web-navigation agent trajectories. The rubric has already been scored per criterion (results below). Your job is to produce the overall outcome verdict.\n\nUse the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible — classify as task_validity.is_invalid with code 8.1).\n\n**Action History:**\n$action_history\n\n**Selected Trajectory Evidence:**\n$outcome_evidence_summary\n\n**Final trajectory state** (authoritative — page content and screenshot captured at the very end of the run; treat as ground truth for what the agent saw on its final page, even when no \\`extract\\`/\\`observe\\` step appears in the action history):\n$final_state_block\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Pre-Scored Rubric (per-criterion earned points + justifications):**\n$rubric_summary\n\n**Optional sections in the response:**\n- Failure analysis: $fold_failure_analysis\n- Task validity classification: $fold_task_validity\n\nWhen failure analysis is requested and you judge \\`output_success: false\\`, populate \\`failure_point\\` using the error taxonomy below:\n\n$taxonomy_block\n\nWhen task validity is requested, populate \\`task_validity\\` with the booleans \\`is_ambiguous\\` / \\`is_invalid\\` and, when each is true, a single one-line free-form reason in \\`ambiguity_reason\\` / \\`invalid_reason\\` (e.g., \"Requested dates are in the past relative to the current date\"). Leave the reason field empty when the corresponding flag is false.\n\n---\n\n**Outcome judgment:**\n\\`output_success\\` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them — a task can have high process score and still fail (right approach, wrong final answer), or have lower process score and still succeed.\n\nApply these rules when making the outcome verdict:\n\n1. **Judge the delivered result, not the route.** If the final answer satisfies the user's requested deliverable and the trajectory does not contradict it, mark success even if the agent used an inefficient path, clicked around after finding the answer, or used a slightly different but reasonable search control. Do not fail solely because the process was imperfect.\n\n2. **Use the simulated trajectory as truth.** Do not use outside/current-world knowledge to override what the saved browser state shows. If search results, page text, URLs, or screenshots in the trajectory support a concrete answer, judge against that evidence even if the real web may differ. Do not introduce an alternate \"correct\" answer from your own knowledge unless that alternate answer is visible in the supplied trajectory evidence.\n\n3. **Treat attached screenshots as partial evidence.** Screenshots may be sampled from a longer trajectory. If the final answer is concrete and the action history shows the agent reached a relevant source (page, list, chart, search result, product page, map result), assume the answer may have been extracted from that source unless the attached screenshots or action history directly contradict it. Do not call a concrete answer fabricated merely because the exact supporting text is absent from the sampled screenshots.\n\n4. **Require the actual deliverable.** The final answer is the user-visible deliverable. For tasks asking to find, show, get, browse, list, summarize, report, compare, or provide values/content, the final answer must contain the requested content or a direct correct link to the requested document/page. Merely saying the agent \"found\", \"extracted\", \"provided\", \"displayed\", \"retrieved\", or \"located\" the information is not enough, even if the browser reached the right page. The selected trajectory evidence is only context; it cannot fill in missing values, lists, links, or summaries that are absent from the final answer. Only purely navigational tasks phrased as opening a page can succeed from navigation alone.\n\n5. **Report/document tasks need the report/document.** If the task asks to get, open, retrieve, download, or provide a report/document/page, the final answer must include a direct correct URL, the opened document/page itself must be the final browser state, or the final answer must include the requested document content. Merely naming the document title is not enough unless the user only asked for the title/name.\n\n6. **Do not invent equivalence.** If the final answer gives a URL, document, article id, product, entity, date, or value that differs from what the trajectory visibly reached, do not assume an unseen redirect, canonicalization, or equivalence. Treat the mismatch as a contradiction unless the supplied trajectory evidence itself shows they are equivalent.\n\n7. **Hard constraints still matter.** If the latest relevant attached screenshot, selected trajectory evidence, or action-history URL clearly shows that an explicit constraint was not met (for example wrong sort/filter/date/class, permanently closed location reported as active, visible product/list item outside the requested color/size/status, wrong URL/article id, or wrong entity), that is a critical issue.\n\n8. **Dynamic sources can disagree.** Search results, charts, rankings, prices, and listings can differ across pages or update times in the saved trajectory. If the final answer is concrete and matches at least one relevant trajectory source, do not fail solely because another relevant source in the trajectory shows a different dynamic value, unless the final answer clearly used the contradicted source or the task required that exact source.\n\n9. **Shopping/search constraints may be source-level.** For product-search tasks, a search query, filter, result page, or product title/snippet can satisfy a descriptive constraint such as \"hypoallergenic\" when the final answer provides concrete candidate products and no supplied evidence directly contradicts the constraint. Do not fail solely because a later product page does not repeat every search constraint as a formal specification.\n\n10. **Separate nitpicks from critical issues.** Minor wording, harmless method choices, unsupported extra details that are not part of the requested deliverable, caveats about imperfect verification, or small presentation differences should not flip a successful answer to failure when the final answer still provides concrete requested content and the trajectory does not directly contradict it. Wrong requested entities, unsupported fabricated requested facts, active-vs-closed mistakes, wrong sort/filter results, wrong dates/classes, and missing requested content are critical.\n\nUse this decision order:\n\n- First ask whether the final answer contains the requested user-facing artifact: values, list items, summary, comparison, report link, product candidates, or page/document URL. If it only narrates that the artifact was found/extracted/provided, mark failure.\n- Then check for direct contradictions in the supplied trajectory evidence. Contradictions must come from the provided action history, URLs, screenshots, or final answer, not from outside knowledge.\n- If the final answer is concrete, the trajectory reached a relevant source, and the supplied evidence does not directly contradict the final answer, mark success.\n\n**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.\n\n---\n\n**Output Format:**\n\nOutput one JSON object:\n\n{{\n  \"outcome\": {{\n    \"primary_intent\": \"<one-sentence restatement of what the task was asking for>\",\n    \"reasoning\": \"<your reasoning for the success / failure verdict>\",\n    \"output_success\": true,\n    \"findings\": [\n      {{\n        \"category\": \"agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other\",\n        \"severity\": \"info|warning|blocking\",\n        \"description\": \"...\",\n        \"suggestedAction\": \"...\",\n        \"relatedSteps\": [3, 4]\n      }}\n    ]\n  }},\n  \"failure_point\": {{\n    \"step_index\": 17,\n    \"error_code\": \"1.4\",\n    \"error_category\": \"Selection\",\n    \"description\": \"<one-line description of what went wrong at this step>\"\n  }},\n  \"task_validity\": {{\n    \"is_ambiguous\": false,\n    \"ambiguity_reason\": \"\",\n    \"is_invalid\": false,\n    \"invalid_reason\": \"\"\n  }}\n}}\n\n- Omit \\`failure_point\\` when \\`output_success\\` is true or failure analysis was not requested.\n- Omit \\`task_validity\\` when task-validity classification was not requested.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n`;\n"]}

package/dist/esm/lib/v3/verifier/prompts/index.d.ts

@@ -0,0 +1,8 @@
+export { RUBRIC_GENERATION_PROMPT } from "./rubricGeneration.js";
+export { FIRST_POINT_OF_FAILURE_PROMPT, parseFailureStepNumbers, } from "./firstPointOfFailure.js";
+export { TASK_VALIDITY_PROMPT } from "./taskValidity.js";
+export { MM_BATCHED_RELEVANCE_PROMPT } from "./batchedRelevance.js";
+export { MM_PER_CRITERION_SCORE_PROMPT } from "./perCriterionScore.js";
+export { FUSED_JUDGMENT_PROMPT } from "./fusedJudgment.js";
+export { FUSED_OUTCOME_PROMPT } from "./fusedOutcome.js";
+export { renderPrompt, buildInitUrlContext } from "./render.js";