@browserbasehq/orca 3.5.0-vertex-test → 3.6.0-gemini-cua

package/dist/cjs/lib/v3/verifier/prompts/fusedOutcome.js

@@ -0,0 +1,137 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FUSED_OUTCOME_PROMPT = void 0;
+/**
+ * Fused outcome prompt — Approach A's combined Step 8 + optional folded
+ * 9a/10 call. Consumes a pre-scored rubric (computed deterministically from
+ * Approach A's per-criterion analyses) and emits the outcome verdict,
+ * findings, and optionally the first point of failure + task validity.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - outcome_evidence_summary — selected text evidence snippets from the
+ *                                trajectory, ordered by step
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - rubric_summary           — pre-scored rubric: per-criterion earned/max
+ *                                + the justifications from per-criterion calls
+ *   - taxonomy_block           — error taxonomy text (only when
+ *                                fold_failure_analysis is true; "" otherwise)
+ *   - fold_failure_analysis    — "true" / "false"
+ *   - fold_task_validity       — "true" / "false"
+ *   - final_state_block        — always-attached final URL + ariaTree of the
+ *                                last step probe and finalObservation; this
+ *                                bypasses the keyword-based excerpt selection
+ *                                used by outcome_evidence_summary so the
+ *                                judge always has the closing page content.
+ */
+exports.FUSED_OUTCOME_PROMPT = `Task: $task_definition$init_url_context
+
+**Current Date:** $current_date
+
+You are an expert evaluator of web-navigation agent trajectories. The rubric has already been scored per criterion (results below). Your job is to produce the overall outcome verdict.
+
+Use the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible — classify as task_validity.is_invalid with code 8.1).
+
+**Action History:**
+$action_history
+
+**Selected Trajectory Evidence:**
+$outcome_evidence_summary
+
+**Final trajectory state** (authoritative — page content and screenshot captured at the very end of the run; treat as ground truth for what the agent saw on its final page, even when no \`extract\`/\`observe\` step appears in the action history):
+$final_state_block
+
+**Agent's Predicted Output (Final Answer):**
+$agent_predicted_output
+
+**Pre-Scored Rubric (per-criterion earned points + justifications):**
+$rubric_summary
+
+**Optional sections in the response:**
+- Failure analysis: $fold_failure_analysis
+- Task validity classification: $fold_task_validity
+
+When failure analysis is requested and you judge \`output_success: false\`, populate \`failure_point\` using the error taxonomy below:
+
+$taxonomy_block
+
+When task validity is requested, populate \`task_validity\` with the booleans \`is_ambiguous\` / \`is_invalid\` and, when each is true, a single one-line free-form reason in \`ambiguity_reason\` / \`invalid_reason\` (e.g., "Requested dates are in the past relative to the current date"). Leave the reason field empty when the corresponding flag is false.
+
+---
+
+**Outcome judgment:**
+\`output_success\` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them — a task can have high process score and still fail (right approach, wrong final answer), or have lower process score and still succeed.
+
+Apply these rules when making the outcome verdict:
+
+1. **Judge the delivered result, not the route.** If the final answer satisfies the user's requested deliverable and the trajectory does not contradict it, mark success even if the agent used an inefficient path, clicked around after finding the answer, or used a slightly different but reasonable search control. Do not fail solely because the process was imperfect.
+
+2. **Use the simulated trajectory as truth.** Do not use outside/current-world knowledge to override what the saved browser state shows. If search results, page text, URLs, or screenshots in the trajectory support a concrete answer, judge against that evidence even if the real web may differ. Do not introduce an alternate "correct" answer from your own knowledge unless that alternate answer is visible in the supplied trajectory evidence.
+
+3. **Treat attached screenshots as partial evidence.** Screenshots may be sampled from a longer trajectory. If the final answer is concrete and the action history shows the agent reached a relevant source (page, list, chart, search result, product page, map result), assume the answer may have been extracted from that source unless the attached screenshots or action history directly contradict it. Do not call a concrete answer fabricated merely because the exact supporting text is absent from the sampled screenshots.
+
+4. **Require the actual deliverable.** The final answer is the user-visible deliverable. For tasks asking to find, show, get, browse, list, summarize, report, compare, or provide values/content, the final answer must contain the requested content or a direct correct link to the requested document/page. Merely saying the agent "found", "extracted", "provided", "displayed", "retrieved", or "located" the information is not enough, even if the browser reached the right page. The selected trajectory evidence is only context; it cannot fill in missing values, lists, links, or summaries that are absent from the final answer. Only purely navigational tasks phrased as opening a page can succeed from navigation alone.
+
+5. **Report/document tasks need the report/document.** If the task asks to get, open, retrieve, download, or provide a report/document/page, the final answer must include a direct correct URL, the opened document/page itself must be the final browser state, or the final answer must include the requested document content. Merely naming the document title is not enough unless the user only asked for the title/name.
+
+6. **Do not invent equivalence.** If the final answer gives a URL, document, article id, product, entity, date, or value that differs from what the trajectory visibly reached, do not assume an unseen redirect, canonicalization, or equivalence. Treat the mismatch as a contradiction unless the supplied trajectory evidence itself shows they are equivalent.
+
+7. **Hard constraints still matter.** If the latest relevant attached screenshot, selected trajectory evidence, or action-history URL clearly shows that an explicit constraint was not met (for example wrong sort/filter/date/class, permanently closed location reported as active, visible product/list item outside the requested color/size/status, wrong URL/article id, or wrong entity), that is a critical issue.
+
+8. **Dynamic sources can disagree.** Search results, charts, rankings, prices, and listings can differ across pages or update times in the saved trajectory. If the final answer is concrete and matches at least one relevant trajectory source, do not fail solely because another relevant source in the trajectory shows a different dynamic value, unless the final answer clearly used the contradicted source or the task required that exact source.
+
+9. **Shopping/search constraints may be source-level.** For product-search tasks, a search query, filter, result page, or product title/snippet can satisfy a descriptive constraint such as "hypoallergenic" when the final answer provides concrete candidate products and no supplied evidence directly contradicts the constraint. Do not fail solely because a later product page does not repeat every search constraint as a formal specification.
+
+10. **Separate nitpicks from critical issues.** Minor wording, harmless method choices, unsupported extra details that are not part of the requested deliverable, caveats about imperfect verification, or small presentation differences should not flip a successful answer to failure when the final answer still provides concrete requested content and the trajectory does not directly contradict it. Wrong requested entities, unsupported fabricated requested facts, active-vs-closed mistakes, wrong sort/filter results, wrong dates/classes, and missing requested content are critical.
+
+Use this decision order:
+
+- First ask whether the final answer contains the requested user-facing artifact: values, list items, summary, comparison, report link, product candidates, or page/document URL. If it only narrates that the artifact was found/extracted/provided, mark failure.
+- Then check for direct contradictions in the supplied trajectory evidence. Contradictions must come from the provided action history, URLs, screenshots, or final answer, not from outside knowledge.
+- If the final answer is concrete, the trajectory reached a relevant source, and the supplied evidence does not directly contradict the final answer, mark success.
+
+**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.
+
+---
+
+**Output Format:**
+
+Output one JSON object:
+
+{{
+  "outcome": {{
+    "primary_intent": "<one-sentence restatement of what the task was asking for>",
+    "reasoning": "<your reasoning for the success / failure verdict>",
+    "output_success": true,
+    "findings": [
+      {{
+        "category": "agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other",
+        "severity": "info|warning|blocking",
+        "description": "...",
+        "suggestedAction": "...",
+        "relatedSteps": [3, 4]
+      }}
+    ]
+  }},
+  "failure_point": {{
+    "step_index": 17,
+    "error_code": "1.4",
+    "error_category": "Selection",
+    "description": "<one-line description of what went wrong at this step>"
+  }},
+  "task_validity": {{
+    "is_ambiguous": false,
+    "ambiguity_reason": "",
+    "is_invalid": false,
+    "invalid_reason": ""
+  }}
+}}
+
+- Omit \`failure_point\` when \`output_success\` is true or failure analysis was not requested.
+- Omit \`task_validity\` when task-validity classification was not requested.
+
+DO NOT OUTPUT ANYTHING OTHER THAN JSON.
+`;
+//# sourceMappingURL=fusedOutcome.js.map

package/dist/cjs/lib/v3/verifier/prompts/fusedOutcome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fusedOutcome.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/fusedOutcome.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACU,QAAA,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2GnC,CAAC","sourcesContent":["/**\n * Fused outcome prompt — Approach A's combined Step 8 + optional folded\n * 9a/10 call. Consumes a pre-scored rubric (computed deterministically from\n * Approach A's per-criterion analyses) and emits the outcome verdict,\n * findings, and optionally the first point of failure + task validity.\n *\n * Variables:\n *   - task_definition          — instruction string\n *   - init_url_context         — \"Starting URL: ...\" or empty\n *   - action_history           — compact textual action history\n *   - outcome_evidence_summary — selected text evidence snippets from the\n *                                trajectory, ordered by step\n *   - agent_predicted_output   — agent's final answer / message\n *   - rubric_summary           — pre-scored rubric: per-criterion earned/max\n *                                + the justifications from per-criterion calls\n *   - taxonomy_block           — error taxonomy text (only when\n *                                fold_failure_analysis is true; \"\" otherwise)\n *   - fold_failure_analysis    — \"true\" / \"false\"\n *   - fold_task_validity       — \"true\" / \"false\"\n *   - final_state_block        — always-attached final URL + ariaTree of the\n *                                last step probe and finalObservation; this\n *                                bypasses the keyword-based excerpt selection\n *                                used by outcome_evidence_summary so the\n *                                judge always has the closing page content.\n */\nexport const FUSED_OUTCOME_PROMPT = `Task: $task_definition$init_url_context\n\n**Current Date:** $current_date\n\nYou are an expert evaluator of web-navigation agent trajectories. The rubric has already been scored per criterion (results below). Your job is to produce the overall outcome verdict.\n\nUse the current date above to assess time-sensitive constraints in the task (e.g., a task referencing dates in the past relative to the current date is impossible — classify as task_validity.is_invalid with code 8.1).\n\n**Action History:**\n$action_history\n\n**Selected Trajectory Evidence:**\n$outcome_evidence_summary\n\n**Final trajectory state** (authoritative — page content and screenshot captured at the very end of the run; treat as ground truth for what the agent saw on its final page, even when no \\`extract\\`/\\`observe\\` step appears in the action history):\n$final_state_block\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Pre-Scored Rubric (per-criterion earned points + justifications):**\n$rubric_summary\n\n**Optional sections in the response:**\n- Failure analysis: $fold_failure_analysis\n- Task validity classification: $fold_task_validity\n\nWhen failure analysis is requested and you judge \\`output_success: false\\`, populate \\`failure_point\\` using the error taxonomy below:\n\n$taxonomy_block\n\nWhen task validity is requested, populate \\`task_validity\\` with the booleans \\`is_ambiguous\\` / \\`is_invalid\\` and, when each is true, a single one-line free-form reason in \\`ambiguity_reason\\` / \\`invalid_reason\\` (e.g., \"Requested dates are in the past relative to the current date\"). Leave the reason field empty when the corresponding flag is false.\n\n---\n\n**Outcome judgment:**\n\\`output_success\\` is your independent binary verdict on whether the agent completed the task. It is informed by the per-criterion scores but is not a function of them — a task can have high process score and still fail (right approach, wrong final answer), or have lower process score and still succeed.\n\nApply these rules when making the outcome verdict:\n\n1. **Judge the delivered result, not the route.** If the final answer satisfies the user's requested deliverable and the trajectory does not contradict it, mark success even if the agent used an inefficient path, clicked around after finding the answer, or used a slightly different but reasonable search control. Do not fail solely because the process was imperfect.\n\n2. **Use the simulated trajectory as truth.** Do not use outside/current-world knowledge to override what the saved browser state shows. If search results, page text, URLs, or screenshots in the trajectory support a concrete answer, judge against that evidence even if the real web may differ. Do not introduce an alternate \"correct\" answer from your own knowledge unless that alternate answer is visible in the supplied trajectory evidence.\n\n3. **Treat attached screenshots as partial evidence.** Screenshots may be sampled from a longer trajectory. If the final answer is concrete and the action history shows the agent reached a relevant source (page, list, chart, search result, product page, map result), assume the answer may have been extracted from that source unless the attached screenshots or action history directly contradict it. Do not call a concrete answer fabricated merely because the exact supporting text is absent from the sampled screenshots.\n\n4. **Require the actual deliverable.** The final answer is the user-visible deliverable. For tasks asking to find, show, get, browse, list, summarize, report, compare, or provide values/content, the final answer must contain the requested content or a direct correct link to the requested document/page. Merely saying the agent \"found\", \"extracted\", \"provided\", \"displayed\", \"retrieved\", or \"located\" the information is not enough, even if the browser reached the right page. The selected trajectory evidence is only context; it cannot fill in missing values, lists, links, or summaries that are absent from the final answer. Only purely navigational tasks phrased as opening a page can succeed from navigation alone.\n\n5. **Report/document tasks need the report/document.** If the task asks to get, open, retrieve, download, or provide a report/document/page, the final answer must include a direct correct URL, the opened document/page itself must be the final browser state, or the final answer must include the requested document content. Merely naming the document title is not enough unless the user only asked for the title/name.\n\n6. **Do not invent equivalence.** If the final answer gives a URL, document, article id, product, entity, date, or value that differs from what the trajectory visibly reached, do not assume an unseen redirect, canonicalization, or equivalence. Treat the mismatch as a contradiction unless the supplied trajectory evidence itself shows they are equivalent.\n\n7. **Hard constraints still matter.** If the latest relevant attached screenshot, selected trajectory evidence, or action-history URL clearly shows that an explicit constraint was not met (for example wrong sort/filter/date/class, permanently closed location reported as active, visible product/list item outside the requested color/size/status, wrong URL/article id, or wrong entity), that is a critical issue.\n\n8. **Dynamic sources can disagree.** Search results, charts, rankings, prices, and listings can differ across pages or update times in the saved trajectory. If the final answer is concrete and matches at least one relevant trajectory source, do not fail solely because another relevant source in the trajectory shows a different dynamic value, unless the final answer clearly used the contradicted source or the task required that exact source.\n\n9. **Shopping/search constraints may be source-level.** For product-search tasks, a search query, filter, result page, or product title/snippet can satisfy a descriptive constraint such as \"hypoallergenic\" when the final answer provides concrete candidate products and no supplied evidence directly contradicts the constraint. Do not fail solely because a later product page does not repeat every search constraint as a formal specification.\n\n10. **Separate nitpicks from critical issues.** Minor wording, harmless method choices, unsupported extra details that are not part of the requested deliverable, caveats about imperfect verification, or small presentation differences should not flip a successful answer to failure when the final answer still provides concrete requested content and the trajectory does not directly contradict it. Wrong requested entities, unsupported fabricated requested facts, active-vs-closed mistakes, wrong sort/filter results, wrong dates/classes, and missing requested content are critical.\n\nUse this decision order:\n\n- First ask whether the final answer contains the requested user-facing artifact: values, list items, summary, comparison, report link, product candidates, or page/document URL. If it only narrates that the artifact was found/extracted/provided, mark failure.\n- Then check for direct contradictions in the supplied trajectory evidence. Contradictions must come from the provided action history, URLs, screenshots, or final answer, not from outside knowledge.\n- If the final answer is concrete, the trajectory reached a relevant source, and the supplied evidence does not directly contradict the final answer, mark success.\n\n**Findings:** Surface actionable patterns: failed tool usage, agent-strategy issues, rubric quality problems, capture gaps. Each finding gets a category, severity, description, and (optional) related steps + suggested action. Keep findings sparse and load-bearing.\n\n---\n\n**Output Format:**\n\nOutput one JSON object:\n\n{{\n  \"outcome\": {{\n    \"primary_intent\": \"<one-sentence restatement of what the task was asking for>\",\n    \"reasoning\": \"<your reasoning for the success / failure verdict>\",\n    \"output_success\": true,\n    \"findings\": [\n      {{\n        \"category\": \"agent_tool_usage|agent_strategy|rubric_quality|trajectory_capture|task_specification|verifier_uncertainty|other\",\n        \"severity\": \"info|warning|blocking\",\n        \"description\": \"...\",\n        \"suggestedAction\": \"...\",\n        \"relatedSteps\": [3, 4]\n      }}\n    ]\n  }},\n  \"failure_point\": {{\n    \"step_index\": 17,\n    \"error_code\": \"1.4\",\n    \"error_category\": \"Selection\",\n    \"description\": \"<one-line description of what went wrong at this step>\"\n  }},\n  \"task_validity\": {{\n    \"is_ambiguous\": false,\n    \"ambiguity_reason\": \"\",\n    \"is_invalid\": false,\n    \"invalid_reason\": \"\"\n  }}\n}}\n\n- Omit \\`failure_point\\` when \\`output_success\\` is true or failure analysis was not requested.\n- Omit \\`task_validity\\` when task-validity classification was not requested.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n`;\n"]}

package/dist/cjs/lib/v3/verifier/prompts/index.d.ts

@@ -0,0 +1,8 @@
+export { RUBRIC_GENERATION_PROMPT } from "./rubricGeneration.js";
+export { FIRST_POINT_OF_FAILURE_PROMPT, parseFailureStepNumbers, } from "./firstPointOfFailure.js";
+export { TASK_VALIDITY_PROMPT } from "./taskValidity.js";
+export { MM_BATCHED_RELEVANCE_PROMPT } from "./batchedRelevance.js";
+export { MM_PER_CRITERION_SCORE_PROMPT } from "./perCriterionScore.js";
+export { FUSED_JUDGMENT_PROMPT } from "./fusedJudgment.js";
+export { FUSED_OUTCOME_PROMPT } from "./fusedOutcome.js";
+export { renderPrompt, buildInitUrlContext } from "./render.js";

package/dist/cjs/lib/v3/verifier/prompts/index.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildInitUrlContext = exports.renderPrompt = exports.FUSED_OUTCOME_PROMPT = exports.FUSED_JUDGMENT_PROMPT = exports.MM_PER_CRITERION_SCORE_PROMPT = exports.MM_BATCHED_RELEVANCE_PROMPT = exports.TASK_VALIDITY_PROMPT = exports.parseFailureStepNumbers = exports.FIRST_POINT_OF_FAILURE_PROMPT = exports.RUBRIC_GENERATION_PROMPT = void 0;
+var rubricGeneration_js_1 = require("./rubricGeneration.js");
+Object.defineProperty(exports, "RUBRIC_GENERATION_PROMPT", { enumerable: true, get: function () { return rubricGeneration_js_1.RUBRIC_GENERATION_PROMPT; } });
+var firstPointOfFailure_js_1 = require("./firstPointOfFailure.js");
+Object.defineProperty(exports, "FIRST_POINT_OF_FAILURE_PROMPT", { enumerable: true, get: function () { return firstPointOfFailure_js_1.FIRST_POINT_OF_FAILURE_PROMPT; } });
+Object.defineProperty(exports, "parseFailureStepNumbers", { enumerable: true, get: function () { return firstPointOfFailure_js_1.parseFailureStepNumbers; } });
+var taskValidity_js_1 = require("./taskValidity.js");
+Object.defineProperty(exports, "TASK_VALIDITY_PROMPT", { enumerable: true, get: function () { return taskValidity_js_1.TASK_VALIDITY_PROMPT; } });
+var batchedRelevance_js_1 = require("./batchedRelevance.js");
+Object.defineProperty(exports, "MM_BATCHED_RELEVANCE_PROMPT", { enumerable: true, get: function () { return batchedRelevance_js_1.MM_BATCHED_RELEVANCE_PROMPT; } });
+var perCriterionScore_js_1 = require("./perCriterionScore.js");
+Object.defineProperty(exports, "MM_PER_CRITERION_SCORE_PROMPT", { enumerable: true, get: function () { return perCriterionScore_js_1.MM_PER_CRITERION_SCORE_PROMPT; } });
+var fusedJudgment_js_1 = require("./fusedJudgment.js");
+Object.defineProperty(exports, "FUSED_JUDGMENT_PROMPT", { enumerable: true, get: function () { return fusedJudgment_js_1.FUSED_JUDGMENT_PROMPT; } });
+var fusedOutcome_js_1 = require("./fusedOutcome.js");
+Object.defineProperty(exports, "FUSED_OUTCOME_PROMPT", { enumerable: true, get: function () { return fusedOutcome_js_1.FUSED_OUTCOME_PROMPT; } });
+var render_js_1 = require("./render.js");
+Object.defineProperty(exports, "renderPrompt", { enumerable: true, get: function () { return render_js_1.renderPrompt; } });
+Object.defineProperty(exports, "buildInitUrlContext", { enumerable: true, get: function () { return render_js_1.buildInitUrlContext; } });
+//# sourceMappingURL=index.js.map

package/dist/cjs/lib/v3/verifier/prompts/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/index.ts"],"names":[],"mappings":";;;AAAA,6DAAiE;AAAxD,+HAAA,wBAAwB,OAAA;AACjC,mEAGkC;AAFhC,uIAAA,6BAA6B,OAAA;AAC7B,iIAAA,uBAAuB,OAAA;AAEzB,qDAAyD;AAAhD,uHAAA,oBAAoB,OAAA;AAC7B,6DAAoE;AAA3D,kIAAA,2BAA2B,OAAA;AACpC,+DAAuE;AAA9D,qIAAA,6BAA6B,OAAA;AACtC,uDAA2D;AAAlD,yHAAA,qBAAqB,OAAA;AAC9B,qDAAyD;AAAhD,uHAAA,oBAAoB,OAAA;AAC7B,yCAAgE;AAAvD,yGAAA,YAAY,OAAA;AAAE,gHAAA,mBAAmB,OAAA","sourcesContent":["export { RUBRIC_GENERATION_PROMPT } from \"./rubricGeneration.js\";\nexport {\n  FIRST_POINT_OF_FAILURE_PROMPT,\n  parseFailureStepNumbers,\n} from \"./firstPointOfFailure.js\";\nexport { TASK_VALIDITY_PROMPT } from \"./taskValidity.js\";\nexport { MM_BATCHED_RELEVANCE_PROMPT } from \"./batchedRelevance.js\";\nexport { MM_PER_CRITERION_SCORE_PROMPT } from \"./perCriterionScore.js\";\nexport { FUSED_JUDGMENT_PROMPT } from \"./fusedJudgment.js\";\nexport { FUSED_OUTCOME_PROMPT } from \"./fusedOutcome.js\";\nexport { renderPrompt, buildInitUrlContext } from \"./render.js\";\n"]}

package/dist/cjs/lib/v3/verifier/prompts/perCriterionScore.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Per-criterion scoring prompt — Approach A's replacement for Steps 4 + 6.
+ *
+ * One call grades ONE criterion against its top-K evidence (images + text),
+ * returning both an analysis and an earned-points score. With N criteria
+ * this is N calls (parallelizable). Drops the Step-6 whole-rubric rescore
+ * entirely since `processScore` becomes Σ earned_points / Σ max_points.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - criterion_idx            — index in the rubric
+ *   - criterion_name           — the criterion text
+ *   - criterion_description    — description of what's being measured
+ *   - criterion_max_points     — max points for this criterion
+ *   - criterion_condition      — optional "Condition: ..." line, or empty
+ *   - evidence_manifest        — labelled list of the top-K evidence points
+ *                                attached below (image refs + ariaTree
+ *                                snippets in chronological order).
+ */
+export declare const MM_PER_CRITERION_SCORE_PROMPT = "Task: $task_definition$init_url_context\n\nYou are scoring ONE rubric criterion against the relevant evidence from an agent's trajectory.\n\n**Action History:**\n$action_history\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Criterion #$criterion_idx \u2014 \"$criterion_name\"**\n- Description: $criterion_description\n- Max points: $criterion_max_points\n$criterion_condition\n\n**Evidence (top-K most relevant):**\n$evidence_manifest\n\nEach evidence reference points to an image attached below or to a text snippet shown inline above. Screenshots are listed in chronological order; when two screenshots show the same element, **the LATER screenshot reflects the final state and takes precedence**.\n\n**Core Evaluation Principles:**\n\n1. **Best Effort.** Reward effort within constraints the agent cannot control.\n2. **Uncontrollable blockers** (CAPTCHA, login walls, sold out, site down, entity nonexistence) \u2192 award full credit when screenshots confirm the blocker.\n3. **Controllable failures** (wrong selections when correct options exist, hallucinations, premature giveup) \u2192 penalize per severity.\n4. **Hard constraints in the task** (specific qualifications, attributes, filters) \u2192 only award full credit when the constraint is actually met in the final evidence, not just searched for.\n5. **Conditional criteria.** If this criterion has a Condition and the condition is NOT met, set `earned_points` to `criterion_max_points` (criterion is not applicable) and note this in the justification.\n6. **Nitpick vs critical error scoring:**\n   - Only nitpicks \u2192 75\u2013100% of max\n   - Correct approach, wrong final answer \u2192 40\u201380%\n   - Critical error \u2192 penalize per severity\n\n**Output Format:**\n\nOutput one JSON object:\n\n{{\n  \"criterion_idx\": $criterion_idx,\n  \"applicable_evidence\": \"Which evidence supports the score; cite by 'Screenshot N \u2014 step=K' or aria-tree step number. If no evidence is applicable, state that.\",\n  \"justification\": \"How the evidence supports the score. If using condition-not-met rule, explain.\",\n  \"earned_points\": <number in [0, $criterion_max_points]>,\n  \"evidence_sufficient\": true,\n  \"condition_met\": null\n}}\n\n- `earned_points` must be in [0, $criterion_max_points].\n- `evidence_sufficient` = false when the available evidence is genuinely too sparse to grade fairly. The verifier will mark the criterion as evidence-insufficient.\n- `condition_met` is a boolean when the criterion has a Condition; otherwise null.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n";

package/dist/cjs/lib/v3/verifier/prompts/perCriterionScore.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MM_PER_CRITERION_SCORE_PROMPT = void 0;
+/**
+ * Per-criterion scoring prompt — Approach A's replacement for Steps 4 + 6.
+ *
+ * One call grades ONE criterion against its top-K evidence (images + text),
+ * returning both an analysis and an earned-points score. With N criteria
+ * this is N calls (parallelizable). Drops the Step-6 whole-rubric rescore
+ * entirely since `processScore` becomes Σ earned_points / Σ max_points.
+ *
+ * Variables:
+ *   - task_definition          — instruction string
+ *   - init_url_context         — "Starting URL: ..." or empty
+ *   - action_history           — compact textual action history
+ *   - agent_predicted_output   — agent's final answer / message
+ *   - criterion_idx            — index in the rubric
+ *   - criterion_name           — the criterion text
+ *   - criterion_description    — description of what's being measured
+ *   - criterion_max_points     — max points for this criterion
+ *   - criterion_condition      — optional "Condition: ..." line, or empty
+ *   - evidence_manifest        — labelled list of the top-K evidence points
+ *                                attached below (image refs + ariaTree
+ *                                snippets in chronological order).
+ */
+exports.MM_PER_CRITERION_SCORE_PROMPT = `Task: $task_definition$init_url_context
+
+You are scoring ONE rubric criterion against the relevant evidence from an agent's trajectory.
+
+**Action History:**
+$action_history
+
+**Agent's Predicted Output (Final Answer):**
+$agent_predicted_output
+
+**Criterion #$criterion_idx — "$criterion_name"**
+- Description: $criterion_description
+- Max points: $criterion_max_points
+$criterion_condition
+
+**Evidence (top-K most relevant):**
+$evidence_manifest
+
+Each evidence reference points to an image attached below or to a text snippet shown inline above. Screenshots are listed in chronological order; when two screenshots show the same element, **the LATER screenshot reflects the final state and takes precedence**.
+
+**Core Evaluation Principles:**
+
+1. **Best Effort.** Reward effort within constraints the agent cannot control.
+2. **Uncontrollable blockers** (CAPTCHA, login walls, sold out, site down, entity nonexistence) → award full credit when screenshots confirm the blocker.
+3. **Controllable failures** (wrong selections when correct options exist, hallucinations, premature giveup) → penalize per severity.
+4. **Hard constraints in the task** (specific qualifications, attributes, filters) → only award full credit when the constraint is actually met in the final evidence, not just searched for.
+5. **Conditional criteria.** If this criterion has a Condition and the condition is NOT met, set \`earned_points\` to \`criterion_max_points\` (criterion is not applicable) and note this in the justification.
+6. **Nitpick vs critical error scoring:**
+   - Only nitpicks → 75–100% of max
+   - Correct approach, wrong final answer → 40–80%
+   - Critical error → penalize per severity
+
+**Output Format:**
+
+Output one JSON object:
+
+{{
+  "criterion_idx": $criterion_idx,
+  "applicable_evidence": "Which evidence supports the score; cite by 'Screenshot N — step=K' or aria-tree step number. If no evidence is applicable, state that.",
+  "justification": "How the evidence supports the score. If using condition-not-met rule, explain.",
+  "earned_points": <number in [0, $criterion_max_points]>,
+  "evidence_sufficient": true,
+  "condition_met": null
+}}
+
+- \`earned_points\` must be in [0, $criterion_max_points].
+- \`evidence_sufficient\` = false when the available evidence is genuinely too sparse to grade fairly. The verifier will mark the criterion as evidence-insufficient.
+- \`condition_met\` is a boolean when the criterion has a Condition; otherwise null.
+
+DO NOT OUTPUT ANYTHING OTHER THAN JSON.
+`;
+//# sourceMappingURL=perCriterionScore.js.map

package/dist/cjs/lib/v3/verifier/prompts/perCriterionScore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"perCriterionScore.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/perCriterionScore.ts"],"names":[],"mappings":";;;AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACU,QAAA,6BAA6B,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkD5C,CAAC","sourcesContent":["/**\n * Per-criterion scoring prompt — Approach A's replacement for Steps 4 + 6.\n *\n * One call grades ONE criterion against its top-K evidence (images + text),\n * returning both an analysis and an earned-points score. With N criteria\n * this is N calls (parallelizable). Drops the Step-6 whole-rubric rescore\n * entirely since `processScore` becomes Σ earned_points / Σ max_points.\n *\n * Variables:\n *   - task_definition          — instruction string\n *   - init_url_context         — \"Starting URL: ...\" or empty\n *   - action_history           — compact textual action history\n *   - agent_predicted_output   — agent's final answer / message\n *   - criterion_idx            — index in the rubric\n *   - criterion_name           — the criterion text\n *   - criterion_description    — description of what's being measured\n *   - criterion_max_points     — max points for this criterion\n *   - criterion_condition      — optional \"Condition: ...\" line, or empty\n *   - evidence_manifest        — labelled list of the top-K evidence points\n *                                attached below (image refs + ariaTree\n *                                snippets in chronological order).\n */\nexport const MM_PER_CRITERION_SCORE_PROMPT = `Task: $task_definition$init_url_context\n\nYou are scoring ONE rubric criterion against the relevant evidence from an agent's trajectory.\n\n**Action History:**\n$action_history\n\n**Agent's Predicted Output (Final Answer):**\n$agent_predicted_output\n\n**Criterion #$criterion_idx — \"$criterion_name\"**\n- Description: $criterion_description\n- Max points: $criterion_max_points\n$criterion_condition\n\n**Evidence (top-K most relevant):**\n$evidence_manifest\n\nEach evidence reference points to an image attached below or to a text snippet shown inline above. Screenshots are listed in chronological order; when two screenshots show the same element, **the LATER screenshot reflects the final state and takes precedence**.\n\n**Core Evaluation Principles:**\n\n1. **Best Effort.** Reward effort within constraints the agent cannot control.\n2. **Uncontrollable blockers** (CAPTCHA, login walls, sold out, site down, entity nonexistence) → award full credit when screenshots confirm the blocker.\n3. **Controllable failures** (wrong selections when correct options exist, hallucinations, premature giveup) → penalize per severity.\n4. **Hard constraints in the task** (specific qualifications, attributes, filters) → only award full credit when the constraint is actually met in the final evidence, not just searched for.\n5. **Conditional criteria.** If this criterion has a Condition and the condition is NOT met, set \\`earned_points\\` to \\`criterion_max_points\\` (criterion is not applicable) and note this in the justification.\n6. **Nitpick vs critical error scoring:**\n   - Only nitpicks → 75–100% of max\n   - Correct approach, wrong final answer → 40–80%\n   - Critical error → penalize per severity\n\n**Output Format:**\n\nOutput one JSON object:\n\n{{\n  \"criterion_idx\": $criterion_idx,\n  \"applicable_evidence\": \"Which evidence supports the score; cite by 'Screenshot N — step=K' or aria-tree step number. If no evidence is applicable, state that.\",\n  \"justification\": \"How the evidence supports the score. If using condition-not-met rule, explain.\",\n  \"earned_points\": <number in [0, $criterion_max_points]>,\n  \"evidence_sufficient\": true,\n  \"condition_met\": null\n}}\n\n- \\`earned_points\\` must be in [0, $criterion_max_points].\n- \\`evidence_sufficient\\` = false when the available evidence is genuinely too sparse to grade fairly. The verifier will mark the criterion as evidence-insufficient.\n- \\`condition_met\\` is a boolean when the criterion has a Condition; otherwise null.\n\nDO NOT OUTPUT ANYTHING OTHER THAN JSON.\n`;\n"]}

package/dist/cjs/lib/v3/verifier/prompts/render.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Minimal Python-`string.Template`-compatible renderer for verifier prompts.
+ *
+ * The verifier prompt templates use Python's `string.Template` semantics:
+ *   - `$identifier` is a substitution placeholder.
+ *   - `$$` is a literal dollar sign.
+ *
+ * Porting strategy: keep the prompt strings verbatim (including `$$` for
+ * literal dollars), and render them through this helper instead of switching
+ * to TS template literals — the latter would require manually escaping every
+ * `$` in the prose, which is error-prone for 2000+ lines of prompts.
+ *
+ * @example
+ *   renderPrompt("Task: $task", { task: "Buy flour" }) === "Task: Buy flour"
+ *   renderPrompt("Costs $$5", {}) === "Costs $5"
+ */
+export declare function renderPrompt(template: string, vars: Record<string, string | number | boolean | undefined>): string;
+/**
+ * Build the optional "init URL context" sentence used by most prompts.
+ * When the task carries a starting URL, append
+ * "  Starting URL: <url>" after the task identifier; otherwise return empty.
+ */
+export declare function buildInitUrlContext(initUrl?: string): string;

package/dist/cjs/lib/v3/verifier/prompts/render.js

@@ -0,0 +1,45 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderPrompt = renderPrompt;
+exports.buildInitUrlContext = buildInitUrlContext;
+/**
+ * Minimal Python-`string.Template`-compatible renderer for verifier prompts.
+ *
+ * The verifier prompt templates use Python's `string.Template` semantics:
+ *   - `$identifier` is a substitution placeholder.
+ *   - `$$` is a literal dollar sign.
+ *
+ * Porting strategy: keep the prompt strings verbatim (including `$$` for
+ * literal dollars), and render them through this helper instead of switching
+ * to TS template literals — the latter would require manually escaping every
+ * `$` in the prose, which is error-prone for 2000+ lines of prompts.
+ *
+ * @example
+ *   renderPrompt("Task: $task", { task: "Buy flour" }) === "Task: Buy flour"
+ *   renderPrompt("Costs $$5", {}) === "Costs $5"
+ */
+function renderPrompt(template, vars) {
+    // Two-pass: first protect literal $$, then interpolate, then restore.
+    const placeholder = "__VERIFIER_DOUBLE_DOLLAR__";
+    let out = template.replaceAll("$$", placeholder);
+    out = out.replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (_match, key) => {
+        if (key in vars) {
+            const v = vars[key];
+            return v === undefined ? "" : String(v);
+        }
+        // Unknown variable: leave it intact so a missing-binding bug surfaces.
+        return `$${key}`;
+    });
+    return out.replaceAll(placeholder, "$");
+}
+/**
+ * Build the optional "init URL context" sentence used by most prompts.
+ * When the task carries a starting URL, append
+ * "  Starting URL: <url>" after the task identifier; otherwise return empty.
+ */
+function buildInitUrlContext(initUrl) {
+    if (!initUrl)
+        return "";
+    return `\n    Starting URL: ${initUrl}`;
+}
+//# sourceMappingURL=render.js.map

package/dist/cjs/lib/v3/verifier/prompts/render.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"render.js","sourceRoot":"","sources":["../../../../../../lib/v3/verifier/prompts/render.ts"],"names":[],"mappings":";;AAgBA,oCAgBC;AAOD,kDAGC;AA1CD;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,YAAY,CAC1B,QAAgB,EAChB,IAA2D;IAE3D,sEAAsE;IACtE,MAAM,WAAW,GAAG,4BAA4B,CAAC;IACjD,IAAI,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACjD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,6BAA6B,EAAE,CAAC,MAAM,EAAE,GAAW,EAAE,EAAE;QACvE,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,uEAAuE;QACvE,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;AAC1C,CAAC;AAED;;;;GAIG;AACH,SAAgB,mBAAmB,CAAC,OAAgB;IAClD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IACxB,OAAO,uBAAuB,OAAO,EAAE,CAAC;AAC1C,CAAC","sourcesContent":["/**\n * Minimal Python-`string.Template`-compatible renderer for verifier prompts.\n *\n * The verifier prompt templates use Python's `string.Template` semantics:\n *   - `$identifier` is a substitution placeholder.\n *   - `$$` is a literal dollar sign.\n *\n * Porting strategy: keep the prompt strings verbatim (including `$$` for\n * literal dollars), and render them through this helper instead of switching\n * to TS template literals — the latter would require manually escaping every\n * `$` in the prose, which is error-prone for 2000+ lines of prompts.\n *\n * @example\n *   renderPrompt(\"Task: $task\", { task: \"Buy flour\" }) === \"Task: Buy flour\"\n *   renderPrompt(\"Costs $$5\", {}) === \"Costs $5\"\n */\nexport function renderPrompt(\n  template: string,\n  vars: Record<string, string | number | boolean | undefined>,\n): string {\n  // Two-pass: first protect literal $$, then interpolate, then restore.\n  const placeholder = \"__VERIFIER_DOUBLE_DOLLAR__\";\n  let out = template.replaceAll(\"$$\", placeholder);\n  out = out.replace(/\\$([A-Za-z_][A-Za-z0-9_]*)/g, (_match, key: string) => {\n    if (key in vars) {\n      const v = vars[key];\n      return v === undefined ? \"\" : String(v);\n    }\n    // Unknown variable: leave it intact so a missing-binding bug surfaces.\n    return `$${key}`;\n  });\n  return out.replaceAll(placeholder, \"$\");\n}\n\n/**\n * Build the optional \"init URL context\" sentence used by most prompts.\n * When the task carries a starting URL, append\n * \"  Starting URL: <url>\" after the task identifier; otherwise return empty.\n */\nexport function buildInitUrlContext(initUrl?: string): string {\n  if (!initUrl) return \"\";\n  return `\\n    Starting URL: ${initUrl}`;\n}\n"]}

package/dist/cjs/lib/v3/verifier/prompts/rubricGeneration.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Rubric generation prompt — Step 0a of the rubric verifier pipeline.
+ *
+ * Used when a task has no precomputed_rubric (i.e., everything except
+ * upstream WebTailBench). The LLM generates a structured rubric of criteria
+ * from the task description alone, which gets cached to disk per task id.
+ *
+ * Variables:
+ *   - task_id           — the task instruction string
+ *   - init_url_context  — optional "Starting URL: ..." appendix (use buildInitUrlContext)
+ *
+ * Note on `$$`: Python's `string.Template` treats `$$` as a literal `$`.
+ * The renderPrompt() helper preserves that semantics — `$$200` in the
+ * template renders as `$200` in the final prompt sent to the model.
+ */
+export declare const RUBRIC_GENERATION_PROMPT = "Task: $task_id$init_url_context\n    You are an expert tasked with analyzing a given task to identify the key points and sub-goals explicitly stated in the task description to create a rubric for evaluation.\n\n    **Objective**: Carefully analyze the task and extract the critical elements/goals/success criteria **explicitly** mentioned in the task for achieving its goal. Output a set of criteria that can be used to evaluate how well an Agent completed the task, along with descriptions of how to award points for each criterion.\n\n    **Critical Point Definition**\n    NOTE: The rubric should **not** contain criteria that violate a \"Critical Point\" or penalize scores for not crossing a Critical Point. A Critical Point, e.g. involving 'Checkout', 'Book', 'Purchase', 'Call', 'Email', etc, is a binding transaction/agreement that would require the user's permission to use personal or sensitive information (name, email, credit card, address, email, resume, etc) in order to complete a transaction (purchase, reservation, etc) or enter into a communication that a human would be expected to do (e.g. call or email).\n    - For example, when purchasing a product, it is acceptable have a criterion to \"Add to Cart\", or select the desired timeslot to book a reservation, but it **NOT** acceptable to propose a criteria to actually complete the checkout process (requiring entering the user's name, address, credit card, or sign into their account, etc unless specified in the Task). Stopping on the \"Customer Details\" is generally acceptable. If some stores/restaurants require a location before showing the product or reservation, the agent can enter a location (e.g. city) if known, but should not enter any personal information (e.g. name, email, phone number, etc).\n    - For example, if the task is to \"call a restaurant to make a reservation,\" the success criteria should not say to make the call but rather to **find** the phone number.\n    - Similarly, if the task is to \"order new size 12 running shoes\" the criteria should not say place the order but instead find the right shoes and add them to the cart.\n    - If the task contains user information like name/email/address/etc, make a criterion stating that **other** user information not provided in the task is not \"made up\", and only what is given is reflected accurately in e.g. the checkout process.\n\n    **Controllable vs. Uncontrollable Factors** (Critical for Fair Evaluation):\n    When creating rubric criteria, distinguish between factors within and outside the agent's control. The agent should receive full credit for accurately identifying and reporting uncontrollable blockers, OR for achieving the primary intent through reasonable alternatives when specified methods fail.\n\n    **UNCONTROLLABLE FACTORS** (Full credit for identifying/reporting these):\n    1. **Platform/Infrastructure Issues**: Website down, blocked by CAPTCHA, login walls (without credentials), server errors, missing functionality\n    2. **Entity Non-Existence**: Restaurant/business closed or doesn't exist, product discontinued, service no longer available, person/entity not in directory\n    3. **Availability/Inventory Constraints**: No reservations/flights on requested date, out of stock, sold out, seasonal unavailability\n    4. **Search Result Limitations**: No exact match exists, requested attribute combination impossible, empty result sets\n    5. **Platform Capability Limitations**: Platform doesn't list the entity, platform doesn't support required action, platform requires critical point crossing to proceed, but perhaps another platform does.\n    6. **Information Accuracy Issues Beyond Agent Control**: Third-party data outdated, price/availability changed during browsing, conflicting information across sources\n\n    **CONTROLLABLE FACTORS** (Should be penalized):\n    1. **Primary Intent Violations**: Wrong entity (restaurant/product name), wrong critical attributes (date/location) when correct ones available, ignoring explicit constraints when alternatives exist\n    2. **Navigation and Search Errors**: Not attempting specified platform when accessible, poor search strategy, not using available filters\n    3. **Execution Errors**: Incorrect data entry, skipping required steps, wrong selections from available options\n    4. **Communication Failures**: Not reporting blockers encountered, hallucinations (claiming success without evidence), incomplete reporting, false unavailability claims\n    5. **Insufficient Effort**: Premature abandonment after single attempt, not trying alternatives when appropriate, immediately using alternatives without attempting specified approach\n    6. **Misunderstanding Task Requirements**: Crossing critical points unnecessarily, adding unrequested steps, ignoring task scope\n\n    **Interpret Task Verbs Charitably (Do NOT Over-Literalize)**:\n    Many tasks use casual, everyday language. Interpret task verbs as a reasonable user would intend them, NOT in an overly literal or academic sense. The agent is a web navigation assistant \u2014 the user wants the agent to find things, navigate to pages, and report back useful information. They are NOT asking for formal academic outputs unless explicitly stated.\n\n    Common examples of verbs and similar subjective terms that should be interpreted charitably:\n    - **\"Locate\"**, **\"find\"**, **\"pull up\"**, **\"look up\"**, **\"check\"**: Navigate to the relevant page/content and report the key information. Do NOT require the agent to output a URL unless explicitly asked.\n    - **\"Review\"**, **\"read\"**, **\"look at\"**, **\"go through\"**: Navigate to the content and provide a reasonable summary or overview of what was found. Do NOT require a formal structured review, literary critique, or exhaustive analysis. A brief summary of the key points visible on the page is sufficient.\n    - **\"Show me\"**, **\"get me\"**: Find and present the relevant information. The user wants to see the content, not a URL.\n    - **\"Research\"**, **\"explore\"**: Browse relevant sources and report findings. Do NOT require academic-level depth unless explicitly stated.\n\n    When creating rubric criteria for these or similar kinds of subjective verbs, the success criteria should focus on whether the agent **found the right content and reported useful information**, NOT on the depth, format, or presentation style of the output.\n\n    **SubGoal Definition**:\n    A **subgoal** is a critical element, condition, or step **explicitly** mentioned in the task description required for success.\n    - Do not infer or add any unstated subgoals or criteria, e.g. if the task is \"what standard length of vinyl outside corner trim does HomeDepot sell?\", do NOT add a criterion requiring the URL of the product, because it was not asked.\n    - **Intermediate Discovery vs. Required Output**: Criteria may verify that the agent *found, viewed, or interacted with* the correct intermediate results during navigation (e.g., \"searched for and reviewed relevant Azure courses on Coursera\"), but should NOT require the agent to *output or list* those intermediate results unless the task explicitly asks for them. For example, if the task is \"find which Azure course on Coursera has the most flexible schedule,\" the agent should receive credit for browsing and reviewing multiple Azure courses (visible in screenshots/actions), but should NOT be penalized for only reporting the most flexible one in its final answer \u2014 that is all the task asked for.\n    - Do not make redundant or overlapping criteria (e.g. for the task \"book a flight on air asia\", do NOT make separate criteria for \"access airasia.com\" and \"ensure AirAsia as the booking platform\" since they are redundant)\n    - Separate **what** the subgoals are from **how** to evaluate them\n\n    **CRITICAL: Handling Conditional Criteria**:\n\n    Some tasks contain **conditional requirements** that only apply when specific conditions are met. These must be modeled explicitly using a \"condition\" field.\n\n    **When to create a conditional criterion**:\n    - Task contains phrases like \"if...\", \"let me know if...\", \"report any issues\", \"in case of unavailability\"\n    - The requirement only applies when a specific trigger condition occurs\n    - Examples:\n      * \"Add flour to cart. Let me know if there are availability issues.\" \u2192 Reporting is conditional on encountering issues\n      * \"Book a direct flight, or if none available, book a one-stop flight.\" \u2192 One-stop criterion is conditional on no direct flights existing\n      * \"Buy organic blueberries, or if unavailable, buy non-organic.\" \u2192 Non-organic criterion is conditional on organic being unavailable\n\n    **How to structure conditional criteria**:\n    1. Add a \"condition\" field (string) that describes the triggering condition\n    2. In the \"description\" field, explain both the condition AND how to score if condition is met\n    3. Make it clear that points are ONLY counted if the condition is met\n\n    **Schema for conditional criteria**:\n    {{\n        \"criterion\": \"Brief name of what's being evaluated\",\n        \"condition\": \"Clear description of when this criterion applies (e.g., 'Only applies if organic blueberries are unavailable')\",\n        \"task_span\": \"Verbatim substring of the original task that THIS criterion is evaluating (e.g., 'organic blueberries')\",\n        \"description\": \"What to evaluate and how to score IF the condition is met. Full credit for..., partial credit for...\",\n        \"max_points\": N,\n        \"justification\": \"\",\n        \"earned_points\": \"\"\n    }}\n\n    **Schema for non-conditional criteria** (most criteria):\n    {{\n        \"criterion\": \"Brief name of what's being evaluated\",\n        \"task_span\": \"Verbatim substring of the original task that THIS criterion is evaluating (e.g., 'add flour to cart')\",\n        \"description\": \"What to evaluate and how to score. Full credit for..., partial credit for...\",\n        \"max_points\": N,\n        \"justification\": \"\",\n        \"earned_points\": \"\"\n    }}\n    (Note: No \"condition\" field means the criterion always applies)\n\n    **task_span \u2014 ANTI-HALLUCINATION ANCHOR (REQUIRED)**:\n    Every criterion **must** include a \"task_span\" field whose value is a **verbatim substring** of the original task description (the text after \"Task:\" above). This is the literal phrase from the task that justifies including this criterion.\n    - If you cannot copy a contiguous substring of the task that justifies the criterion, **the criterion does not belong in the rubric** \u2014 the task did not ask for it. Drop it.\n    - Substrings shorter than 3 words are not enough \u2014 pick a span that makes the connection unambiguous.\n    - Do NOT paraphrase, summarize, or normalize the span. Copy it character-for-character so a downstream check can verify it is a substring of the task.\n    - For setup/platform criteria implied by the task's specified platform, copy the platform name verbatim (e.g., task_span: \"drugssquare.com\").\n    - For Critical Point boundary criteria, you may use the special token \"<critical-point>\" \u2014 this is the ONE permitted non-substring value, since the Critical Point rule is supplied by these instructions, not the task text.\n\n    **Important**: Do NOT create conditional criteria for requirements that are implicitly satisfied by successful task completion.\n    - Example: \"Add flour to cart. Let me know if unavailable.\"\n      * WRONG: Separate conditional criterion \"Report unavailability (condition: flour unavailable)\"\n      * RIGHT: Single criterion \"Add flour to cart\" with description: \"Full credit if flour added to cart successfully OR if flour is unavailable and agent reports this\"\n    - Why? Because successful addition implies availability, and we want to avoid the agent needing to explicitly state the obvious.\n\n    **When TO create a conditional criterion**:\n    - When the task explicitly requests an alternative action or fallback behavior\n    - When you have mutually exclusive options (only one should be counted based on circumstances)\n\n    **IMPORTANT: Mutually Exclusive Conditionals**:\n    When a task has mutually exclusive alternatives (only one should apply), make ALL alternatives conditional with opposite conditions. This ensures only ONE is counted.\n\n    Common pattern example: \"do X, or if X unavailable, do Y\"\n    - Make BOTH X and Y conditional with opposite conditions\n    - Only the applicable one will be counted during scoring\n\n    Concrete example: \"Buy organic blueberries, or if unavailable, buy non-organic\"\n    - Criterion 1: \"Buy organic blueberries\" (condition: \"Only applies if organic blueberries are available\")\n    - Criterion 2: \"Buy non-organic blueberries\" (condition: \"Only applies if organic blueberries are unavailable\")\n    - During scoring: Only ONE will have is_condition_met=true, so only ONE is counted\n\n    **Examples**:\n\n    Example 1: \"Add flour and vegetable oil to cart at Ralphs. Let me know if there are availability issues.\"\n    - Criterion 1: Add flour to cart [no condition field] - Description includes: \"Full credit if flour added OR if unavailable and agent reports this\"\n    - Criterion 2: Add vegetable oil to cart [no condition field] - Description includes: \"Full credit if oil added OR if unavailable and agent reports this\"\n    - Do NOT create separate conditional criteria for reporting, since successful addition implies availability\n\n    Example 2: \"Buy organic blueberries at Whole Foods. If they don't have organic, buy non-organic ones. If they don't have any blueberries at all, let me know.\"\n    - Criterion 1: Buy organic blueberries [condition: \"Only applies if organic blueberries are available\"]\n    - Criterion 2: Buy non-organic blueberries [condition: \"Only applies if organic blueberries are unavailable but non-organic are available\"]\n    - Criterion 3: Report complete unavailability [condition: \"Only applies if neither organic nor non-organic blueberries are available\"]\n    - Note: Exactly ONE of these three criteria will have is_condition_met=true during scoring\n\n    **Ensure Criterion Disjointness (Avoid Double-Penalty Structures)**:\n    Make criteria as disjoint and non-overlapping as possible to avoid penalizing the same mistake multiple times.\n\n    **Key Principle**: If criterion A penalizes for not using platform/method X, then other criteria should evaluate task completion aspects (finding entities, making selections, progressing workflows) **independently** of whether platform/method X was used.\n\n    **Pattern to AVOID** (Double-penalty):\n    - Criterion 1: \"Identify a Mexican restaurant on gayot.com\"\n    - Criterion 2: \"Reach reservation interface on gayot.com\"\n    - Problem: Both penalize for not using gayot.com \u2192 agent loses points twice for same mistake\n\n    **Pattern to FOLLOW** (Disjoint):\n    - Criterion 1: \"Attempt gayot.com as the specified platform\" (evaluates platform usage)\n    - Criterion 2: \"Identify a Mexican restaurant in Chicago's Northside\" (evaluates entity identification, independent of platform)\n    - Criterion 3: \"Reach a reservation booking interface for the restaurant\" (evaluates workflow progress, independent of platform)\n    - Result: If gayot.com fails, agent only loses points on Criterion 1, not on 2 and 3\n\n    **Do Not Create Duplicate Criteria**:\n    Do NOT create multiple criteria that penalize the same mistake. Duplicate criteria create \"double jeopardy\" \u2014 the agent is unfairly penalized twice for a single error, which distorts scores. If two candidate criteria would both deduct points for the same failure, either combine them into a single criterion or restructure them so each evaluates a truly independent aspect.\n\n    **Examples of duplicates to avoid**:\n    - \"Progress booking flow up to Critical Point\" + \"Respect Critical Point and avoid personal info\"\n      \u2192 Instead, create ONE criterion: \"Progress booking flow up to (but not beyond) the Critical Point, stopping before entering personal/payment information\"\n    - \"Add product to cart\" + \"Do not complete checkout\"\n      \u2192 Instead, create ONE criterion: \"Add product to cart and stop before entering personal/payment details\"\n    - \"Add the liquid ingredients to the Target cart\" + \"Ensure only liquid ingredients are added to the cart\"\n      \u2192 Instead, create ONE criterion: \"Add only the liquid ingredients to the Target cart\" \u2014 the action and the constraint evaluate the same thing: whether the correct items (and only the correct items) were added. Adding a wrong item should be penalized once, not twice.\n    - \"Identify suitable online purchase options for each listed spice/seasoning\" + \"Limit purchasing scope to spices/seasonings only\"\n      \u2192 Instead, create ONE criterion: \"Identify suitable online purchase options for only the listed spices/seasonings\" \u2014 the scope constraint is already implicit in the action: if the agent correctly identifies options for each listed spice, it has necessarily limited scope to spices.\n\n    **Test for duplication**: Ask yourself: \"If the agent makes mistake X, would it lose points in multiple criteria?\" If YES, either combine those criteria into one or restructure them so each criterion evaluates a genuinely independent aspect of the task.\n\n    **Decompose List-Style Tasks into Per-Item Criteria**:\n    When a task contains a \"laundry list\" of items to perform the same action on (e.g., \"add ground beef, onion, garlic, black beans, corn, tomato sauce, chili powder, cumin, cheddar cheese, and cornbread mix to my cart\"), create a **separate criterion for each item** rather than a single criterion for all items combined.\n\n    **When to apply this rule**:\n    - The task lists 3 or more items/entities that all require the same type of action (e.g., add to cart, search for, book, find, look up, etc.)\n    - The items are independently actionable (success/failure on one item does not inherently depend on another)\n\n    **Why**: Each item may have independent failure modes (out of stock, not found, wrong product selected, different availability). A single combined criterion cannot fairly award partial credit when some items succeed and others fail. Separate criteria allow precise, per-item scoring.\n\n    **How**: Create one criterion per item, each with its own max_points and description including item-specific partial credit guidance (e.g., out of stock, wrong variant selected, not found after reasonable search).\n\n    **Example**: Task: \"Add ground beef, onion, garlic, black beans, and corn to my cart on Stop & Shop\"\n    - WRONG: Single criterion \"Add all grocery items to cart\" [10 points]\n    - RIGHT: Separate criteria:\n      * \"Add ground beef to cart\" [2 points] \u2014 Full credit if added, or if unavailable and agent reports this. Partial credit for wrong cut/variant when correct one is available.\n      * \"Add onion to cart\" [2 points] \u2014 Full credit if added, or if unavailable and agent reports this.\n      * \"Add garlic to cart\" [2 points] \u2014 Full credit if added, or if unavailable and agent reports this.\n      * \"Add black beans to cart\" [2 points] \u2014 Full credit if added, or if unavailable and agent reports this.\n      * \"Add corn to cart\" [2 points] \u2014 Full credit if added, or if unavailable and agent reports this.\n\n    **Note**: You may still have additional criteria for platform access (e.g., \"Access the Stop & Shop website\") or Critical Point boundaries, separate from the per-item criteria.\n\n    **Partial Credit Guidance** (to be incorporated into the \"description\" field of each criterion):\n    Each criterion's \"description\" field must specify how to handle both successful completion AND common failure modes caused by external factors. Use the framework below:\n\n    **1. Primary Intent vs. Secondary Constraints**:\n    - **Primary intent**: The core objective that defines task success (e.g., \"book reservation at Restaurant X on date Z\")\n    - **Secondary constraints**: Preferred methods/platforms (e.g., \"using platform Y\")\n    - Award **FULL credit** if primary intent is achieved through reasonable alternatives when secondary constraints are impossible due to uncontrollable factors\n    - Award **PARTIAL credit** if secondary constraints are ignored without attempting them first, but primary intent is still achieved\n    - Award **NO credit** if primary intent is violated (e.g., booking wrong restaurant name or wrong date when correct ones are available)\n\n    **2. Entity Non-Existence Scenarios**:\n    - If a specified entity (restaurant, product, business, service) no longer exists or cannot be found after reasonable search, award **FULL credit** for clearly reporting this finding\n    - If an entity doesn't exist, award **FULL credit** for either: (a) reporting non-existence OR (b) identifying a reasonable alternative that matches the primary intent\n    - Examples: Restaurant closed permanently, product discontinued, airline no longer operates that route, person not in directory\n\n    **3. Availability and Inventory Constraints**:\n    - For time-sensitive requests (dates, times, reservation slots), award **FULL credit** for accurately reporting unavailability when the requested option genuinely doesn't exist\n    - Award **FULL credit** for either: (a) clearly stating unavailability OR (b) selecting the best available alternative that matches primary intent\n    - Examples: No direct flights on requested date, restaurant fully booked, product out of stock, hotel no vacancy\n\n    **4. Platform Capability Limitations**:\n    - If a specified platform cannot support the required action (e.g., platform doesn't list the entity, platform blocks access, platform requires critical point crossing), award **FULL credit** for either: (a) reporting this limitation OR (b) achieving the goal through an alternative platform\n    - Do NOT penalize for using alternative platforms when the specified platform is genuinely incapable or inaccessible\n    - Example: Platform Y doesn't have Restaurant X listed \u2192 full credit for booking Restaurant X on Platform Z instead\n\n    **5. Search Results and Filtering Constraints**:\n    - When a task involves multiple filtering criteria but no result satisfies them all, award **FULL credit** if the agent: (a) identifies the best available option matching the **primary intent**, OR (b) states that no exact match exists, OR (c) both\n    - For subjective tasks (e.g., \"find the best new sushi restaurant\"), award **FULL credit** for consulting authoritative sources and making reasonable selections\n    - Example: Task requests \"100% cotton Power Rangers hoodie\" but only polyester hoodie exists \u2192 full credit for recommending the hoodie OR stating no exact match\n\n    **6. Reasonable Effort Requirements**:\n    - Award **FULL credit** only if the agent demonstrates reasonable effort before resorting to alternatives (attempting the specified approach, encountering genuine blocker, THEN reporting/finding alternative)\n    - Award **PARTIAL credit** if agent immediately defaults to alternatives without attempting the specified approach when it was accessible\n    - Award **NO credit** for premature abandonment without reasonable attempt\n\n    **Instructions**:\n    1. Read the task description carefully.\n    2. Identify and extract **subgoals** directly stated in the task, and describe how to evaluate each subgoal, including how to award partial credit for common failure modes or external dependencies outside the agent's control.\n    3. Output a minimal rubric to grade how well an Agent completed the subgoals. You will format your output as a rubric with the following elements/fields\n\n    The rubric must be:\n    1. Formatted as json dictionary of a (possibly nested) list of \"items\"\n    2. Each Item in the rubric must contain the following fields IN ORDER:\n       - For CONDITIONAL criteria: [\"criterion\", \"condition\", \"task_span\", \"description\", \"max_points\", \"justification\", \"earned_points\"]\n       - For NON-CONDITIONAL criteria: [\"criterion\", \"task_span\", \"description\", \"max_points\", \"justification\", \"earned_points\"]\n       - The \"condition\" field should ONLY be present for conditional criteria (criteria that only apply when specific conditions are met)\n       - The \"task_span\" field is REQUIRED on every criterion (verbatim substring of the task, or \"<critical-point>\" for Critical Point boundary criteria only)\n    3. Choose the \"max_points\" judiciously to account for possible failure modes that could earn partial credit: goals that would have more failure modes deserve higher max_points.\n    4. The \"description\" should explain *what* goal the criteria is evaluating and *how* partial credit could be awarded to fairly penalize the agent's mistakes while accounting for external dependencies outside the agent's control.\n    5. For conditional criteria, the \"condition\" field must clearly state when the criterion applies (e.g., \"Only applies if organic blueberries are unavailable\")\n    6. Leave the \"earned_points\" and \"justification\" fields **empty** (since this rubric isn't being evaluated right now).\n    7. Do not make criteria for formatting/style unless stated explicitly in the Task.\n    8. Keep the rubric simple, following ONLY the main keypoints the task required. Do not overcomplicate the criteria or include optional items that were not explicitly mentioned.\n\n    **ANTI-PATTERN \u2014 Over-Specification / Hallucinated Sub-Goals**:\n    A common failure mode is inventing criteria that the task never asked for. If a criterion's task_span isn't a substring of the original task description (and it isn't the Critical Point boundary), the criterion is over-specifying.\n\n    Anti-example task: \"Find the citation page for the City of Vancouver.\"\n    - WRONG criterion: \"Display the citation number, fine amount, and payment due date\" \u2014 the task did not ask for any of these specific fields. There is no \"citation number\" or \"fine amount\" in the task text.\n      * task_span: <none possible> \u2192 drop.\n    - WRONG criterion: \"Provide instructions for paying the citation online\" \u2014 the task did not ask for payment instructions.\n      * task_span: <none possible> \u2192 drop.\n    - RIGHT criterion: \"Locate the citation page on the City of Vancouver site\" [3 points]\n      * task_span: \"citation page for the City of Vancouver\"\n      * Description: Full credit if agent navigates to the citation/ticket page on the official cityofvancouver site, or if no such page exists and agent reports this. Partial credit for finding an unrelated city services page.\n\n    Anti-example task: \"Search arXiv for the most recent paper on retrieval-augmented generation.\"\n    - WRONG criterion: \"Output the abstract and author list of the paper\" \u2014 task didn't ask for abstract or author list. The agent just needs to identify and report the paper.\n      * task_span: <none possible> \u2192 drop.\n    - RIGHT criterion: \"Identify the most recent arXiv paper on retrieval-augmented generation\" [4 points]\n      * task_span: \"most recent paper on retrieval-augmented generation\"\n\n    **The test**: For each criterion you draft, ask: \"Can I copy a contiguous substring of the task that this criterion is evaluating?\" If no, the criterion is over-specifying and must be dropped.\n\n    ==================\n    Example 1 (high level): \"Book a reservation at a Mexican restaurant in the Northside of Chicago on 09/29/2025 for 2:45 PM at gayot.com.\"\n    High level rubric:\n        - Start on gayot.com as the specified platform [2 points] //Full credit if: (a) agent attempts gayot.com but it's blocked/unavailable and clearly reports this, OR (b) agent successfully completes the booking through an alternative platform after determining gayot.com cannot complete the booking\n        - Identify a Mexican restaurant in Chicago's Northside [3 points] //Full credit for correct identification. Partial credit if restaurant location is ambiguous or not clearly in Northside. No credit if wrong cuisine type when Mexican options exist\n        - Reach a reservation booking interface for the restaurant [2 points] //Full credit for ANY reservation booking interface (decoupled from platform criterion to avoid double-penalizing). Full credit if restaurant doesn't offer online booking and agent reports this\n        - Attempt date/time selection for 09/29/2025 at 2:45 PM [3 points] //Full credit if: (a) correct date/time selected, OR (b) that date/time is unavailable and agent clearly indicates this, OR (c) agent selects closest available alternative and explains why. No credit if wrong date selected when correct date is available\n        - Respect Critical Point boundaries (no personal info or final booking) [2 points] //Full credit for stopping before entering personal information. Deduct if agent unnecessarily crosses critical point\n\n    Takeaway: Make criteria as disjoint and non-overlapping as possible, e.g. do NOT make \"Reach the reservation interface starting from gayot.com\" a criterion bc this overlaps with the first criterion. Each criterion should clearly specify full credit scenarios for both success AND uncontrollable failure modes.\n\n    ==================\n    Example 2 (detailed): Help purchase 'Fosbait: Lanthanum Carbonate 500mg Tablets' for kidney diseases on drugssquare.com.\n\n    Example Output Rubric:\n    {{\n    \"items\": [\n        {{\n        \"criterion\": \"Access drugssquare.com website\",\n        \"task_span\": \"drugssquare.com\",\n        \"description\": \"Successfully navigate to or access the drugssquare.com website as specified in the task. Full credit if: (a) site is successfully accessed, OR (b) site is inaccessible (blocked, down, error) and agent clearly indicates this and either stops or attempts alternative source. Partial credit if agent immediately uses alternative without attempting drugssquare.com first.\",\n        \"max_points\": 2,\n        \"justification\": \"\",\n        \"earned_points\": \"\"\n        }},\n        {{\n        \"criterion\": \"Locate the specific product\",\n        \"task_span\": \"Fosbait: Lanthanum Carbonate 500mg Tablets\",\n        \"description\": \"Find and identify the exact product 'Fosbait: Lanthanum Carbonate 500mg Tablets' on the website. Full credit if: (a) exact product found, OR (b) product is discontinued/out of stock/not found after reasonable search and agent clearly reports this, OR (c) agent identifies closest alternative (different dosage of same medication) when exact product unavailable and explains the substitution. Partial credit for selecting different dosage when the intended 500mg dosage IS available on the site. No credit for selecting entirely different medication when Fosbait is available.\",\n        \"max_points\": 3,\n        \"justification\": \"\",\n        \"earned_points\": \"\"\n        }},\n        {{\n        \"criterion\": \"Facilitate the purchase process until a Critical Point\",\n        \"task_span\": \"<critical-point>\",\n        \"description\": \"Help complete the purchase of the identified product up to but not crossing the Critical Point. Full credit if: (a) product is successfully added to cart and agent stops before entering personal/payment information, OR (b) the website's purchase flow is broken/unavailable and agent reports this, OR (c) website requires account creation or payment info before allowing add-to-cart and agent reports this limitation. Partial credit if agent proceeds partially but makes errors in the purchase flow (wrong quantity, wrong shipping option, etc.). No credit if agent either skips the cart entirely OR unnecessarily enters personal information when not required.\",\n        \"max_points\": 3,\n        \"justification\": \"\",\n        \"earned_points\": \"\"\n        }}\n    ]\n    }}\n    Please output an answer in pure JSON format according to the following schema. The JSON object must be parsable as-is. DO NOT OUTPUT ANYTHING OTHER THAN JSON, AND DO NOT DEVIATE FROM THE ABOVE SCHEMA:";