@enricai/barnacle 1.6.13 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +3 -5
  2. package/dist/api/schemas/submissions.d.ts +8 -8
  3. package/dist/lib/bedrock.d.ts +10 -7
  4. package/dist/lib/bedrock.d.ts.map +1 -1
  5. package/dist/lib/bedrock.js +8 -5
  6. package/dist/lib/bedrock.js.map +1 -1
  7. package/dist/lib/http.d.ts +4 -3
  8. package/dist/lib/http.d.ts.map +1 -1
  9. package/dist/lib/http.js +25 -3
  10. package/dist/lib/http.js.map +1 -1
  11. package/dist/lib/llm/anthropic-client.d.ts +12 -0
  12. package/dist/lib/llm/anthropic-client.d.ts.map +1 -1
  13. package/dist/lib/llm/anthropic-client.js +25 -0
  14. package/dist/lib/llm/anthropic-client.js.map +1 -1
  15. package/dist/lib/telemetry/beacon-capture.d.ts +4 -3
  16. package/dist/lib/telemetry/beacon-capture.d.ts.map +1 -1
  17. package/dist/lib/telemetry/beacon-capture.js.map +1 -1
  18. package/dist/lib/telemetry/reconciliation-record.d.ts +15 -33
  19. package/dist/lib/telemetry/reconciliation-record.d.ts.map +1 -1
  20. package/dist/lib/telemetry/reconciliation-record.js +11 -49
  21. package/dist/lib/telemetry/reconciliation-record.js.map +1 -1
  22. package/dist/lib/telemetry/submission-capture.d.ts +8 -13
  23. package/dist/lib/telemetry/submission-capture.d.ts.map +1 -1
  24. package/dist/lib/telemetry/submission-capture.js +2 -4
  25. package/dist/lib/telemetry/submission-capture.js.map +1 -1
  26. package/dist/lib/telemetry/submission-reader.d.ts +3 -4
  27. package/dist/lib/telemetry/submission-reader.d.ts.map +1 -1
  28. package/dist/lib/telemetry/submission-reader.js +3 -4
  29. package/dist/lib/telemetry/submission-reader.js.map +1 -1
  30. package/dist/plugins/config-plugin.d.ts.map +1 -1
  31. package/dist/plugins/config-plugin.js +25 -3
  32. package/dist/plugins/config-plugin.js.map +1 -1
  33. package/dist/plugins/loader.d.ts +0 -7
  34. package/dist/plugins/loader.d.ts.map +1 -1
  35. package/dist/plugins/loader.js +0 -9
  36. package/dist/plugins/loader.js.map +1 -1
  37. package/dist/scraper/deep-locator-actuate.d.ts +24 -19
  38. package/dist/scraper/deep-locator-actuate.d.ts.map +1 -1
  39. package/dist/scraper/deep-locator-actuate.js +68 -23
  40. package/dist/scraper/deep-locator-actuate.js.map +1 -1
  41. package/dist/scraper/deep-locator-scan.d.ts +20 -0
  42. package/dist/scraper/deep-locator-scan.d.ts.map +1 -1
  43. package/dist/scraper/deep-locator-scan.js +25 -0
  44. package/dist/scraper/deep-locator-scan.js.map +1 -1
  45. package/dist/scraper/flow-runner.d.ts +86 -14
  46. package/dist/scraper/flow-runner.d.ts.map +1 -1
  47. package/dist/scraper/flow-runner.js +233 -87
  48. package/dist/scraper/flow-runner.js.map +1 -1
  49. package/dist/scripts/recon-browser.d.ts +6 -5
  50. package/dist/scripts/recon-browser.d.ts.map +1 -1
  51. package/dist/scripts/recon-browser.js +24 -21
  52. package/dist/scripts/recon-browser.js.map +1 -1
  53. package/dist/scripts/recon-generate.d.ts +2 -2
  54. package/dist/scripts/recon-generate.d.ts.map +1 -1
  55. package/dist/scripts/recon-generate.js +21 -88
  56. package/dist/scripts/recon-generate.js.map +1 -1
  57. package/dist/scripts/recon-shared.d.ts +0 -10
  58. package/dist/scripts/recon-shared.d.ts.map +1 -1
  59. package/dist/scripts/recon-shared.js +0 -11
  60. package/dist/scripts/recon-shared.js.map +1 -1
  61. package/dist/site-plugin.d.ts +3 -3
  62. package/dist/testing/coverage-guard-suite.d.mts +7 -7
  63. package/dist/testing/coverage-guard-suite.mjs +5 -5
  64. package/package.json +5 -2
  65. package/dist/scripts/migrate-telemetry-dir-names.d.ts +0 -57
  66. package/dist/scripts/migrate-telemetry-dir-names.d.ts.map +0 -1
  67. package/dist/scripts/migrate-telemetry-dir-names.js +0 -156
  68. package/dist/scripts/migrate-telemetry-dir-names.js.map +0 -1
@@ -29,6 +29,7 @@ exports.windowHasTransitionBody = windowHasTransitionBody;
29
29
  exports.windowHasAdvanceTransition = windowHasAdvanceTransition;
30
30
  exports.shouldVetoFallbackAdvance = shouldVetoFallbackAdvance;
31
31
  exports.isDomOnlyAdvanceVerified = isDomOnlyAdvanceVerified;
32
+ exports.isClickViewSwapVerified = isClickViewSwapVerified;
32
33
  exports.countNgInvalidContainers = countNgInvalidContainers;
33
34
  exports.describeAttemptEffectSignals = describeAttemptEffectSignals;
34
35
  exports.shouldSkipTechnique = shouldSkipTechnique;
@@ -76,7 +77,7 @@ const node_crypto_1 = require("node:crypto");
76
77
  const node_fs_1 = require("node:fs");
77
78
  const node_os_1 = require("node:os");
78
79
  const node_path_1 = require("node:path");
79
- const zod_1 = require("@anthropic-ai/sdk/helpers/zod");
80
+ const ai_1 = require("ai");
80
81
  const config_1 = require("../config");
81
82
  const errors_1 = require("../lib/errors");
82
83
  const error_messages_1 = require("../lib/llm/judges/error-messages");
@@ -479,7 +480,7 @@ exports.TRAILING_GRACE_WINDOW = 2;
479
480
  * means no injection surface. Runs in browser context and returns a typed-narrow
480
481
  * shape via Runtime.callFunctionOn.
481
482
  */
482
- const DOM_SNAPSHOT_EXPR = `(() => { const b = document.body; if (!b) return { html: 0, text: "" }; const t = b.innerText || ""; return { html: (b.outerHTML || "").length, text: t.length + ":" + t.slice(0, 200) }; })()`;
483
+ const DOM_SNAPSHOT_EXPR = `(() => { const b = document.body; if (!b) return { html: 0, text: "", values: "" }; const t = b.innerText || ""; const controls = Array.from(b.querySelectorAll("input, textarea, select")).filter((el) => el.offsetParent !== null); const values = controls.map((el) => { if (el.type === "checkbox" || el.type === "radio") return el.checked ? "1" : "0"; return el.value || ""; }).join("|").slice(0, 2000); return { html: (b.outerHTML || "").length, text: t.length + ":" + t.slice(0, 200), values }; })()`;
483
484
  /**
484
485
  * Captures the pre/post signal triple the submit-verify cascade diffs.
485
486
  * Accepts the optional `page` so a resolved child `FrameTarget` whose
@@ -490,6 +491,7 @@ const DOM_SNAPSHOT_EXPR = `(() => { const b = document.body; if (!b) return { ht
490
491
  async function snapshotPage(target, signalCounter, page) {
491
492
  let bodyHtmlLength = 0;
492
493
  let visibleTextSignature = "";
494
+ let formValueSignature = "";
493
495
  try {
494
496
  const result = await target.evaluate(DOM_SNAPSHOT_EXPR);
495
497
  if (result !== null &&
@@ -500,6 +502,8 @@ async function snapshotPage(target, signalCounter, page) {
500
502
  typeof result.text === "string") {
501
503
  bodyHtmlLength = result.html;
502
504
  visibleTextSignature = result.text;
505
+ const rawValues = result.values;
506
+ formValueSignature = typeof rawValues === "string" ? rawValues : "";
503
507
  }
504
508
  }
505
509
  catch {
@@ -518,6 +522,7 @@ async function snapshotPage(target, signalCounter, page) {
518
522
  url,
519
523
  bodyHtmlLength,
520
524
  visibleTextSignature,
525
+ formValueSignature,
521
526
  };
522
527
  }
523
528
  /**
@@ -892,6 +897,43 @@ function isDomOnlyAdvanceVerified(params) {
892
897
  // non-advancing POST) does not verify it.
893
898
  return networkIsRealAdvance || urlChanged;
894
899
  }
900
+ /**
901
+ * Client-side view-swap verification gate. Credits a click that produces
902
+ * substantial DOM growth with zero network as a verified step outcome when
903
+ * the step is a plain client-side reveal (not a submit/final step, not an
904
+ * advance/'Next' step on a site with advanceTransitionBodyPattern configured).
905
+ *
906
+ * **Why this exists:** SPA/iframe wizards often swap visible screens
907
+ * (e.g. method-chooser → basic-info form) via client-side DOM manipulation
908
+ * with zero network requests. The existing verifyDomEffect (line 4866-4891)
909
+ * explicitly returns false for non-radio/checkbox clicks, routing those
910
+ * to the network/URL signal — which correctly verifies navigation but
911
+ * false-negatives client-side view swaps. This gate fills that gap.
912
+ *
913
+ * **Threshold rationale:** 5000B is 10× the trivial boundary (500B) and
914
+ * safely distinguishes real view swaps (measured case: +49518B) from
915
+ * validation-error reflows (typically < 2KB per codebase evidence at
916
+ * line 2040-2044: error text capped at 200 chars, container markup adds
917
+ * 150-350B per error, worst-case ~10 errors = ~3500B max).
918
+ *
919
+ * **Scope guards:** Excludes final/submit steps (those require real network
920
+ * per isSubmitRevealedInvalid + LLM submit judge) and advance-pattern steps
921
+ * (those require real transition per isDomOnlyAdvanceVerified/isAdvanceStalled
922
+ * to avoid wizard-ATS autosave-vs-transition ambiguity).
923
+ */
924
+ function isClickViewSwapVerified(params) {
925
+ const VIEW_SWAP_MIN_BYTES = 5000;
926
+ const { resolvedAction, isFinalStep, submitStep, isAdvanceWithPattern, networkDelta, bytesDelta, } = params;
927
+ if (resolvedAction?.method !== "click")
928
+ return false;
929
+ if (isFinalStep || submitStep)
930
+ return false;
931
+ if (isAdvanceWithPattern)
932
+ return false;
933
+ if (networkDelta !== 0)
934
+ return false;
935
+ return bytesDelta >= VIEW_SWAP_MIN_BYTES;
936
+ }
895
937
  /**
896
938
  * Read-only count of ng-invalid form controls on the resolved frame. Side-effect-free
897
939
  * counterpart to `probeFormValidityBeforeSubmit` (which also auto-fills
@@ -934,6 +976,7 @@ function describeAttemptEffectSignals(pre, post, recentCaptureMeta, preMetaLengt
934
976
  const bytesDelta = post.bodyHtmlLength - pre.bodyHtmlLength;
935
977
  const networkDelta = post.networkCount - pre.networkCount;
936
978
  const textChanged = post.visibleTextSignature !== pre.visibleTextSignature;
979
+ const valueChanged = post.formValueSignature !== pre.formValueSignature;
937
980
  const windowCaptures = recentCaptureMeta.slice(preMetaLength);
938
981
  const observations = [];
939
982
  if (networkDelta === 0 && bytesDelta >= 500) {
@@ -948,6 +991,9 @@ function describeAttemptEffectSignals(pre, post, recentCaptureMeta, preMetaLengt
948
991
  if (textChanged && networkDelta === 0 && bytesDelta < 500) {
949
992
  observations.push("visible-text-changed-without-network: page reflowed slightly (likely tooltip/focus state)");
950
993
  }
994
+ if (valueChanged && !textChanged && networkDelta === 0) {
995
+ observations.push("form-value-changed-without-network: a visible input/textarea/select value changed with no other observable effect (plain field fill)");
996
+ }
951
997
  return observations.join(" | ");
952
998
  }
953
999
  /**
@@ -998,7 +1044,7 @@ function shouldSkipTechnique(params) {
998
1044
  // structured-click / observe-act-exclude re-resolves the SAME
999
1045
  // light-DOM-only view of the page and would no-op identically, so skip
1000
1046
  // straight to deep-submit-locator (attempt 2) instead. llm-rephrase
1001
- // (attempt 5) is never skipped — a differently-worded instruction is still
1047
+ // (attempt 4) is never skipped — a differently-worded instruction is still
1002
1048
  // a distinct attempt worth trying if the deep locator also fails. Gated on
1003
1049
  // submitShapedStep: the deep submit-control locator ranks submit-shaped
1004
1050
  // candidates only, so on a non-submit control (e.g. a radio/checkbox) it
@@ -1105,8 +1151,8 @@ function isAdvanceStalled(params) {
1105
1151
  return networkFired;
1106
1152
  }
1107
1153
  /**
1108
- * Lazy Anthropic client for attempt 4's rephrase. Returns null when the
1109
- * deployment is Bedrock-only (no ANTHROPIC_API_KEY) — attempt 4 then becomes
1154
+ * Lazy Anthropic client for attempt 5's rephrase. Returns null when the
1155
+ * deployment is Bedrock-only (no ANTHROPIC_API_KEY) — attempt 5 then becomes
1110
1156
  * a no-op and the executor escalates straight to the failure dump. The other
1111
1157
  * three attempts already cover the lion's share of recovery.
1112
1158
  */
@@ -1127,7 +1173,7 @@ function anthropicModelName() {
1127
1173
  */
1128
1174
  const REPHRASE_SYSTEM_PROMPT = "You rewrite a failed Stagehand act() instruction so the next attempt targets a different element than those already tried. If no different element exists on the page that could plausibly unblock the flow, return outcome=impossible. Never re-propose a selector or wording from the lists labeled SELECTORS ALREADY TRIED or INSTRUCTION TEXT ALREADY TRIED.";
1129
1175
  /**
1130
- * Attempt-4 of the step-healing cascade: when three mechanical retry variations
1176
+ * Attempt-5 of the step-healing cascade: when three mechanical retry variations
1131
1177
  * all fail, this is the last resort before the step is declared terminal. Exported
1132
1178
  * so tests can inject a fake capture sink without touching the browser session.
1133
1179
  */
@@ -1181,13 +1227,21 @@ priorAttempts,
1181
1227
  * "click harder" into "fill the actually-missing field." Telemetry
1182
1228
  * the engine already captures but never showed the LLM until now.
1183
1229
  */
1184
- gaEventEvidence) {
1230
+ gaEventEvidence,
1231
+ /**
1232
+ * Anthropic client for the modal-priority judge inside
1233
+ * `renderUnfocusedObserve`. Separate from `client` (the ai-SDK model used
1234
+ * for the rephrase call itself) because that judge pipeline is unrelated
1235
+ * to which provider generates the rephrase, and null on a Bedrock-only
1236
+ * deployment falls back to `renderUnfocusedObserve`'s source-order default.
1237
+ */
1238
+ judgeClient) {
1185
1239
  const candidateList = observeCandidates
1186
1240
  .slice(0, 12)
1187
1241
  .map((a, i) => `${i + 1}. ${a.description} — ${a.selector}`)
1188
1242
  .join("\n");
1189
1243
  const unfocusedList = unfocusedObserve
1190
- ? await renderUnfocusedObserve(unfocusedObserve, { client, captureFn })
1244
+ ? await renderUnfocusedObserve(unfocusedObserve, { client: judgeClient, captureFn })
1191
1245
  : "";
1192
1246
  const triedList = triedSelectors.length > 0 ? triedSelectors.join("\n") : "(none)";
1193
1247
  const reasonList = failureReasons.map((r, i) => `attempt ${i + 1}: ${r}`).join("\n");
@@ -1259,35 +1313,28 @@ PAGE TRANSITION + VALIDATOR TELEMETRY (parsed from Google Analytics Measurement
1259
1313
  ${gaEventEvidence || "(none)"}
1260
1314
 
1261
1315
  Rewrite the instruction so a Stagehand act() call can resolve it unambiguously to a different element than the ones already tried. Keep it short — one sentence, natural language, no quotes around it. If the invalid-fields section above names a field AND the interactive-targets section below lists clickable options inside that same container, your rewrite picks one of those options (e.g. for a yes/no question rendered as two radio labels, choose the candidate-favorable answer — 'No' for "do you have a non-compete", 'Yes' for "are you authorized to work", 'Prefer not to say' for demographic disclosures) rather than retrying the original click. If the unfocused observe section shows an open modal/dialog with a Save or Close action, your rewrite invokes that action so the underlying form can clear its blocking state. Set outcome to "impossible" with a one-line reason only when the original instruction's element does not exist on the current page and no redirect target is available.`;
1262
- const model = anthropicModelName();
1316
+ const model = client.modelId;
1263
1317
  const t0 = performance.now();
1264
1318
  try {
1265
- const response = await client.messages.parse({
1266
- model,
1267
- max_tokens: 400,
1319
+ const response = await (0, ai_1.generateObject)({
1320
+ model: client,
1321
+ maxOutputTokens: 400,
1268
1322
  system: REPHRASE_SYSTEM_PROMPT,
1269
- messages: [{ role: "user", content: prompt }],
1270
- output_config: {
1271
- format: (0, zod_1.zodOutputFormat)(schemas_1.REPHRASE_RESPONSE_SCHEMA),
1272
- },
1323
+ prompt,
1324
+ schema: schemas_1.REPHRASE_RESPONSE_SCHEMA,
1273
1325
  });
1274
1326
  const latencyMs = performance.now() - t0;
1275
- const parsed = response.parsed_output;
1276
- if (parsed === null) {
1277
- throw new Error("structured-output enabled but parsed_output is null");
1278
- }
1279
- const textBlock = response.content.find((b) => b.type === "text");
1280
- const rawText = textBlock?.type === "text" ? textBlock.text : "";
1327
+ const parsed = response.object;
1281
1328
  await captureFn({
1282
1329
  callId: (0, node_crypto_1.randomUUID)(),
1283
1330
  callType: call_types_1.CALL_TYPE_RECON_REPHRASE,
1284
1331
  model,
1285
1332
  systemPrompt: REPHRASE_SYSTEM_PROMPT,
1286
1333
  userContent: prompt,
1287
- responseContent: rawText,
1334
+ responseContent: JSON.stringify(parsed),
1288
1335
  parsedOk: true,
1289
- inputTokens: response.usage?.input_tokens ?? null,
1290
- outputTokens: response.usage?.output_tokens ?? null,
1336
+ inputTokens: response.usage.inputTokens ?? null,
1337
+ outputTokens: response.usage.outputTokens ?? null,
1291
1338
  latencyMs,
1292
1339
  success: true,
1293
1340
  errorMessage: null,
@@ -2295,7 +2342,7 @@ function isUploadAffordanceLabel(label) {
2295
2342
  return /\b(upload|browse|select file|choose file|attach|add (a )?(resume|cv|file)|resume\/cv)\b/.test(norm);
2296
2343
  }
2297
2344
  /**
2298
- * Materialize the in-memory resume fixture to a temp file so CDP
2345
+ * Materialize the in-memory upload fixture to a temp file so CDP
2299
2346
  * `DOM.setFileInputFiles` (which requires a filesystem path, unlike
2300
2347
  * Playwright's `locator.setInputFiles`) can reference it. Only used as a
2301
2348
  * fallback when the on-disk fixture path is unavailable. Process-scoped — the
@@ -2805,6 +2852,24 @@ function resolveDeepLocatorActuation(step) {
2805
2852
  function normalizeFieldLabel(text) {
2806
2853
  return text.replace(/\s+/g, " ").trim().toLowerCase();
2807
2854
  }
2855
+ /**
2856
+ * Parses a fill/select step into the field it names plus the value to write,
2857
+ * or `null` for any other step shape (click, a select whose question is
2858
+ * un-quoted, etc). Shared by both the observe-empty deepLocator fallback and
2859
+ * the field-label-first routing ahead of Stagehand's own act()/observe()
2860
+ * resolution — see `findDeepLocatorCandidateByFieldLabel`'s docblock for why
2861
+ * the field LABEL (not the value) is what a caller matches candidates
2862
+ * against.
2863
+ */
2864
+ function parseFieldLabelTarget(step) {
2865
+ const fillStep = parseFillStep(step);
2866
+ if (fillStep)
2867
+ return { kind: "fill", fieldLabel: fillStep.fieldLabel, value: fillStep.value };
2868
+ const selectStep = parseSelectStep(step);
2869
+ return selectStep?.questionLabel
2870
+ ? { kind: "select", fieldLabel: selectStep.questionLabel, value: selectStep.option }
2871
+ : null;
2872
+ }
2808
2873
  /**
2809
2874
  * Finds the candidate whose accessible name identifies the named field —
2810
2875
  * `fieldLabel` (from {@link parseFillStep}/{@link parseSelectStep}), NOT the
@@ -4526,6 +4591,36 @@ async function resolveDeepLocatorCandidatesWithWidening(page, frameSelector, ins
4526
4591
  const widened = await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameSelector, "*", instruction, timeoutOptions);
4527
4592
  return { candidates: widened, innerSelector: "*" };
4528
4593
  }
4594
+ /**
4595
+ * Resolves and actuates a fill/select step's NAMED FIELD through the
4596
+ * deterministic accessible-name match (`findDeepLocatorCandidateByFieldLabel`)
4597
+ * against the frame's own deepLocator scan, independent of what Stagehand's
4598
+ * `act()`/`observe()` resolved. A same-frame accessible-name label match
4599
+ * cannot confuse a City input for an unrelated header search box the way
4600
+ * Stagehand's semantic resolution can — Stagehand's own candidate is only a
4601
+ * fallback for the caller to use when this returns `no-match` or
4602
+ * `not-fill-or-select`, never the first attempt for a step that names a
4603
+ * field. `triedSelectors` (already-attempted selectors from prior
4604
+ * cascade attempts) is excluded so a step that already tried and failed on
4605
+ * this candidate doesn't just re-pick it.
4606
+ */
4607
+ async function tryDeterministicFieldLabelActuation(params) {
4608
+ const { page, frameTarget, step, triedSelectors, timeoutOptions } = params;
4609
+ const fieldTarget = parseFieldLabelTarget(step);
4610
+ if (!fieldTarget)
4611
+ return { kind: "not-fill-or-select" };
4612
+ const { candidates, innerSelector } = await resolveDeepLocatorCandidatesWithWidening(page, frameTarget.frameSelector, step, timeoutOptions);
4613
+ const unexcluded = candidates.filter((c) => !triedSelectors.includes(c.selector));
4614
+ const matched = findDeepLocatorCandidateByFieldLabel(unexcluded, fieldTarget.fieldLabel);
4615
+ if (!matched)
4616
+ return { kind: "no-match", fieldTarget };
4617
+ const actuated = fieldTarget.kind === "fill"
4618
+ ? await (0, deep_locator_actuate_1.fillDeepLocatorCandidate)(page, frameTarget.frameSelector, innerSelector, matched.index, fieldTarget.value, { frameTarget })
4619
+ : await (0, deep_locator_actuate_1.selectDeepLocatorCandidateOption)(page, frameTarget.frameSelector, innerSelector, matched.index, fieldTarget.value, { frameTarget });
4620
+ return actuated
4621
+ ? { kind: "actuated", matched, fieldTarget }
4622
+ : { kind: "actuation-failed", matched, fieldTarget };
4623
+ }
4529
4624
  /**
4530
4625
  * Adapts `resolveDeepLocatorCandidatesWithWidening` results into
4531
4626
  * `Action`-shaped evidence for `rephraseWithLLM`, which only knows about
@@ -4636,7 +4731,7 @@ async function probeStepBeforeAttempts(params) {
4636
4731
  }
4637
4732
  }
4638
4733
  async function executeStepWithHealing(params) {
4639
- const { stagehand, page, step, optional, upload, submitStep, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, logger, captureFn, resumeFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
4734
+ const { stagehand, page, step, optional, upload, submitStep, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, rephraseModel, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
4640
4735
  // Mutable (not the destructured const above) so a lost frame-attach race
4641
4736
  // can be upgraded in place once the OOPIF attaches later in the cascade —
4642
4737
  // see reresolveFrameTargetIfLost below. Every existing reference in this
@@ -4710,7 +4805,7 @@ async function executeStepWithHealing(params) {
4710
4805
  page,
4711
4806
  target: frameTarget ?? (0, frame_target_1.mainFrameTarget)(page),
4712
4807
  isUploadStep: upload,
4713
- fixture: resumeFixture,
4808
+ fixture: uploadFixture,
4714
4809
  logger,
4715
4810
  signalCounter,
4716
4811
  recentCaptureMeta,
@@ -4950,6 +5045,7 @@ async function executeStepWithHealing(params) {
4950
5045
  url: page.url(),
4951
5046
  bodyHtmlLength: 0,
4952
5047
  visibleTextSignature: "",
5048
+ formValueSignature: "",
4953
5049
  };
4954
5050
  attempts.push({
4955
5051
  attempt: 0,
@@ -5051,6 +5147,57 @@ async function executeStepWithHealing(params) {
5051
5147
  // from DOM re-read. Captured here so both branches of the cascade write to it.
5052
5148
  let resolvedAction = null;
5053
5149
  try {
5150
+ // Field-label-first routing: for a frame-scoped fill/select step,
5151
+ // Stagehand's own act()/observe() can resolve a real, in-DOM, plausible-
5152
+ // looking control that is nonetheless the WRONG one (a City field
5153
+ // resolving to an unrelated header search box whose accessible name
5154
+ // overlaps semantically with "city/location" is the motivating case).
5155
+ // A same-frame accessible-name match against the field's own label
5156
+ // can't make that mistake, so it's tried BEFORE any Stagehand
5157
+ // act/observe call for every attempt, not only once observe() has
5158
+ // already come back empty (bugfix-001's finding: observe() returning a
5159
+ // wrong-but-nonempty candidate was exactly what let the phantom-fill
5160
+ // through the old empty-candidates-only gate). Falls through to the
5161
+ // existing Stagehand-driven attempt when no candidate names the field
5162
+ // at all — this only preempts Stagehand's resolution, it never expands
5163
+ // what counts as a match.
5164
+ const fieldLabelOutcome = attempt !== 3 && frameTarget?.frame
5165
+ ? await tryDeterministicFieldLabelActuation({
5166
+ page,
5167
+ frameTarget,
5168
+ step,
5169
+ triedSelectors,
5170
+ timeoutOptions: { frameTarget },
5171
+ })
5172
+ : { kind: "not-fill-or-select" };
5173
+ if (fieldLabelOutcome.kind === "actuated") {
5174
+ record.technique = "deep-locator-field-label";
5175
+ record.instruction = `deepLocator: ${fieldLabelOutcome.matched.accessibleText || "(no accessible text)"}`;
5176
+ triedSelectors.push(fieldLabelOutcome.matched.selector);
5177
+ record.triedSelectors = [fieldLabelOutcome.matched.selector];
5178
+ record.actResultSuccess = true;
5179
+ record.actResultDescription = `deepLocator ${fieldLabelOutcome.fieldTarget.kind === "fill" ? "filled" : "selected"} "${fieldLabelOutcome.matched.accessibleText || fieldLabelOutcome.matched.selector}" with "${fieldLabelOutcome.fieldTarget.value}"`;
5180
+ // Same rationale as the pre-existing candidates===0 branch below:
5181
+ // the write + read-back the actuation helper already performed IS
5182
+ // the verification signal — no resolvedAction/network/url verifier
5183
+ // can score a `deeplocator=` selector.
5184
+ record.verifiedBy = "dom";
5185
+ attempts.push(record);
5186
+ logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${record.actResultDescription}`);
5187
+ trajectory?.push({ stepIndex, verifiedBy: "dom" });
5188
+ return "completed";
5189
+ }
5190
+ if (fieldLabelOutcome.kind === "actuation-failed") {
5191
+ triedSelectors.push(fieldLabelOutcome.matched.selector);
5192
+ record.triedSelectors = [fieldLabelOutcome.matched.selector];
5193
+ const failureMessage = `deepLocator: ${fieldLabelOutcome.fieldTarget.kind} on "${fieldLabelOutcome.matched.accessibleText || fieldLabelOutcome.matched.selector}" did not verify (read-back mismatch or rejected write)`;
5194
+ record.actResultSuccess = false;
5195
+ record.errorMessage = failureMessage;
5196
+ attempts.push(record);
5197
+ failureReasons.push(failureMessage);
5198
+ logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${failureMessage}`);
5199
+ continue;
5200
+ }
5054
5201
  if (attempt === 1) {
5055
5202
  record.technique = "act-string";
5056
5203
  record.instruction = step;
@@ -5229,58 +5376,19 @@ async function executeStepWithHealing(params) {
5229
5376
  const deepLocatorInnerSelector = deepLocatorResolution?.innerSelector ?? deep_locator_scan_1.INTERACTIVE_CANDIDATE_SELECTOR;
5230
5377
  const deepLocatorCandidates = (deepLocatorResolution?.candidates ?? []).filter((c) => !triedSelectors.includes(c.selector));
5231
5378
  if (candidates.length === 0 && deepLocatorCandidates.length > 0) {
5232
- // A fill/select step must actuate the NAMED FIELD through the
5233
- // fill/select seam, never the click-only walk below: candidate
5234
- // ranking scores by the instruction's quoted VALUE (see
5235
- // parseFillStep's docblock) for "Fill in the First Name field
5236
- // with 'Reginald'" no control's accessible name is ever 'Reginald',
5237
- // so every candidate ties at score 0 and the click walk would fire
5238
- // on whatever sits first in DOM order (the bug report's wizard
5239
- // 'Close' mis-click). Detect that shape first and route
5240
- // deterministically to the candidate matching the field's own
5241
- // label, refusing to click when no candidate names that field.
5242
- const fillStep = parseFillStep(step);
5243
- const selectStep = fillStep ? null : parseSelectStep(step);
5244
- const fieldTarget = fillStep
5245
- ? { kind: "fill", fieldLabel: fillStep.fieldLabel, value: fillStep.value }
5246
- : selectStep?.questionLabel
5247
- ? { kind: "select", fieldLabel: selectStep.questionLabel, value: selectStep.option }
5248
- : null;
5379
+ // Field-label routing for a fill/select-shaped step already ran
5380
+ // (and returned/continued on a MATCH) in the field-label-first
5381
+ // pre-check above, before this attempt's own act/observe call —
5382
+ // see `tryDeterministicFieldLabelActuation`. Reaching this
5383
+ // `if (fieldTarget)` means the pre-check found no candidate
5384
+ // naming the field either the walk below scores by the
5385
+ // instruction's quoted VALUE (see parseFillStep's docblock), so
5386
+ // every candidate ties at score 0 and it would otherwise click
5387
+ // whichever sits first in DOM order (the bug report's wizard
5388
+ // 'Close' mis-click). Refuse instead of guessing.
5389
+ const fieldTarget = parseFieldLabelTarget(step);
5249
5390
  if (fieldTarget) {
5250
- const matched = findDeepLocatorCandidateByFieldLabel(deepLocatorCandidates, fieldTarget.fieldLabel);
5251
- if (!matched) {
5252
- const failureMessage = `deepLocator: no candidate matched field "${fieldTarget.fieldLabel}" (refusing to click an unrelated control)`;
5253
- record.actResultSuccess = false;
5254
- record.errorMessage = failureMessage;
5255
- attempts.push(record);
5256
- failureReasons.push(failureMessage);
5257
- logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${failureMessage}`);
5258
- continue;
5259
- }
5260
- triedSelectors.push(matched.selector);
5261
- record.triedSelectors = [matched.selector];
5262
- const actuated = fieldTarget.kind === "fill"
5263
- ? await (0, deep_locator_actuate_1.fillDeepLocatorCandidate)(page, frameTarget?.frameSelector, deepLocatorInnerSelector, matched.index, fieldTarget.value, { frameTarget })
5264
- : await (0, deep_locator_actuate_1.selectDeepLocatorCandidateOption)(page, frameTarget?.frameSelector, deepLocatorInnerSelector, matched.index, fieldTarget.value, { frameTarget });
5265
- if (actuated) {
5266
- record.instruction = `deepLocator: ${matched.accessibleText || "(no accessible text)"}`;
5267
- record.actResultSuccess = true;
5268
- record.actResultDescription = `deepLocator ${fieldTarget.kind === "fill" ? "filled" : "selected"} "${matched.accessibleText || matched.selector}" with "${fieldTarget.value}"`;
5269
- // verifyDomEffect can't resolve a `deeplocator=` selector (see
5270
- // deep-locator-actuate.ts's module docblock: target.locator()
5271
- // has no meaning for cross-origin OOPIF hop notation) — the
5272
- // write + read-back fillDeepLocatorCandidate/
5273
- // selectDeepLocatorCandidateOption already performed IS the
5274
- // verification signal, so record it directly instead of
5275
- // synthesizing a resolvedAction that a verifier can only ever
5276
- // score false for.
5277
- record.verifiedBy = "dom";
5278
- attempts.push(record);
5279
- logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${record.actResultDescription}`);
5280
- trajectory?.push({ stepIndex, verifiedBy: "dom" });
5281
- return "completed";
5282
- }
5283
- const failureMessage = `deepLocator: ${fieldTarget.kind} on "${matched.accessibleText || matched.selector}" did not verify (read-back mismatch or rejected write)`;
5391
+ const failureMessage = `deepLocator: no candidate matched field "${fieldTarget.fieldLabel}" (refusing to click an unrelated control)`;
5284
5392
  record.actResultSuccess = false;
5285
5393
  record.errorMessage = failureMessage;
5286
5394
  attempts.push(record);
@@ -5288,6 +5396,7 @@ async function executeStepWithHealing(params) {
5288
5396
  logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${failureMessage}`);
5289
5397
  continue;
5290
5398
  }
5399
+ const selectStep = parseSelectStep(step);
5291
5400
  // A select step whose question is phrased un-quoted (parseSelectStep
5292
5401
  // returns `questionLabel: null`) has no fieldLabel to route through
5293
5402
  // findDeepLocatorCandidateByFieldLabel above, so ranking below falls
@@ -5671,8 +5780,8 @@ async function executeStepWithHealing(params) {
5671
5780
  }
5672
5781
  else {
5673
5782
  record.technique = "llm-rephrase";
5674
- if (!anthropic) {
5675
- record.errorMessage = "no anthropic client (bedrock-only deployment); skipping rephrase";
5783
+ if (!rephraseModel) {
5784
+ record.errorMessage = "no rephrase model configured; skipping rephrase";
5676
5785
  }
5677
5786
  else if (hasBillingErrorBeenLogged()) {
5678
5787
  // Telemetry-driven skip: when Anthropic billing has already been
@@ -5720,7 +5829,7 @@ async function executeStepWithHealing(params) {
5720
5829
  instruction: a.instruction,
5721
5830
  verdict: a.errorMessage ?? failureReasons[i] ?? null,
5722
5831
  }));
5723
- const rephrased = await rephraseWithLLM(anthropic, step, triedSelectors, candidates, failureReasons, captureFn, livePageEvidence, unfocused, submitFailureList, priorAttemptsForPrompt, gaEventList);
5832
+ const rephrased = await rephraseWithLLM(rephraseModel, step, triedSelectors, candidates, failureReasons, captureFn, livePageEvidence, unfocused, submitFailureList, priorAttemptsForPrompt, gaEventList, anthropic);
5724
5833
  if (!rephrased) {
5725
5834
  record.errorMessage = "llm declined to rephrase or returned outcome=impossible";
5726
5835
  }
@@ -5839,7 +5948,31 @@ async function executeStepWithHealing(params) {
5839
5948
  if (domVerified && !domVerifiedForStep) {
5840
5949
  logger.info(`${formatStepPrefix(stepIndex, totalSteps)} advance step succeeded only via DOM state change (field toggle / non-advancing POST), not a real transition; not treating as verified`);
5841
5950
  }
5842
- let verified = networkIsRealAdvance || urlChanged || domVerifiedForStep;
5951
+ // Client-side view-swap gate: credit a click that produces substantial
5952
+ // DOM growth (≥5KB) with zero network when it's NOT a submit/final step
5953
+ // and NOT an advance-pattern step. Fixes the UCHealth "Manual Application"
5954
+ // case where a +49KB DOM-only view swap was scored as "no observable effect".
5955
+ const clickViewSwapVerified = isClickViewSwapVerified({
5956
+ resolvedAction,
5957
+ isFinalStep,
5958
+ submitStep,
5959
+ isAdvanceWithPattern: isAdvanceStep(step) && advanceTransitionBodyPattern !== null,
5960
+ networkDelta: post.networkCount - pre.networkCount,
5961
+ bytesDelta: post.bodyHtmlLength - pre.bodyHtmlLength,
5962
+ });
5963
+ // Form-value-diff signal: `visibleTextSignature` never reflects a plain
5964
+ // <input>/<textarea>/<select>'s `value` property, so a fill with no
5965
+ // secondary UI side effect (no toggle, no formatted display) had ZERO
5966
+ // verification signal — every cascade attempt reported "no observable
5967
+ // effect" even though the value genuinely landed. Scoped to state-class
5968
+ // actions (fill/check/etc.), same as domVerified, so it never lets an
5969
+ // advance/submit step ride a stray value mutation elsewhere on the page.
5970
+ const formValueVerified = isStateClass && post.formValueSignature !== pre.formValueSignature;
5971
+ let verified = networkIsRealAdvance ||
5972
+ urlChanged ||
5973
+ domVerifiedForStep ||
5974
+ clickViewSwapVerified ||
5975
+ formValueVerified;
5843
5976
  // Final-step submit-verification gate. Replaces the deterministic
5844
5977
  // submitEndpointPattern regex with a Haiku 4.5 LLM judgment over
5845
5978
  // multi-signal evidence (network captures, page URL/title, DOM
@@ -5952,7 +6085,17 @@ async function executeStepWithHealing(params) {
5952
6085
  if (verified && record.verifiedBy === null) {
5953
6086
  // Preserve a richer verdict set earlier in this attempt (e.g.
5954
6087
  // "submitted-state-dom" from the final-step DOM fallback).
5955
- record.verifiedBy = urlChanged ? "url" : networkFired ? "network" : "dom";
6088
+ record.verifiedBy = clickViewSwapVerified
6089
+ ? "view-swap"
6090
+ : urlChanged
6091
+ ? "url"
6092
+ : networkFired
6093
+ ? "network"
6094
+ : domVerifiedForStep
6095
+ ? "dom"
6096
+ : formValueVerified
6097
+ ? "form-value"
6098
+ : "dom";
5956
6099
  }
5957
6100
  // N+16 probe: Stagehand's CDP click sometimes lands on the button without
5958
6101
  // triggering React's SyntheticEvent layer (or jQuery delegated handlers).
@@ -6039,6 +6182,7 @@ async function executeStepWithHealing(params) {
6039
6182
  const retryUrlChanged = retryPost.url !== pre.url;
6040
6183
  const retryHtmlDelta = retryPost.bodyHtmlLength - pre.bodyHtmlLength;
6041
6184
  const retryTextChanged = retryPost.visibleTextSignature !== pre.visibleTextSignature;
6185
+ const retryFormValueChanged = retryPost.formValueSignature !== pre.formValueSignature;
6042
6186
  // RC2: an advance/`kind=click` "Next" that only grew the DOM
6043
6187
  // (validation errors rendered) with NO network/URL change is a
6044
6188
  // validation-blocked no-op, not a real transition — but the
@@ -6094,6 +6238,7 @@ async function executeStepWithHealing(params) {
6094
6238
  retryUrlChanged ||
6095
6239
  retryHtmlDelta !== 0 ||
6096
6240
  retryTextChanged ||
6241
+ retryFormValueChanged ||
6097
6242
  checkboxStateVerified);
6098
6243
  // Apply the same submit-endpoint gate the primary verifier uses.
6099
6244
  // Without this, the n+16 fallback would still ride past a
@@ -6167,7 +6312,7 @@ async function executeStepWithHealing(params) {
6167
6312
  failureReasons.push(`n+16 fallback: submit-judge-rejected: ${judgeVerdict.reason}`);
6168
6313
  }
6169
6314
  }
6170
- logger.info(`n+16 probe: step=${stepIndex + 1}/${totalSteps?.() ?? "?"} attempt=${attempt} el.click() fallback fired=${fired === true} kind=${probeResult.kind ?? "none"} checkboxStateVerified=${checkboxStateVerified} ancestorStillInvalid=${ancestorStillInvalid}; network=${retryNetworkFired} url=${retryUrlChanged} htmlDelta=${retryHtmlDelta} textChanged=${retryTextChanged} verified=${retryVerified}`);
6315
+ logger.info(`n+16 probe: step=${stepIndex + 1}/${totalSteps?.() ?? "?"} attempt=${attempt} el.click() fallback fired=${fired === true} kind=${probeResult.kind ?? "none"} checkboxStateVerified=${checkboxStateVerified} ancestorStillInvalid=${ancestorStillInvalid}; network=${retryNetworkFired} url=${retryUrlChanged} htmlDelta=${retryHtmlDelta} textChanged=${retryTextChanged} formValueChanged=${retryFormValueChanged} verified=${retryVerified}`);
6171
6316
  if (retryVerified) {
6172
6317
  if (record.verifiedBy === null) {
6173
6318
  record.verifiedBy = retryUrlChanged ? "url" : retryNetworkFired ? "network" : "dom";
@@ -6432,7 +6577,7 @@ async function waitForSpaReady(page, logger, opts = {}) {
6432
6577
  * than let the flow report phantom success.
6433
6578
  */
6434
6579
  async function runHealingFlow(deps) {
6435
- const { stagehand, page, steps, logger, anthropic, resumeFixture, maxFlowMs } = deps;
6580
+ const { stagehand, page, steps, logger, anthropic, rephraseModel, uploadFixture, maxFlowMs } = deps;
6436
6581
  const counter = { n: 0 };
6437
6582
  const signalCounter = { n: 0 };
6438
6583
  const recentCaptures = [];
@@ -6484,8 +6629,9 @@ async function runHealingFlow(deps) {
6484
6629
  recentCaptures,
6485
6630
  recentCaptureMeta,
6486
6631
  anthropic,
6632
+ rephraseModel,
6487
6633
  logger,
6488
- resumeFixture,
6634
+ uploadFixture,
6489
6635
  frameTarget,
6490
6636
  isFinalStep: i === steps.length - 1,
6491
6637
  submitEndpointPattern: deps.submitEndpointPattern ?? null,