@enricai/barnacle 1.6.14 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/api/schemas/submissions.d.ts +8 -8
- package/dist/lib/bedrock.d.ts +2 -2
- package/dist/lib/bedrock.d.ts.map +1 -1
- package/dist/lib/bedrock.js.map +1 -1
- package/dist/lib/llm/anthropic-client.d.ts +12 -0
- package/dist/lib/llm/anthropic-client.d.ts.map +1 -1
- package/dist/lib/llm/anthropic-client.js +25 -0
- package/dist/lib/llm/anthropic-client.js.map +1 -1
- package/dist/lib/telemetry/beacon-capture.d.ts +4 -3
- package/dist/lib/telemetry/beacon-capture.d.ts.map +1 -1
- package/dist/lib/telemetry/beacon-capture.js.map +1 -1
- package/dist/lib/telemetry/reconciliation-record.d.ts +15 -33
- package/dist/lib/telemetry/reconciliation-record.d.ts.map +1 -1
- package/dist/lib/telemetry/reconciliation-record.js +11 -49
- package/dist/lib/telemetry/reconciliation-record.js.map +1 -1
- package/dist/lib/telemetry/submission-capture.d.ts +8 -13
- package/dist/lib/telemetry/submission-capture.d.ts.map +1 -1
- package/dist/lib/telemetry/submission-capture.js +2 -4
- package/dist/lib/telemetry/submission-capture.js.map +1 -1
- package/dist/lib/telemetry/submission-reader.d.ts +3 -4
- package/dist/lib/telemetry/submission-reader.d.ts.map +1 -1
- package/dist/lib/telemetry/submission-reader.js +3 -4
- package/dist/lib/telemetry/submission-reader.js.map +1 -1
- package/dist/plugins/config-plugin.d.ts.map +1 -1
- package/dist/plugins/config-plugin.js +22 -0
- package/dist/plugins/config-plugin.js.map +1 -1
- package/dist/plugins/loader.d.ts +0 -7
- package/dist/plugins/loader.d.ts.map +1 -1
- package/dist/plugins/loader.js +0 -9
- package/dist/plugins/loader.js.map +1 -1
- package/dist/scraper/deep-locator-actuate.d.ts +24 -19
- package/dist/scraper/deep-locator-actuate.d.ts.map +1 -1
- package/dist/scraper/deep-locator-actuate.js +68 -23
- package/dist/scraper/deep-locator-actuate.js.map +1 -1
- package/dist/scraper/deep-locator-scan.d.ts +20 -0
- package/dist/scraper/deep-locator-scan.d.ts.map +1 -1
- package/dist/scraper/deep-locator-scan.js +25 -0
- package/dist/scraper/deep-locator-scan.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts +43 -7
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +174 -84
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scripts/recon-browser.d.ts +5 -4
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +8 -5
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +2 -2
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +18 -85
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/scripts/recon-shared.d.ts +0 -10
- package/dist/scripts/recon-shared.d.ts.map +1 -1
- package/dist/scripts/recon-shared.js +0 -11
- package/dist/scripts/recon-shared.js.map +1 -1
- package/dist/site-plugin.d.ts +3 -3
- package/dist/testing/coverage-guard-suite.d.mts +7 -7
- package/dist/testing/coverage-guard-suite.mjs +5 -5
- package/package.json +3 -1
- package/dist/scripts/migrate-telemetry-dir-names.d.ts +0 -57
- package/dist/scripts/migrate-telemetry-dir-names.d.ts.map +0 -1
- package/dist/scripts/migrate-telemetry-dir-names.js +0 -156
- package/dist/scripts/migrate-telemetry-dir-names.js.map +0 -1
|
@@ -77,7 +77,7 @@ const node_crypto_1 = require("node:crypto");
|
|
|
77
77
|
const node_fs_1 = require("node:fs");
|
|
78
78
|
const node_os_1 = require("node:os");
|
|
79
79
|
const node_path_1 = require("node:path");
|
|
80
|
-
const
|
|
80
|
+
const ai_1 = require("ai");
|
|
81
81
|
const config_1 = require("../config");
|
|
82
82
|
const errors_1 = require("../lib/errors");
|
|
83
83
|
const error_messages_1 = require("../lib/llm/judges/error-messages");
|
|
@@ -480,7 +480,7 @@ exports.TRAILING_GRACE_WINDOW = 2;
|
|
|
480
480
|
* means no injection surface. Runs in browser context and returns a typed-narrow
|
|
481
481
|
* shape via Runtime.callFunctionOn.
|
|
482
482
|
*/
|
|
483
|
-
const DOM_SNAPSHOT_EXPR = `(() => { const b = document.body; if (!b) return { html: 0, text: "" }; const t = b.innerText || ""; return { html: (b.outerHTML || "").length, text: t.length + ":" + t.slice(0, 200) }; })()`;
|
|
483
|
+
const DOM_SNAPSHOT_EXPR = `(() => { const b = document.body; if (!b) return { html: 0, text: "", values: "" }; const t = b.innerText || ""; const controls = Array.from(b.querySelectorAll("input, textarea, select")).filter((el) => el.offsetParent !== null); const values = controls.map((el) => { if (el.type === "checkbox" || el.type === "radio") return el.checked ? "1" : "0"; return el.value || ""; }).join("|").slice(0, 2000); return { html: (b.outerHTML || "").length, text: t.length + ":" + t.slice(0, 200), values }; })()`;
|
|
484
484
|
/**
|
|
485
485
|
* Captures the pre/post signal triple the submit-verify cascade diffs.
|
|
486
486
|
* Accepts the optional `page` so a resolved child `FrameTarget` whose
|
|
@@ -491,6 +491,7 @@ const DOM_SNAPSHOT_EXPR = `(() => { const b = document.body; if (!b) return { ht
|
|
|
491
491
|
async function snapshotPage(target, signalCounter, page) {
|
|
492
492
|
let bodyHtmlLength = 0;
|
|
493
493
|
let visibleTextSignature = "";
|
|
494
|
+
let formValueSignature = "";
|
|
494
495
|
try {
|
|
495
496
|
const result = await target.evaluate(DOM_SNAPSHOT_EXPR);
|
|
496
497
|
if (result !== null &&
|
|
@@ -501,6 +502,8 @@ async function snapshotPage(target, signalCounter, page) {
|
|
|
501
502
|
typeof result.text === "string") {
|
|
502
503
|
bodyHtmlLength = result.html;
|
|
503
504
|
visibleTextSignature = result.text;
|
|
505
|
+
const rawValues = result.values;
|
|
506
|
+
formValueSignature = typeof rawValues === "string" ? rawValues : "";
|
|
504
507
|
}
|
|
505
508
|
}
|
|
506
509
|
catch {
|
|
@@ -519,6 +522,7 @@ async function snapshotPage(target, signalCounter, page) {
|
|
|
519
522
|
url,
|
|
520
523
|
bodyHtmlLength,
|
|
521
524
|
visibleTextSignature,
|
|
525
|
+
formValueSignature,
|
|
522
526
|
};
|
|
523
527
|
}
|
|
524
528
|
/**
|
|
@@ -972,6 +976,7 @@ function describeAttemptEffectSignals(pre, post, recentCaptureMeta, preMetaLengt
|
|
|
972
976
|
const bytesDelta = post.bodyHtmlLength - pre.bodyHtmlLength;
|
|
973
977
|
const networkDelta = post.networkCount - pre.networkCount;
|
|
974
978
|
const textChanged = post.visibleTextSignature !== pre.visibleTextSignature;
|
|
979
|
+
const valueChanged = post.formValueSignature !== pre.formValueSignature;
|
|
975
980
|
const windowCaptures = recentCaptureMeta.slice(preMetaLength);
|
|
976
981
|
const observations = [];
|
|
977
982
|
if (networkDelta === 0 && bytesDelta >= 500) {
|
|
@@ -986,6 +991,9 @@ function describeAttemptEffectSignals(pre, post, recentCaptureMeta, preMetaLengt
|
|
|
986
991
|
if (textChanged && networkDelta === 0 && bytesDelta < 500) {
|
|
987
992
|
observations.push("visible-text-changed-without-network: page reflowed slightly (likely tooltip/focus state)");
|
|
988
993
|
}
|
|
994
|
+
if (valueChanged && !textChanged && networkDelta === 0) {
|
|
995
|
+
observations.push("form-value-changed-without-network: a visible input/textarea/select value changed with no other observable effect (plain field fill)");
|
|
996
|
+
}
|
|
989
997
|
return observations.join(" | ");
|
|
990
998
|
}
|
|
991
999
|
/**
|
|
@@ -1036,7 +1044,7 @@ function shouldSkipTechnique(params) {
|
|
|
1036
1044
|
// structured-click / observe-act-exclude re-resolves the SAME
|
|
1037
1045
|
// light-DOM-only view of the page and would no-op identically, so skip
|
|
1038
1046
|
// straight to deep-submit-locator (attempt 2) instead. llm-rephrase
|
|
1039
|
-
// (attempt
|
|
1047
|
+
// (attempt 4) is never skipped — a differently-worded instruction is still
|
|
1040
1048
|
// a distinct attempt worth trying if the deep locator also fails. Gated on
|
|
1041
1049
|
// submitShapedStep: the deep submit-control locator ranks submit-shaped
|
|
1042
1050
|
// candidates only, so on a non-submit control (e.g. a radio/checkbox) it
|
|
@@ -1143,8 +1151,8 @@ function isAdvanceStalled(params) {
|
|
|
1143
1151
|
return networkFired;
|
|
1144
1152
|
}
|
|
1145
1153
|
/**
|
|
1146
|
-
* Lazy Anthropic client for attempt
|
|
1147
|
-
* deployment is Bedrock-only (no ANTHROPIC_API_KEY) — attempt
|
|
1154
|
+
* Lazy Anthropic client for attempt 5's rephrase. Returns null when the
|
|
1155
|
+
* deployment is Bedrock-only (no ANTHROPIC_API_KEY) — attempt 5 then becomes
|
|
1148
1156
|
* a no-op and the executor escalates straight to the failure dump. The other
|
|
1149
1157
|
* three attempts already cover the lion's share of recovery.
|
|
1150
1158
|
*/
|
|
@@ -1165,7 +1173,7 @@ function anthropicModelName() {
|
|
|
1165
1173
|
*/
|
|
1166
1174
|
const REPHRASE_SYSTEM_PROMPT = "You rewrite a failed Stagehand act() instruction so the next attempt targets a different element than those already tried. If no different element exists on the page that could plausibly unblock the flow, return outcome=impossible. Never re-propose a selector or wording from the lists labeled SELECTORS ALREADY TRIED or INSTRUCTION TEXT ALREADY TRIED.";
|
|
1167
1175
|
/**
|
|
1168
|
-
* Attempt-
|
|
1176
|
+
* Attempt-5 of the step-healing cascade: when three mechanical retry variations
|
|
1169
1177
|
* all fail, this is the last resort before the step is declared terminal. Exported
|
|
1170
1178
|
* so tests can inject a fake capture sink without touching the browser session.
|
|
1171
1179
|
*/
|
|
@@ -1219,13 +1227,21 @@ priorAttempts,
|
|
|
1219
1227
|
* "click harder" into "fill the actually-missing field." Telemetry
|
|
1220
1228
|
* the engine already captures but never showed the LLM until now.
|
|
1221
1229
|
*/
|
|
1222
|
-
gaEventEvidence
|
|
1230
|
+
gaEventEvidence,
|
|
1231
|
+
/**
|
|
1232
|
+
* Anthropic client for the modal-priority judge inside
|
|
1233
|
+
* `renderUnfocusedObserve`. Separate from `client` (the ai-SDK model used
|
|
1234
|
+
* for the rephrase call itself) because that judge pipeline is unrelated
|
|
1235
|
+
* to which provider generates the rephrase, and null on a Bedrock-only
|
|
1236
|
+
* deployment falls back to `renderUnfocusedObserve`'s source-order default.
|
|
1237
|
+
*/
|
|
1238
|
+
judgeClient) {
|
|
1223
1239
|
const candidateList = observeCandidates
|
|
1224
1240
|
.slice(0, 12)
|
|
1225
1241
|
.map((a, i) => `${i + 1}. ${a.description} — ${a.selector}`)
|
|
1226
1242
|
.join("\n");
|
|
1227
1243
|
const unfocusedList = unfocusedObserve
|
|
1228
|
-
? await renderUnfocusedObserve(unfocusedObserve, { client, captureFn })
|
|
1244
|
+
? await renderUnfocusedObserve(unfocusedObserve, { client: judgeClient, captureFn })
|
|
1229
1245
|
: "";
|
|
1230
1246
|
const triedList = triedSelectors.length > 0 ? triedSelectors.join("\n") : "(none)";
|
|
1231
1247
|
const reasonList = failureReasons.map((r, i) => `attempt ${i + 1}: ${r}`).join("\n");
|
|
@@ -1297,35 +1313,28 @@ PAGE TRANSITION + VALIDATOR TELEMETRY (parsed from Google Analytics Measurement
|
|
|
1297
1313
|
${gaEventEvidence || "(none)"}
|
|
1298
1314
|
|
|
1299
1315
|
Rewrite the instruction so a Stagehand act() call can resolve it unambiguously to a different element than the ones already tried. Keep it short — one sentence, natural language, no quotes around it. If the invalid-fields section above names a field AND the interactive-targets section below lists clickable options inside that same container, your rewrite picks one of those options (e.g. for a yes/no question rendered as two radio labels, choose the candidate-favorable answer — 'No' for "do you have a non-compete", 'Yes' for "are you authorized to work", 'Prefer not to say' for demographic disclosures) rather than retrying the original click. If the unfocused observe section shows an open modal/dialog with a Save or Close action, your rewrite invokes that action so the underlying form can clear its blocking state. Set outcome to "impossible" with a one-line reason only when the original instruction's element does not exist on the current page and no redirect target is available.`;
|
|
1300
|
-
const model =
|
|
1316
|
+
const model = client.modelId;
|
|
1301
1317
|
const t0 = performance.now();
|
|
1302
1318
|
try {
|
|
1303
|
-
const response = await
|
|
1304
|
-
model,
|
|
1305
|
-
|
|
1319
|
+
const response = await (0, ai_1.generateObject)({
|
|
1320
|
+
model: client,
|
|
1321
|
+
maxOutputTokens: 400,
|
|
1306
1322
|
system: REPHRASE_SYSTEM_PROMPT,
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
format: (0, zod_1.zodOutputFormat)(schemas_1.REPHRASE_RESPONSE_SCHEMA),
|
|
1310
|
-
},
|
|
1323
|
+
prompt,
|
|
1324
|
+
schema: schemas_1.REPHRASE_RESPONSE_SCHEMA,
|
|
1311
1325
|
});
|
|
1312
1326
|
const latencyMs = performance.now() - t0;
|
|
1313
|
-
const parsed = response.
|
|
1314
|
-
if (parsed === null) {
|
|
1315
|
-
throw new Error("structured-output enabled but parsed_output is null");
|
|
1316
|
-
}
|
|
1317
|
-
const textBlock = response.content.find((b) => b.type === "text");
|
|
1318
|
-
const rawText = textBlock?.type === "text" ? textBlock.text : "";
|
|
1327
|
+
const parsed = response.object;
|
|
1319
1328
|
await captureFn({
|
|
1320
1329
|
callId: (0, node_crypto_1.randomUUID)(),
|
|
1321
1330
|
callType: call_types_1.CALL_TYPE_RECON_REPHRASE,
|
|
1322
1331
|
model,
|
|
1323
1332
|
systemPrompt: REPHRASE_SYSTEM_PROMPT,
|
|
1324
1333
|
userContent: prompt,
|
|
1325
|
-
responseContent:
|
|
1334
|
+
responseContent: JSON.stringify(parsed),
|
|
1326
1335
|
parsedOk: true,
|
|
1327
|
-
inputTokens: response.usage
|
|
1328
|
-
outputTokens: response.usage
|
|
1336
|
+
inputTokens: response.usage.inputTokens ?? null,
|
|
1337
|
+
outputTokens: response.usage.outputTokens ?? null,
|
|
1329
1338
|
latencyMs,
|
|
1330
1339
|
success: true,
|
|
1331
1340
|
errorMessage: null,
|
|
@@ -2843,6 +2852,24 @@ function resolveDeepLocatorActuation(step) {
|
|
|
2843
2852
|
function normalizeFieldLabel(text) {
|
|
2844
2853
|
return text.replace(/\s+/g, " ").trim().toLowerCase();
|
|
2845
2854
|
}
|
|
2855
|
+
/**
|
|
2856
|
+
* Parses a fill/select step into the field it names plus the value to write,
|
|
2857
|
+
* or `null` for any other step shape (click, a select whose question is
|
|
2858
|
+
* un-quoted, etc). Shared by both the observe-empty deepLocator fallback and
|
|
2859
|
+
* the field-label-first routing ahead of Stagehand's own act()/observe()
|
|
2860
|
+
* resolution — see `findDeepLocatorCandidateByFieldLabel`'s docblock for why
|
|
2861
|
+
* the field LABEL (not the value) is what a caller matches candidates
|
|
2862
|
+
* against.
|
|
2863
|
+
*/
|
|
2864
|
+
function parseFieldLabelTarget(step) {
|
|
2865
|
+
const fillStep = parseFillStep(step);
|
|
2866
|
+
if (fillStep)
|
|
2867
|
+
return { kind: "fill", fieldLabel: fillStep.fieldLabel, value: fillStep.value };
|
|
2868
|
+
const selectStep = parseSelectStep(step);
|
|
2869
|
+
return selectStep?.questionLabel
|
|
2870
|
+
? { kind: "select", fieldLabel: selectStep.questionLabel, value: selectStep.option }
|
|
2871
|
+
: null;
|
|
2872
|
+
}
|
|
2846
2873
|
/**
|
|
2847
2874
|
* Finds the candidate whose accessible name identifies the named field —
|
|
2848
2875
|
* `fieldLabel` (from {@link parseFillStep}/{@link parseSelectStep}), NOT the
|
|
@@ -4564,6 +4591,36 @@ async function resolveDeepLocatorCandidatesWithWidening(page, frameSelector, ins
|
|
|
4564
4591
|
const widened = await (0, deep_locator_candidates_1.resolveDeepLocatorCandidates)(page, frameSelector, "*", instruction, timeoutOptions);
|
|
4565
4592
|
return { candidates: widened, innerSelector: "*" };
|
|
4566
4593
|
}
|
|
4594
|
+
/**
|
|
4595
|
+
* Resolves and actuates a fill/select step's NAMED FIELD through the
|
|
4596
|
+
* deterministic accessible-name match (`findDeepLocatorCandidateByFieldLabel`)
|
|
4597
|
+
* against the frame's own deepLocator scan, independent of what Stagehand's
|
|
4598
|
+
* `act()`/`observe()` resolved. A same-frame accessible-name label match
|
|
4599
|
+
* cannot confuse a City input for an unrelated header search box the way
|
|
4600
|
+
* Stagehand's semantic resolution can — Stagehand's own candidate is only a
|
|
4601
|
+
* fallback for the caller to use when this returns `no-match` or
|
|
4602
|
+
* `not-fill-or-select`, never the first attempt for a step that names a
|
|
4603
|
+
* field. `triedSelectors` (already-attempted selectors from prior
|
|
4604
|
+
* cascade attempts) is excluded so a step that already tried and failed on
|
|
4605
|
+
* this candidate doesn't just re-pick it.
|
|
4606
|
+
*/
|
|
4607
|
+
async function tryDeterministicFieldLabelActuation(params) {
|
|
4608
|
+
const { page, frameTarget, step, triedSelectors, timeoutOptions } = params;
|
|
4609
|
+
const fieldTarget = parseFieldLabelTarget(step);
|
|
4610
|
+
if (!fieldTarget)
|
|
4611
|
+
return { kind: "not-fill-or-select" };
|
|
4612
|
+
const { candidates, innerSelector } = await resolveDeepLocatorCandidatesWithWidening(page, frameTarget.frameSelector, step, timeoutOptions);
|
|
4613
|
+
const unexcluded = candidates.filter((c) => !triedSelectors.includes(c.selector));
|
|
4614
|
+
const matched = findDeepLocatorCandidateByFieldLabel(unexcluded, fieldTarget.fieldLabel);
|
|
4615
|
+
if (!matched)
|
|
4616
|
+
return { kind: "no-match", fieldTarget };
|
|
4617
|
+
const actuated = fieldTarget.kind === "fill"
|
|
4618
|
+
? await (0, deep_locator_actuate_1.fillDeepLocatorCandidate)(page, frameTarget.frameSelector, innerSelector, matched.index, fieldTarget.value, { frameTarget })
|
|
4619
|
+
: await (0, deep_locator_actuate_1.selectDeepLocatorCandidateOption)(page, frameTarget.frameSelector, innerSelector, matched.index, fieldTarget.value, { frameTarget });
|
|
4620
|
+
return actuated
|
|
4621
|
+
? { kind: "actuated", matched, fieldTarget }
|
|
4622
|
+
: { kind: "actuation-failed", matched, fieldTarget };
|
|
4623
|
+
}
|
|
4567
4624
|
/**
|
|
4568
4625
|
* Adapts `resolveDeepLocatorCandidatesWithWidening` results into
|
|
4569
4626
|
* `Action`-shaped evidence for `rephraseWithLLM`, which only knows about
|
|
@@ -4674,7 +4731,7 @@ async function probeStepBeforeAttempts(params) {
|
|
|
4674
4731
|
}
|
|
4675
4732
|
}
|
|
4676
4733
|
async function executeStepWithHealing(params) {
|
|
4677
|
-
const { stagehand, page, step, optional, upload, submitStep, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
|
|
4734
|
+
const { stagehand, page, step, optional, upload, submitStep, stepIndex, totalSteps, phase, signalCounter, recentCaptures, recentCaptureMeta, anthropic, rephraseModel, logger, captureFn, uploadFixture, isFinalStep, submitEndpointPattern, submittedStateSelectors, requireSubmitEndpointMatch, advanceTransitionBodyPattern, successUrlFragments, successPageTitleHints, ownBackendHostnames, knownErrorClassPrefixes, wizardExitButtonLabels, getSuppressedAisdkElementIdErrorCount, trajectory, onStepFailure, } = params;
|
|
4678
4735
|
// Mutable (not the destructured const above) so a lost frame-attach race
|
|
4679
4736
|
// can be upgraded in place once the OOPIF attaches later in the cascade —
|
|
4680
4737
|
// see reresolveFrameTargetIfLost below. Every existing reference in this
|
|
@@ -4988,6 +5045,7 @@ async function executeStepWithHealing(params) {
|
|
|
4988
5045
|
url: page.url(),
|
|
4989
5046
|
bodyHtmlLength: 0,
|
|
4990
5047
|
visibleTextSignature: "",
|
|
5048
|
+
formValueSignature: "",
|
|
4991
5049
|
};
|
|
4992
5050
|
attempts.push({
|
|
4993
5051
|
attempt: 0,
|
|
@@ -5089,6 +5147,57 @@ async function executeStepWithHealing(params) {
|
|
|
5089
5147
|
// from DOM re-read. Captured here so both branches of the cascade write to it.
|
|
5090
5148
|
let resolvedAction = null;
|
|
5091
5149
|
try {
|
|
5150
|
+
// Field-label-first routing: for a frame-scoped fill/select step,
|
|
5151
|
+
// Stagehand's own act()/observe() can resolve a real, in-DOM, plausible-
|
|
5152
|
+
// looking control that is nonetheless the WRONG one (a City field
|
|
5153
|
+
// resolving to an unrelated header search box whose accessible name
|
|
5154
|
+
// overlaps semantically with "city/location" is the motivating case).
|
|
5155
|
+
// A same-frame accessible-name match against the field's own label
|
|
5156
|
+
// can't make that mistake, so it's tried BEFORE any Stagehand
|
|
5157
|
+
// act/observe call for every attempt, not only once observe() has
|
|
5158
|
+
// already come back empty (bugfix-001's finding: observe() returning a
|
|
5159
|
+
// wrong-but-nonempty candidate was exactly what let the phantom-fill
|
|
5160
|
+
// through the old empty-candidates-only gate). Falls through to the
|
|
5161
|
+
// existing Stagehand-driven attempt when no candidate names the field
|
|
5162
|
+
// at all — this only preempts Stagehand's resolution, it never expands
|
|
5163
|
+
// what counts as a match.
|
|
5164
|
+
const fieldLabelOutcome = attempt !== 3 && frameTarget?.frame
|
|
5165
|
+
? await tryDeterministicFieldLabelActuation({
|
|
5166
|
+
page,
|
|
5167
|
+
frameTarget,
|
|
5168
|
+
step,
|
|
5169
|
+
triedSelectors,
|
|
5170
|
+
timeoutOptions: { frameTarget },
|
|
5171
|
+
})
|
|
5172
|
+
: { kind: "not-fill-or-select" };
|
|
5173
|
+
if (fieldLabelOutcome.kind === "actuated") {
|
|
5174
|
+
record.technique = "deep-locator-field-label";
|
|
5175
|
+
record.instruction = `deepLocator: ${fieldLabelOutcome.matched.accessibleText || "(no accessible text)"}`;
|
|
5176
|
+
triedSelectors.push(fieldLabelOutcome.matched.selector);
|
|
5177
|
+
record.triedSelectors = [fieldLabelOutcome.matched.selector];
|
|
5178
|
+
record.actResultSuccess = true;
|
|
5179
|
+
record.actResultDescription = `deepLocator ${fieldLabelOutcome.fieldTarget.kind === "fill" ? "filled" : "selected"} "${fieldLabelOutcome.matched.accessibleText || fieldLabelOutcome.matched.selector}" with "${fieldLabelOutcome.fieldTarget.value}"`;
|
|
5180
|
+
// Same rationale as the pre-existing candidates===0 branch below:
|
|
5181
|
+
// the write + read-back the actuation helper already performed IS
|
|
5182
|
+
// the verification signal — no resolvedAction/network/url verifier
|
|
5183
|
+
// can score a `deeplocator=` selector.
|
|
5184
|
+
record.verifiedBy = "dom";
|
|
5185
|
+
attempts.push(record);
|
|
5186
|
+
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${record.actResultDescription}`);
|
|
5187
|
+
trajectory?.push({ stepIndex, verifiedBy: "dom" });
|
|
5188
|
+
return "completed";
|
|
5189
|
+
}
|
|
5190
|
+
if (fieldLabelOutcome.kind === "actuation-failed") {
|
|
5191
|
+
triedSelectors.push(fieldLabelOutcome.matched.selector);
|
|
5192
|
+
record.triedSelectors = [fieldLabelOutcome.matched.selector];
|
|
5193
|
+
const failureMessage = `deepLocator: ${fieldLabelOutcome.fieldTarget.kind} on "${fieldLabelOutcome.matched.accessibleText || fieldLabelOutcome.matched.selector}" did not verify (read-back mismatch or rejected write)`;
|
|
5194
|
+
record.actResultSuccess = false;
|
|
5195
|
+
record.errorMessage = failureMessage;
|
|
5196
|
+
attempts.push(record);
|
|
5197
|
+
failureReasons.push(failureMessage);
|
|
5198
|
+
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${failureMessage}`);
|
|
5199
|
+
continue;
|
|
5200
|
+
}
|
|
5092
5201
|
if (attempt === 1) {
|
|
5093
5202
|
record.technique = "act-string";
|
|
5094
5203
|
record.instruction = step;
|
|
@@ -5267,58 +5376,19 @@ async function executeStepWithHealing(params) {
|
|
|
5267
5376
|
const deepLocatorInnerSelector = deepLocatorResolution?.innerSelector ?? deep_locator_scan_1.INTERACTIVE_CANDIDATE_SELECTOR;
|
|
5268
5377
|
const deepLocatorCandidates = (deepLocatorResolution?.candidates ?? []).filter((c) => !triedSelectors.includes(c.selector));
|
|
5269
5378
|
if (candidates.length === 0 && deepLocatorCandidates.length > 0) {
|
|
5270
|
-
//
|
|
5271
|
-
//
|
|
5272
|
-
//
|
|
5273
|
-
//
|
|
5274
|
-
//
|
|
5275
|
-
//
|
|
5276
|
-
//
|
|
5277
|
-
//
|
|
5278
|
-
//
|
|
5279
|
-
//
|
|
5280
|
-
const
|
|
5281
|
-
const selectStep = fillStep ? null : parseSelectStep(step);
|
|
5282
|
-
const fieldTarget = fillStep
|
|
5283
|
-
? { kind: "fill", fieldLabel: fillStep.fieldLabel, value: fillStep.value }
|
|
5284
|
-
: selectStep?.questionLabel
|
|
5285
|
-
? { kind: "select", fieldLabel: selectStep.questionLabel, value: selectStep.option }
|
|
5286
|
-
: null;
|
|
5379
|
+
// Field-label routing for a fill/select-shaped step already ran
|
|
5380
|
+
// (and returned/continued on a MATCH) in the field-label-first
|
|
5381
|
+
// pre-check above, before this attempt's own act/observe call —
|
|
5382
|
+
// see `tryDeterministicFieldLabelActuation`. Reaching this
|
|
5383
|
+
// `if (fieldTarget)` means the pre-check found no candidate
|
|
5384
|
+
// naming the field either — the walk below scores by the
|
|
5385
|
+
// instruction's quoted VALUE (see parseFillStep's docblock), so
|
|
5386
|
+
// every candidate ties at score 0 and it would otherwise click
|
|
5387
|
+
// whichever sits first in DOM order (the bug report's wizard
|
|
5388
|
+
// 'Close' mis-click). Refuse instead of guessing.
|
|
5389
|
+
const fieldTarget = parseFieldLabelTarget(step);
|
|
5287
5390
|
if (fieldTarget) {
|
|
5288
|
-
const
|
|
5289
|
-
if (!matched) {
|
|
5290
|
-
const failureMessage = `deepLocator: no candidate matched field "${fieldTarget.fieldLabel}" (refusing to click an unrelated control)`;
|
|
5291
|
-
record.actResultSuccess = false;
|
|
5292
|
-
record.errorMessage = failureMessage;
|
|
5293
|
-
attempts.push(record);
|
|
5294
|
-
failureReasons.push(failureMessage);
|
|
5295
|
-
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${failureMessage}`);
|
|
5296
|
-
continue;
|
|
5297
|
-
}
|
|
5298
|
-
triedSelectors.push(matched.selector);
|
|
5299
|
-
record.triedSelectors = [matched.selector];
|
|
5300
|
-
const actuated = fieldTarget.kind === "fill"
|
|
5301
|
-
? await (0, deep_locator_actuate_1.fillDeepLocatorCandidate)(page, frameTarget?.frameSelector, deepLocatorInnerSelector, matched.index, fieldTarget.value, { frameTarget })
|
|
5302
|
-
: await (0, deep_locator_actuate_1.selectDeepLocatorCandidateOption)(page, frameTarget?.frameSelector, deepLocatorInnerSelector, matched.index, fieldTarget.value, { frameTarget });
|
|
5303
|
-
if (actuated) {
|
|
5304
|
-
record.instruction = `deepLocator: ${matched.accessibleText || "(no accessible text)"}`;
|
|
5305
|
-
record.actResultSuccess = true;
|
|
5306
|
-
record.actResultDescription = `deepLocator ${fieldTarget.kind === "fill" ? "filled" : "selected"} "${matched.accessibleText || matched.selector}" with "${fieldTarget.value}"`;
|
|
5307
|
-
// verifyDomEffect can't resolve a `deeplocator=` selector (see
|
|
5308
|
-
// deep-locator-actuate.ts's module docblock: target.locator()
|
|
5309
|
-
// has no meaning for cross-origin OOPIF hop notation) — the
|
|
5310
|
-
// write + read-back fillDeepLocatorCandidate/
|
|
5311
|
-
// selectDeepLocatorCandidateOption already performed IS the
|
|
5312
|
-
// verification signal, so record it directly instead of
|
|
5313
|
-
// synthesizing a resolvedAction that a verifier can only ever
|
|
5314
|
-
// score false for.
|
|
5315
|
-
record.verifiedBy = "dom";
|
|
5316
|
-
attempts.push(record);
|
|
5317
|
-
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${record.actResultDescription}`);
|
|
5318
|
-
trajectory?.push({ stepIndex, verifiedBy: "dom" });
|
|
5319
|
-
return "completed";
|
|
5320
|
-
}
|
|
5321
|
-
const failureMessage = `deepLocator: ${fieldTarget.kind} on "${matched.accessibleText || matched.selector}" did not verify (read-back mismatch or rejected write)`;
|
|
5391
|
+
const failureMessage = `deepLocator: no candidate matched field "${fieldTarget.fieldLabel}" (refusing to click an unrelated control)`;
|
|
5322
5392
|
record.actResultSuccess = false;
|
|
5323
5393
|
record.errorMessage = failureMessage;
|
|
5324
5394
|
attempts.push(record);
|
|
@@ -5326,6 +5396,7 @@ async function executeStepWithHealing(params) {
|
|
|
5326
5396
|
logger.info(`${formatStepPrefix(stepIndex, totalSteps)} attempt ${attempt}: ${failureMessage}`);
|
|
5327
5397
|
continue;
|
|
5328
5398
|
}
|
|
5399
|
+
const selectStep = parseSelectStep(step);
|
|
5329
5400
|
// A select step whose question is phrased un-quoted (parseSelectStep
|
|
5330
5401
|
// returns `questionLabel: null`) has no fieldLabel to route through
|
|
5331
5402
|
// findDeepLocatorCandidateByFieldLabel above, so ranking below falls
|
|
@@ -5709,8 +5780,8 @@ async function executeStepWithHealing(params) {
|
|
|
5709
5780
|
}
|
|
5710
5781
|
else {
|
|
5711
5782
|
record.technique = "llm-rephrase";
|
|
5712
|
-
if (!
|
|
5713
|
-
record.errorMessage = "no
|
|
5783
|
+
if (!rephraseModel) {
|
|
5784
|
+
record.errorMessage = "no rephrase model configured; skipping rephrase";
|
|
5714
5785
|
}
|
|
5715
5786
|
else if (hasBillingErrorBeenLogged()) {
|
|
5716
5787
|
// Telemetry-driven skip: when Anthropic billing has already been
|
|
@@ -5758,7 +5829,7 @@ async function executeStepWithHealing(params) {
|
|
|
5758
5829
|
instruction: a.instruction,
|
|
5759
5830
|
verdict: a.errorMessage ?? failureReasons[i] ?? null,
|
|
5760
5831
|
}));
|
|
5761
|
-
const rephrased = await rephraseWithLLM(
|
|
5832
|
+
const rephrased = await rephraseWithLLM(rephraseModel, step, triedSelectors, candidates, failureReasons, captureFn, livePageEvidence, unfocused, submitFailureList, priorAttemptsForPrompt, gaEventList, anthropic);
|
|
5762
5833
|
if (!rephrased) {
|
|
5763
5834
|
record.errorMessage = "llm declined to rephrase or returned outcome=impossible";
|
|
5764
5835
|
}
|
|
@@ -5889,7 +5960,19 @@ async function executeStepWithHealing(params) {
|
|
|
5889
5960
|
networkDelta: post.networkCount - pre.networkCount,
|
|
5890
5961
|
bytesDelta: post.bodyHtmlLength - pre.bodyHtmlLength,
|
|
5891
5962
|
});
|
|
5892
|
-
|
|
5963
|
+
// Form-value-diff signal: `visibleTextSignature` never reflects a plain
|
|
5964
|
+
// <input>/<textarea>/<select>'s `value` property, so a fill with no
|
|
5965
|
+
// secondary UI side effect (no toggle, no formatted display) had ZERO
|
|
5966
|
+
// verification signal — every cascade attempt reported "no observable
|
|
5967
|
+
// effect" even though the value genuinely landed. Scoped to state-class
|
|
5968
|
+
// actions (fill/check/etc.), same as domVerified, so it never lets an
|
|
5969
|
+
// advance/submit step ride a stray value mutation elsewhere on the page.
|
|
5970
|
+
const formValueVerified = isStateClass && post.formValueSignature !== pre.formValueSignature;
|
|
5971
|
+
let verified = networkIsRealAdvance ||
|
|
5972
|
+
urlChanged ||
|
|
5973
|
+
domVerifiedForStep ||
|
|
5974
|
+
clickViewSwapVerified ||
|
|
5975
|
+
formValueVerified;
|
|
5893
5976
|
// Final-step submit-verification gate. Replaces the deterministic
|
|
5894
5977
|
// submitEndpointPattern regex with a Haiku 4.5 LLM judgment over
|
|
5895
5978
|
// multi-signal evidence (network captures, page URL/title, DOM
|
|
@@ -6008,7 +6091,11 @@ async function executeStepWithHealing(params) {
|
|
|
6008
6091
|
? "url"
|
|
6009
6092
|
: networkFired
|
|
6010
6093
|
? "network"
|
|
6011
|
-
:
|
|
6094
|
+
: domVerifiedForStep
|
|
6095
|
+
? "dom"
|
|
6096
|
+
: formValueVerified
|
|
6097
|
+
? "form-value"
|
|
6098
|
+
: "dom";
|
|
6012
6099
|
}
|
|
6013
6100
|
// N+16 probe: Stagehand's CDP click sometimes lands on the button without
|
|
6014
6101
|
// triggering React's SyntheticEvent layer (or jQuery delegated handlers).
|
|
@@ -6095,6 +6182,7 @@ async function executeStepWithHealing(params) {
|
|
|
6095
6182
|
const retryUrlChanged = retryPost.url !== pre.url;
|
|
6096
6183
|
const retryHtmlDelta = retryPost.bodyHtmlLength - pre.bodyHtmlLength;
|
|
6097
6184
|
const retryTextChanged = retryPost.visibleTextSignature !== pre.visibleTextSignature;
|
|
6185
|
+
const retryFormValueChanged = retryPost.formValueSignature !== pre.formValueSignature;
|
|
6098
6186
|
// RC2: an advance/`kind=click` "Next" that only grew the DOM
|
|
6099
6187
|
// (validation errors rendered) with NO network/URL change is a
|
|
6100
6188
|
// validation-blocked no-op, not a real transition — but the
|
|
@@ -6150,6 +6238,7 @@ async function executeStepWithHealing(params) {
|
|
|
6150
6238
|
retryUrlChanged ||
|
|
6151
6239
|
retryHtmlDelta !== 0 ||
|
|
6152
6240
|
retryTextChanged ||
|
|
6241
|
+
retryFormValueChanged ||
|
|
6153
6242
|
checkboxStateVerified);
|
|
6154
6243
|
// Apply the same submit-endpoint gate the primary verifier uses.
|
|
6155
6244
|
// Without this, the n+16 fallback would still ride past a
|
|
@@ -6223,7 +6312,7 @@ async function executeStepWithHealing(params) {
|
|
|
6223
6312
|
failureReasons.push(`n+16 fallback: submit-judge-rejected: ${judgeVerdict.reason}`);
|
|
6224
6313
|
}
|
|
6225
6314
|
}
|
|
6226
|
-
logger.info(`n+16 probe: step=${stepIndex + 1}/${totalSteps?.() ?? "?"} attempt=${attempt} el.click() fallback fired=${fired === true} kind=${probeResult.kind ?? "none"} checkboxStateVerified=${checkboxStateVerified} ancestorStillInvalid=${ancestorStillInvalid}; network=${retryNetworkFired} url=${retryUrlChanged} htmlDelta=${retryHtmlDelta} textChanged=${retryTextChanged} verified=${retryVerified}`);
|
|
6315
|
+
logger.info(`n+16 probe: step=${stepIndex + 1}/${totalSteps?.() ?? "?"} attempt=${attempt} el.click() fallback fired=${fired === true} kind=${probeResult.kind ?? "none"} checkboxStateVerified=${checkboxStateVerified} ancestorStillInvalid=${ancestorStillInvalid}; network=${retryNetworkFired} url=${retryUrlChanged} htmlDelta=${retryHtmlDelta} textChanged=${retryTextChanged} formValueChanged=${retryFormValueChanged} verified=${retryVerified}`);
|
|
6227
6316
|
if (retryVerified) {
|
|
6228
6317
|
if (record.verifiedBy === null) {
|
|
6229
6318
|
record.verifiedBy = retryUrlChanged ? "url" : retryNetworkFired ? "network" : "dom";
|
|
@@ -6488,7 +6577,7 @@ async function waitForSpaReady(page, logger, opts = {}) {
|
|
|
6488
6577
|
* than let the flow report phantom success.
|
|
6489
6578
|
*/
|
|
6490
6579
|
async function runHealingFlow(deps) {
|
|
6491
|
-
const { stagehand, page, steps, logger, anthropic, uploadFixture, maxFlowMs } = deps;
|
|
6580
|
+
const { stagehand, page, steps, logger, anthropic, rephraseModel, uploadFixture, maxFlowMs } = deps;
|
|
6492
6581
|
const counter = { n: 0 };
|
|
6493
6582
|
const signalCounter = { n: 0 };
|
|
6494
6583
|
const recentCaptures = [];
|
|
@@ -6540,6 +6629,7 @@ async function runHealingFlow(deps) {
|
|
|
6540
6629
|
recentCaptures,
|
|
6541
6630
|
recentCaptureMeta,
|
|
6542
6631
|
anthropic,
|
|
6632
|
+
rephraseModel,
|
|
6543
6633
|
logger,
|
|
6544
6634
|
uploadFixture,
|
|
6545
6635
|
frameTarget,
|