@enricai/barnacle 1.12.80 → 1.12.82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/dist/lib/llm/judges/verify-step-advanced.d.ts +1 -1
  2. package/dist/lib/llm/judges/verify-step-advanced.d.ts.map +1 -1
  3. package/dist/lib/llm/judges/verify-step-advanced.js +2 -1
  4. package/dist/lib/llm/judges/verify-step-advanced.js.map +1 -1
  5. package/dist/plugins/config-plugin.d.ts +1 -0
  6. package/dist/plugins/config-plugin.d.ts.map +1 -1
  7. package/dist/plugins/config-plugin.js +7 -1
  8. package/dist/plugins/config-plugin.js.map +1 -1
  9. package/dist/plugins/loader.d.ts +2 -2
  10. package/dist/plugins/loader.d.ts.map +1 -1
  11. package/dist/plugins/loader.js +6 -3
  12. package/dist/plugins/loader.js.map +1 -1
  13. package/dist/scraper/click-settle.d.ts +41 -1
  14. package/dist/scraper/click-settle.d.ts.map +1 -1
  15. package/dist/scraper/click-settle.js +84 -15
  16. package/dist/scraper/click-settle.js.map +1 -1
  17. package/dist/scraper/errors.d.ts +23 -2
  18. package/dist/scraper/errors.d.ts.map +1 -1
  19. package/dist/scraper/errors.js +32 -4
  20. package/dist/scraper/errors.js.map +1 -1
  21. package/dist/scraper/flow-runner.d.ts +68 -33
  22. package/dist/scraper/flow-runner.d.ts.map +1 -1
  23. package/dist/scraper/flow-runner.js +610 -198
  24. package/dist/scraper/flow-runner.js.map +1 -1
  25. package/dist/scraper/http-client.d.ts +7 -2
  26. package/dist/scraper/http-client.d.ts.map +1 -1
  27. package/dist/scraper/http-client.js +132 -19
  28. package/dist/scraper/http-client.js.map +1 -1
  29. package/dist/scraper/metrics.d.ts +2 -2
  30. package/dist/scraper/metrics.js +2 -2
  31. package/dist/scraper/phantom-click.d.ts +6 -0
  32. package/dist/scraper/phantom-click.d.ts.map +1 -1
  33. package/dist/scraper/phantom-click.js +62 -15
  34. package/dist/scraper/phantom-click.js.map +1 -1
  35. package/dist/scraper/stagehand-guard.d.ts +12 -2
  36. package/dist/scraper/stagehand-guard.d.ts.map +1 -1
  37. package/dist/scraper/stagehand-guard.js +51 -33
  38. package/dist/scraper/stagehand-guard.js.map +1 -1
  39. package/dist/scraper/trusted-click.d.ts +12 -1
  40. package/dist/scraper/trusted-click.d.ts.map +1 -1
  41. package/dist/scraper/trusted-click.js +8 -4
  42. package/dist/scraper/trusted-click.js.map +1 -1
  43. package/dist/scraper/upload-affordance-expr.js +1 -1
  44. package/dist/scraper/upload-affordance-expr.js.map +1 -1
  45. package/dist/scripts/recon-generate.d.ts +5 -7
  46. package/dist/scripts/recon-generate.d.ts.map +1 -1
  47. package/dist/scripts/recon-generate.js +123 -52
  48. package/dist/scripts/recon-generate.js.map +1 -1
  49. package/dist/scripts/recon-shared.d.ts +3 -2
  50. package/dist/scripts/recon-shared.d.ts.map +1 -1
  51. package/dist/scripts/recon-shared.js +6 -11
  52. package/dist/scripts/recon-shared.js.map +1 -1
  53. package/dist/site-plugin.d.ts +7 -4
  54. package/dist/site-plugin.d.ts.map +1 -1
  55. package/package.json +1 -1
@@ -82,7 +82,6 @@ exports.getScanPrimaryCandidateGroupsCallCountForTest = getScanPrimaryCandidateG
82
82
  exports.resetScanPrimaryCandidateGroupsCallCountForTest = resetScanPrimaryCandidateGroupsCallCountForTest;
83
83
  exports.detectDrillDownFoldPlan = detectDrillDownFoldPlan;
84
84
  exports.parseFoldReturnSpec = parseFoldReturnSpec;
85
- exports.parseBrowserFlowOnlyOptIn = parseBrowserFlowOnlyOptIn;
86
85
  exports.parseFallbackGateSpec = parseFallbackGateSpec;
87
86
  exports.readComparableBodyLeaf = readComparableBodyLeaf;
88
87
  exports.collectRequestValuesIncludingHeaders = collectRequestValuesIncludingHeaders;
@@ -96,6 +95,7 @@ exports.emitContractTs = emitContractTs;
96
95
  exports.unreferencedRequiredUrlFields = unreferencedRequiredUrlFields;
97
96
  exports.assertRequiredUrlFieldsReferenced = assertRequiredUrlFieldsReferenced;
98
97
  exports.unreferencedDeclaredPayloadFields = unreferencedDeclaredPayloadFields;
98
+ exports.unreadPayloadFieldChecklistLines = unreadPayloadFieldChecklistLines;
99
99
  exports.buildStepInstructionExpr = buildStepInstructionExpr;
100
100
  exports.emitConfigManifest = emitConfigManifest;
101
101
  exports.emitBrowserFlowTs = emitBrowserFlowTs;
@@ -481,7 +481,7 @@ function walkNavigateToFacets(flowSteps, env) {
481
481
  origin: "step",
482
482
  literals: collectStepFacetLiterals(step.step, env),
483
483
  kind: deriveFacetKind(step.payloadField),
484
- kindHints: [],
484
+ kindHints: collectStepFacetKindHints(step.step, env),
485
485
  });
486
486
  }
487
487
  continue;
@@ -523,6 +523,45 @@ function collectStepFacetLiterals(instruction, env) {
523
523
  const primary = extractStepPersonaValue(instruction, env);
524
524
  return uniqueLiterals(primary, findInstructionSpans(instruction).map((span) => span.text));
525
525
  }
526
+ const STEP_KIND_STOP_WORDS = new Set([
527
+ "facet",
528
+ "filter",
529
+ "field",
530
+ "value",
531
+ "option",
532
+ "selection",
533
+ "the",
534
+ "a",
535
+ "an",
536
+ "dropdown",
537
+ "drop-down",
538
+ "selector",
539
+ "menu",
540
+ "list",
541
+ "widget",
542
+ "question",
543
+ ]);
544
+ /**
545
+ * Element kinds a click step's own prose names: the noun phrase trailing the
546
+ * value span ("select the 'Lisbon' city option" names `city`), so a facet whose
547
+ * field name differs from the page's vocabulary can still own the element of
548
+ * that kind. Reads only the generic instruction grammar, no site vocabulary.
549
+ */
550
+ function collectStepFacetKindHints(instruction, env) {
551
+ const value = extractStepPersonaValue(instruction, env);
552
+ const span = findInstructionSpans(instruction).find((candidate) => candidate.text === value);
553
+ if (span === undefined)
554
+ return [];
555
+ const phrase = instruction
556
+ .slice(span.end)
557
+ .split(/[.,;:]|\b(?:from|in|on|for|with|and|then|to)\b/i)[0];
558
+ const words = phrase
559
+ .split(/\s+/)
560
+ .map((word) => word.replace(/^['"]+|['"]+$/g, ""))
561
+ .filter((word) => /^[A-Za-z][A-Za-z0-9_-]*$/.test(word));
562
+ const kindWords = words.filter((word) => !STEP_KIND_STOP_WORDS.has(word.toLowerCase()));
563
+ return [...new Set([...(kindWords.length > 1 ? [kindWords.join(" ")] : []), ...kindWords])];
564
+ }
526
565
  function collectHashFacetLiterals(url, previousHash) {
527
566
  const primary = extractNavigateToHashFragmentValue(url, previousHash);
528
567
  const hashIndex = url.indexOf("#");
@@ -6800,18 +6839,34 @@ function resolveRecurringNavigateToFacets(declared, actions) {
6800
6839
  // recurs in one vocabulary (a hash slug, a quoted label) says nothing about
6801
6840
  // the same facet's element in another (`WW;filterType=ship`).
6802
6841
  const claimedLiterals = new Set();
6803
- const aliased = declared.flatMap((facet) => {
6804
- const literal = findKindKeyedLiteral(facet, actions);
6805
- if (literal === undefined || declared.some(({ value }) => value === literal))
6842
+ const pairsByField = new Map();
6843
+ const aliased = declared
6844
+ .flatMap((facet) => findKindKeyedLiterals(facet, actions).flatMap((literal) => {
6845
+ if (declared.some(({ value }) => value === literal))
6806
6846
  return [];
6807
6847
  if (claimedLiterals.has(literal))
6808
6848
  return [];
6809
6849
  claimedLiterals.add(literal);
6810
6850
  const known = [facet.value, ...(facet.literals ?? [])];
6851
+ // A case-only variant is already owned by the declared facet, whose
6852
+ // renderer re-cases any caller value to the captured element's casing.
6811
6853
  if (known.some((candidate) => candidate.toLowerCase() === literal.toLowerCase()))
6812
6854
  return [];
6813
- const valueMap = new Map(facet.value.length > 0 ? [[facet.value, literal]] : []);
6814
- return [{ ...facet, value: literal, literals: [literal], keyedByKind: true, valueMap }];
6855
+ // The traffic carries codes, not which declared value each belongs to;
6856
+ // pair only where the code abbreviates the value, never by position.
6857
+ const sameField = declared.filter(({ field, value }) => field === facet.field && value.length > 0);
6858
+ const abbreviated = sameField.filter(({ value }) => abbreviates(literal, value));
6859
+ const owners = abbreviated.length > 0 || sameField.length !== 1 ? abbreviated : sameField;
6860
+ if (owners.length > 0)
6861
+ pairsByField.set(facet.field, [
6862
+ ...(pairsByField.get(facet.field) ?? []),
6863
+ ...owners.map((owner) => [owner.value, literal]),
6864
+ ]);
6865
+ return [{ ...facet, value: literal, literals: [literal], keyedByKind: true }];
6866
+ }))
6867
+ .map((alias) => {
6868
+ const pairs = pairsByField.get(alias.field) ?? [];
6869
+ return pairs.length === 0 ? alias : { ...alias, valueMap: new Map(pairs) };
6815
6870
  });
6816
6871
  if (unreachable.length === 0 && aliased.length === 0)
6817
6872
  return resolvedDeclared;
@@ -6837,13 +6892,15 @@ function resolveRecurringNavigateToFacets(declared, actions) {
6837
6892
  return [...withAliases, ...extras];
6838
6893
  }
6839
6894
  /**
6840
- * The literal a facet takes in the code vocabulary: the captured request
6895
+ * The literals a facet takes in the code vocabulary: the captured request
6841
6896
  * element (query value or body string leaf) whose `<literal><delim>key=<kind>`
6842
6897
  * suffix names the facet's own kind. Derived from the traffic itself, so it
6843
6898
  * never depends on how many other facets or transitions the flow declared.
6844
- * Returns `undefined` unless exactly one distinct literal carries the kind.
6899
+ * A kind carrying several distinct literals across captures yields each of
6900
+ * them, so every element of the kind is owned instead of none; the month
6901
+ * evidence narrows them only when it singles out some of them.
6845
6902
  */
6846
- function findKindKeyedLiteral(facet, actions) {
6903
+ function findKindKeyedLiterals(facet, actions) {
6847
6904
  const byTier = new Map();
6848
6905
  const sameMonth = new Set();
6849
6906
  const facetMonths = new Set([facet.value, ...(facet.literals ?? [])]
@@ -6874,10 +6931,10 @@ function findKindKeyedLiteral(facet, actions) {
6874
6931
  }
6875
6932
  }
6876
6933
  const closest = [...byTier.entries()].sort(([a], [b]) => a - b)[0]?.[1];
6877
- if (closest !== undefined && closest.size === 1)
6878
- return [...closest][0];
6879
- const sameMonthInKind = [...(closest ?? sameMonth)].filter((literal) => sameMonth.has(literal));
6880
- return sameMonthInKind.length === 1 ? sameMonthInKind[0] : undefined;
6934
+ if (closest === undefined)
6935
+ return sameMonth.size === 1 ? [...sameMonth] : [];
6936
+ const sameMonthInKind = [...closest].filter((literal) => sameMonth.has(literal));
6937
+ return sameMonthInKind.length > 0 && closest.size > 1 ? sameMonthInKind : [...closest];
6881
6938
  }
6882
6939
  /**
6883
6940
  * Resolves each zero-recurrence navigateTo facet (in flow order) to the real
@@ -8460,6 +8517,20 @@ function facetCaseMethod(captured, facet) {
8460
8517
  return "toLowerCase";
8461
8518
  return undefined;
8462
8519
  }
8520
+ /** True when `code`'s letters occur in order within `value`, starting at its first letter (`LIS` in `Lisbon`). */
8521
+ function abbreviates(code, value) {
8522
+ const letters = code.toLowerCase().replace(/[^a-z]/g, "");
8523
+ const target = value.toLowerCase();
8524
+ if (letters.length === 0 || letters[0] !== target[0])
8525
+ return false;
8526
+ let from = 0;
8527
+ for (const letter of letters) {
8528
+ from = target.indexOf(letter, from) + 1;
8529
+ if (from === 0)
8530
+ return false;
8531
+ }
8532
+ return true;
8533
+ }
8463
8534
  /**
8464
8535
  * JS expression for the value a facet's payload field carries in the code
8465
8536
  * vocabulary: a kind-keyed alias translates the caller's declared literal
@@ -10013,23 +10084,6 @@ function parseFoldReturnSpec(flowFileContents) {
10013
10084
  return null;
10014
10085
  }
10015
10086
  }
10016
- /**
10017
- * Reads the flow file's `browserFlowOnly: true` opt-in, which forces the
10018
- * browser-flow-only emit for a submission flow whose HTTP analysis would
10019
- * otherwise produce an executeHttp path. Anything but a literal `true` (or an
10020
- * unreadable/legacy bare-array file) is "not opted in".
10021
- */
10022
- function parseBrowserFlowOnlyOptIn(flowFileContents) {
10023
- try {
10024
- const raw = JSON.parse(flowFileContents);
10025
- if (raw === null || typeof raw !== "object" || Array.isArray(raw))
10026
- return false;
10027
- return raw.browserFlowOnly === true;
10028
- }
10029
- catch {
10030
- return false;
10031
- }
10032
- }
10033
10087
  /**
10034
10088
  * Parses `browserFallbackGate`/`httpTimeoutMs` out of an object-form
10035
10089
  * recon-flow.json, mirroring {@link parseFoldReturnSpec}'s null-safe pattern:
@@ -11917,7 +11971,7 @@ function emitContractTs(opts) {
11917
11971
  browserFlowOnly: omitExecuteHttp,
11918
11972
  payloadFieldNames,
11919
11973
  });
11920
- const isReservedByApplicantContactSchema = (name) => usesApplicantContactSchema && applicantContactFieldNames.has(name);
11974
+ const isReservedByApplicantContactSchema = (name) => usesApplicantContactSchema && applicantContactFieldNames.has(canonicalPayloadField(name));
11921
11975
  // A missing inputBody, or one with no ApplicantContactSchema evidence,
11922
11976
  // means this is not a job-application submission flow. The former keeps
11923
11977
  // its own read-flow `{ query }` contract untouched; the latter (a body WAS
@@ -12045,7 +12099,7 @@ function emitContractTs(opts) {
12045
12099
  // The browser-flow-only emit canonicalises its own splices already; this
12046
12100
  // covers a caller handing over vocabulary-row names directly, so one never
12047
12101
  // becomes a required key beside the base schema's own equivalent.
12048
- const canonicaliseNames = usesApplicantContactSchema && omitExecuteHttp && isSubmissionFlow;
12102
+ const canonicaliseNames = usesApplicantContactSchema;
12049
12103
  for (const rawName of [...payloadFieldNames].sort()) {
12050
12104
  const name = canonicaliseNames ? canonicalPayloadField(rawName) : rawName;
12051
12105
  if (isReservedByApplicantContactSchema(name))
@@ -12967,6 +13021,13 @@ function unreferencedDeclaredPayloadFields(contractCode, browserFlowCode, flowSt
12967
13021
  function unreadDeclaredPayloadFields(contractCode, browserFlowCode, flowSteps) {
12968
13022
  return new Set(unreferencedDeclaredPayloadFields(contractCode, contractCode.includes("httpClient(") ? "" : browserFlowCode, flowSteps));
12969
13023
  }
13024
+ /**
13025
+ * Review-checklist lines for declared payload fields no emitted request reads,
13026
+ * so a reviewer sees the silently-optionalized field outside the log stream.
13027
+ */
13028
+ function unreadPayloadFieldChecklistLines(fields) {
13029
+ return [...fields].map((field) => `Declared payloadField ${field} is read by no emitted request and was emitted optional — confirm it should affect the request, or drop it`);
13030
+ }
12970
13031
  /**
12971
13032
  * Escape a literal string segment so it is safe INSIDE a JS backtick template
12972
13033
  * literal — backslashes, backticks, and `${` interpolation starts must all be
@@ -13099,7 +13160,15 @@ function lastFlowStepDeclaresNonSubmit(flowSteps) {
13099
13160
  */
13100
13161
  function emitConfigManifest(opts) {
13101
13162
  const { siteId, displayName, baseUrl, flowSteps, vocabulary, inputBody, recoveredFields, httpModulePath, isSubmissionFlow = false, hasMultipartStep = false, browserFlowOnly = false, env = process.env, } = opts;
13102
- const canonicalise = isSubmissionFlow && browserFlowOnly;
13163
+ // Raw (uncanonicalised) splice names are the evidence for a body-less flow;
13164
+ // the decision must be made on them before canonicalising, or it is circular.
13165
+ const canonicalise = contractUsesApplicantContactSchema({
13166
+ inputBody,
13167
+ isSubmissionFlow,
13168
+ browserFlowOnly,
13169
+ payloadFieldNames: computeFlowPayloadFieldNames(flowSteps, vocabulary, env, false)
13170
+ .payloadFieldNames,
13171
+ });
13103
13172
  const payloadFieldNames = new Set();
13104
13173
  const knownFieldValues = buildKnownFieldValues(flowSteps, vocabulary ?? vocabulary_1.EMPTY_VOCABULARY, env, canonicalise);
13105
13174
  const steps = flowSteps.map((step) => {
@@ -13303,20 +13372,20 @@ function computeFlowPayloadFieldNames(flowSteps, vocabulary, env, canonicalise =
13303
13372
  }
13304
13373
  function emitBrowserFlowTs(opts) {
13305
13374
  const { siteId, pascal, flowSteps, isSubmissionFlow, hasMultipartStep = false, browserFlowOnly = false, inputBody, vocabulary, frameSelector, capturedFallbackFields = new Set(), env = process.env, } = opts;
13306
- const canonicalise = isSubmissionFlow && browserFlowOnly;
13375
+ const canonicalise = contractUsesApplicantContactSchema({
13376
+ inputBody,
13377
+ isSubmissionFlow,
13378
+ browserFlowOnly,
13379
+ payloadFieldNames: computeFlowPayloadFieldNames(flowSteps, vocabulary, env, false)
13380
+ .payloadFieldNames,
13381
+ });
13307
13382
  const { payloadFieldNames, optionalPayloadFieldNames } = computeFlowPayloadFieldNames(flowSteps, vocabulary, env, canonicalise);
13308
13383
  for (const field of capturedFallbackFields) {
13309
13384
  if (payloadFieldNames.has(field))
13310
13385
  optionalPayloadFieldNames.add(field);
13311
13386
  }
13312
13387
  const hasUploadStep = flowSteps.some((s) => typeof s !== "string" && s.upload === true);
13313
- const contractCarriesResume = hasMultipartStep ||
13314
- contractUsesApplicantContactSchema({
13315
- inputBody,
13316
- isSubmissionFlow,
13317
- browserFlowOnly,
13318
- payloadFieldNames,
13319
- });
13388
+ const contractCarriesResume = hasMultipartStep || canonicalise;
13320
13389
  const knownFieldValues = buildKnownFieldValues(flowSteps, vocabulary ?? vocabulary_1.EMPTY_VOCABULARY, env, canonicalise);
13321
13390
  let usesThrowawayPassword = false;
13322
13391
  let usesEmailStep = false;
@@ -13886,14 +13955,7 @@ async function main() {
13886
13955
  };
13887
13956
  }
13888
13957
  })();
13889
- const browserFlowOnlyOptIn = (() => {
13890
- try {
13891
- return parseBrowserFlowOnlyOptIn((0, node_fs_1.readFileSync)(flowFile, "utf8"));
13892
- }
13893
- catch {
13894
- return false;
13895
- }
13896
- })();
13958
+ const browserFlowOnlyOptIn = (0, node_fs_1.existsSync)(flowFile) ? (0, recon_shared_1.readBrowserFlowOnly)(flowFile) : false;
13897
13959
  // Flow-declared signals that isolate the submission POSTs from same-origin
13898
13960
  // page chrome. Threaded into action-sequence extraction and header derivation
13899
13961
  // so both draw from the real submission, not incidental widget/chatbot POSTs.
@@ -13930,8 +13992,14 @@ async function main() {
13930
13992
  * already written its output and there is nothing left for main() to do.
13931
13993
  */
13932
13994
  function generateFromCaptures(activeCaptures) {
13995
+ if (activeCaptures.length === 0 && !allowEmptyCapture) {
13996
+ throw new Error("empty capture pool: no captures remain to derive a base URL from (a placeholder origin would leak into the contract)");
13997
+ }
13998
+ // An opted-in empty capture has no origin to derive; "" leaves the
13999
+ // operator's siteBaseUrls config as the only source instead of a
14000
+ // placeholder origin leaking into the contract.
13933
14001
  const baseUrl = activeCaptures.length === 0
13934
- ? "https://example.com"
14002
+ ? ""
13935
14003
  : deriveBaseUrl(activeCaptures, ownBackendHostnames, submitPatterns);
13936
14004
  // Gate on the flow's declared own-backend hosts (or the registrable-domain
13937
14005
  // fallback of baseUrl) via the same predicate recon-http.ts applies at
@@ -14497,7 +14565,10 @@ async function main() {
14497
14565
  logger.info(`wrote ${outDir}/contract.ts`);
14498
14566
  // Same opts fed to emitContractTs above, so this can never drift from what
14499
14567
  // the header used to embed.
14500
- const checklist = buildContractChecklist(final.contractOpts);
14568
+ const checklist = [
14569
+ ...buildContractChecklist(final.contractOpts),
14570
+ ...unreadPayloadFieldChecklistLines(unreadDeclaredPayloadFields(final.contractCode, final.browserFlow.code, flowSteps)),
14571
+ ];
14501
14572
  logger.info(`review checklist for ${outDir}/contract.ts:\n${checklist.map((item) => ` [ ] ${item}`).join("\n")}`);
14502
14573
  (0, node_fs_1.writeFileSync)(`${outDir}/flows/browser-flow.ts`, final.browserFlow.code);
14503
14574
  logger.info(`wrote ${outDir}/flows/browser-flow.ts`);