@enricai/barnacle 1.4.2 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +29 -0
  2. package/dist/api/errors.d.ts +2 -1
  3. package/dist/api/errors.d.ts.map +1 -1
  4. package/dist/api/errors.js +3 -2
  5. package/dist/api/errors.js.map +1 -1
  6. package/dist/lib/llm/judge.d.ts +1 -1
  7. package/dist/lib/llm/judge.js +1 -1
  8. package/dist/recon/capture-filters.d.ts +40 -0
  9. package/dist/recon/capture-filters.d.ts.map +1 -0
  10. package/dist/recon/capture-filters.js +106 -0
  11. package/dist/recon/capture-filters.js.map +1 -0
  12. package/dist/recon/form-schema.d.ts +51 -0
  13. package/dist/recon/form-schema.d.ts.map +1 -0
  14. package/dist/recon/form-schema.js +20 -0
  15. package/dist/recon/form-schema.js.map +1 -0
  16. package/dist/recon/load-form-schema.d.ts +24 -0
  17. package/dist/recon/load-form-schema.d.ts.map +1 -0
  18. package/dist/recon/load-form-schema.js +66 -0
  19. package/dist/recon/load-form-schema.js.map +1 -0
  20. package/dist/scraper/errors.d.ts +14 -14
  21. package/dist/scraper/errors.d.ts.map +1 -1
  22. package/dist/scraper/errors.js +15 -15
  23. package/dist/scraper/errors.js.map +1 -1
  24. package/dist/scraper/flow-runner.d.ts +12 -0
  25. package/dist/scraper/flow-runner.d.ts.map +1 -1
  26. package/dist/scraper/flow-runner.js +46 -8
  27. package/dist/scraper/flow-runner.js.map +1 -1
  28. package/dist/scraper/http-client.d.ts +12 -0
  29. package/dist/scraper/http-client.d.ts.map +1 -1
  30. package/dist/scraper/http-client.js +11 -15
  31. package/dist/scraper/http-client.js.map +1 -1
  32. package/dist/scraper/parse-json-response.d.ts +7 -5
  33. package/dist/scraper/parse-json-response.d.ts.map +1 -1
  34. package/dist/scraper/parse-json-response.js +9 -8
  35. package/dist/scraper/parse-json-response.js.map +1 -1
  36. package/dist/scraper/rate-limited-json-client.d.ts +8 -1
  37. package/dist/scraper/rate-limited-json-client.d.ts.map +1 -1
  38. package/dist/scraper/rate-limited-json-client.js +6 -1
  39. package/dist/scraper/rate-limited-json-client.js.map +1 -1
  40. package/dist/scraper/session-browserbase.d.ts +6 -0
  41. package/dist/scraper/session-browserbase.d.ts.map +1 -1
  42. package/dist/scraper/session-browserbase.js +6 -0
  43. package/dist/scraper/session-browserbase.js.map +1 -1
  44. package/dist/scripts/recon-browser.d.ts.map +1 -1
  45. package/dist/scripts/recon-browser.js +3 -4
  46. package/dist/scripts/recon-browser.js.map +1 -1
  47. package/dist/scripts/recon-generate.d.ts +69 -20
  48. package/dist/scripts/recon-generate.d.ts.map +1 -1
  49. package/dist/scripts/recon-generate.js +277 -173
  50. package/dist/scripts/recon-generate.js.map +1 -1
  51. package/dist/scripts/recon-http.d.ts +5 -2
  52. package/dist/scripts/recon-http.d.ts.map +1 -1
  53. package/dist/scripts/recon-http.js +16 -4
  54. package/dist/scripts/recon-http.js.map +1 -1
  55. package/dist/testing/persona-fixture.d.ts +2 -2
  56. package/dist/testing/persona-fixture.d.ts.map +1 -1
  57. package/dist/testing/persona-fixture.js +2 -2
  58. package/dist/testing/persona-fixture.js.map +1 -1
  59. package/package.json +9 -1
  60. package/dist/scraper/oracle-sentinels.d.ts +0 -22
  61. package/dist/scraper/oracle-sentinels.d.ts.map +0 -1
  62. package/dist/scraper/oracle-sentinels.js +0 -40
  63. package/dist/scraper/oracle-sentinels.js.map +0 -1
@@ -20,6 +20,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
20
20
  exports.resolveStepPayloadField = resolveStepPayloadField;
21
21
  exports.inferZodSchemaFromSamples = inferZodSchemaFromSamples;
22
22
  exports.selectPayloadAction = selectPayloadAction;
23
+ exports.extractActionSequence = extractActionSequence;
24
+ exports.detectFormSchemaFieldNames = detectFormSchemaFieldNames;
23
25
  exports.indexStateValues = indexStateValues;
24
26
  exports.compileActionSteps = compileActionSteps;
25
27
  exports.loadQuestionPromptKeywords = loadQuestionPromptKeywords;
@@ -34,10 +36,25 @@ const errors_1 = require("../lib/errors");
34
36
  const logging_1 = require("../lib/logging");
35
37
  const plugin_api_version_1 = require("../plugins/plugin-api-version");
36
38
  const plugin_manifest_envelope_1 = require("../plugins/plugin-manifest-envelope");
39
+ const capture_filters_1 = require("../recon/capture-filters");
40
+ const load_form_schema_1 = require("../recon/load-form-schema");
37
41
  const load_vocabulary_1 = require("../recon/load-vocabulary");
38
42
  const vocabulary_1 = require("../recon/vocabulary");
39
43
  const recon_shared_1 = require("../scripts/recon-shared");
40
44
  const logger = (0, logging_1.getScriptLogger)("recon-generate");
45
+ /**
46
+ * Engine imports in GENERATED code must be package subpaths, never the `@/`
47
+ * alias, and the reason is not visible from the source: `tsc-alias` rewrites by
48
+ * text, so it cannot tell an import this module *uses* from one it *emits as a
49
+ * string*. Written as `@/scraper/session`, the build silently rewrote the
50
+ * template literal itself — shipping `dist/` emitters that generated
51
+ * `../scraper/session` and left every out-of-tree consumer with TS2307.
52
+ * (`@/sites/...` survived only because `src/sites/` is empty, so it resolved to
53
+ * no file.) A bare specifier has nothing to resolve against, so the build leaves
54
+ * it alone. `out-of-tree-e2e.test.ts` asserts this against the BUILT dist —
55
+ * asserting it against the source would pass while the shipped artifact is broken.
56
+ */
57
+ const ENGINE_PKG = "@enricai/barnacle";
41
58
  // ── helpers ──────────────────────────────────────────────────────────────────
42
59
  function toPascalCase(siteId) {
43
60
  return siteId
@@ -320,9 +337,8 @@ const IGNORE_REQUEST_HEADERS = new Set([
320
337
  * stateless replay phase can't thread), derive headers from the meaningful
321
338
  * action POSTs instead — same `extractActionSequence` definition used by
322
339
  * the submission-flow detector. This catches load-bearing site-specific
323
- * headers (Workday's `X-CSRF-Token`, Greenhouse's `Job-Boards-API-Token`,
324
- * ClearCompany's `API-ShortName`, etc.) without the generator needing to
325
- * know about any particular site.
340
+ * headers (a `X-CSRF-Token`, a `Job-Boards-API-Token`, an `API-ShortName`,
341
+ * etc.) without the generator needing to know about any particular site.
326
342
  */
327
343
  function deriveRequestHeaders(captures, replays, baseUrl) {
328
344
  const successfulUrls = new Set(replays.filter((r) => r.success).map((r) => endpointKey(r.url)));
@@ -389,30 +405,14 @@ function firstEndpointPath(captures) {
389
405
  }
390
406
  return "/api/search";
391
407
  }
392
- // ── multi-step "submission flow" detection ────────────────────────────────────
393
- //
394
- // For transactional sites (apply forms, multi-step checkout, etc.) the captures
395
- // form an ordered sequence of POSTs that thread state values through subsequent
396
- // requests (auth tokens, candidate IDs, application IDs). Single-endpoint
397
- // sites (job search, pricing APIs) have one action capture and skip this path.
398
- /**
399
- * Path elements we always treat as noise (analytics, logging). Site-specific
400
- * trackers belong in RECON_TELEMETRY_URL_PATTERNS (comma-separated), not here —
401
- * the engine must not carry any one site's ad-tech domains.
402
- */
403
- const TELEMETRY_URL_PATTERNS = [
404
- "/util/logging/vweb/message",
405
- "/blank/page",
406
- "stats.g.doubleclick.net",
407
- "google-analytics.com",
408
- ...(process.env.RECON_TELEMETRY_URL_PATTERNS ?? "")
409
- .split(",")
410
- .map((p) => p.trim())
411
- .filter(Boolean),
412
- ];
413
408
  /**
414
409
  * Extracts the ordered sequence of meaningful POSTs that represent the
415
- * transactional flow. Filters out GETs, telemetry, asset hits, and non-2xx.
410
+ * transactional flow: same-host 2xx POSTs, minus telemetry and error-reporting
411
+ * sinks. Assets need no filter of their own — they arrive as GETs.
412
+ *
413
+ * Exported for tests: this predicate decides what a generated plugin will POST
414
+ * at a live site, and it is the only gate between a browser's incidental
415
+ * chatter and the emitted hot path.
416
416
  */
417
417
  function extractActionSequence(captures, baseUrl) {
418
418
  let host;
@@ -438,7 +438,7 @@ function extractActionSequence(captures, baseUrl) {
438
438
  }
439
439
  if (captureHost !== host)
440
440
  return false;
441
- if (TELEMETRY_URL_PATTERNS.some((p) => capture.url.includes(p)))
441
+ if ((0, capture_filters_1.isNoiseUrl)(capture.url))
442
442
  return false;
443
443
  return true;
444
444
  });
@@ -530,7 +530,7 @@ const KNOWN_TOP_LEVEL_ERROR_KEYS = new Set(["message", "error", "errormessage"])
530
530
  /**
531
531
  * Suffixes that mark a JSON key as carrying validation/data errors when its
532
532
  * value is non-null. Case-sensitive because real APIs use mixed-case in the
533
- * exact form they ship (e.g. ClearCompany's `ResponseValidationErrors`).
533
+ * exact form they ship (e.g. a `ResponseValidationErrors` key).
534
534
  */
535
535
  const NESTED_ERROR_KEY_SUFFIXES = ["ValidationErrors", "DataErrors", "ValidationError"];
536
536
  /**
@@ -540,7 +540,7 @@ const NESTED_ERROR_KEY_SUFFIXES = ["ValidationErrors", "DataErrors", "Validation
540
540
  * responses (so legitimate success-only fields like `Name` aren't false-
541
541
  * flagged as errors).
542
542
  *
543
- * Site-agnostic: ClearCompany uses `Message`/`Sections.ResponseValidationErrors`
543
+ * Site-agnostic: one ATS may use `Message`/`Sections.ResponseValidationErrors`
544
544
  * /`Sections.DataValidationErrors`; a different ATS using `error`/`errors[]`
545
545
  * would emit guards for those instead.
546
546
  */
@@ -601,7 +601,7 @@ function walkForNestedErrorKeys(value, path, candidates) {
601
601
  */
602
602
  const UUID_REGEX = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i;
603
603
  /**
604
- * Converts a FieldSourceCode like "contact.first.name" or "address.country.subdivision"
604
+ * Converts a machine code like "contact.first.name" or "address.country.subdivision"
605
605
  * to PascalCase: "ContactFirstName", "AddressCountrySubdivision". Site-agnostic:
606
606
  * operates only on the input string. Returns null for inputs that don't
607
607
  * produce a valid JS identifier.
@@ -614,7 +614,7 @@ function sourceCodeToPascalCase(sourceCode) {
614
614
  return isValidJsIdentifier(pascal) ? pascal : null;
615
615
  }
616
616
  /**
617
- * Converts a free-form FieldName like "Reference #1 First Name" or "Email" to
617
+ * Converts a free-form field label like "Reference #1 First Name" or "Email" to
618
618
  * PascalCase, stripping punctuation in a way that preserves position (so
619
619
  * "Reference #1 First Name" → "Reference1FirstName" via a section-heading
620
620
  * prefix). Site-agnostic: operates only on input strings.
@@ -650,57 +650,70 @@ function fieldNameToPascalCase(fieldName, prefix) {
650
650
  }
651
651
  /**
652
652
  * Recon-driven detection of form-schema captures. Scans all response bodies
653
- * for arrays whose objects look like SectionField (UUID-shaped FieldId plus
654
- * FieldName or FieldSourceCode). Builds FieldId → PascalCase name map.
653
+ * for arrays whose objects look like form fields (a UUID-shaped field-id key
654
+ * plus at least one of the schema's name keys). Builds field-id → PascalCase
655
+ * name map.
656
+ *
657
+ * Site-agnostic: identifies form-schema captures by structural fingerprint
658
+ * against the consumer-supplied keys, not by URL or site name. Any ATS exposing
659
+ * a matching schema would match.
655
660
  *
656
- * Site-agnostic: identifies form-schema captures by structural fingerprint,
657
- * not by URL or site name. Any ATS exposing a similar schema would match.
661
+ * Exported for unit testing lets tests prove the rewiring: the same field is
662
+ * recovered whether the supplied keys are one vendor's names or a differing
663
+ * vendor's, and that a null schema recovers nothing.
658
664
  */
659
- function detectFormSchemaFieldNames(captures) {
665
+ function detectFormSchemaFieldNames(captures, formSchema) {
660
666
  const fieldNameMap = new Map();
661
667
  const fieldOptionsMap = new Map();
662
668
  const allSchemaUuids = new Set();
669
+ // No form-schema declared → recover nothing. The engine carries no vendor's
670
+ // wire keys, so there is nothing to fingerprint ATS responses against.
671
+ if (formSchema === null)
672
+ return { fieldNameMap, fieldOptionsMap, allSchemaUuids };
663
673
  for (const capture of captures) {
664
- walkForSectionFieldsArrays(capture.responseBody, fieldNameMap, fieldOptionsMap);
665
- walkForSchemaUuids(capture.responseBody, allSchemaUuids);
674
+ walkForSectionFieldsArrays(capture.responseBody, fieldNameMap, fieldOptionsMap, formSchema);
675
+ walkForSchemaUuids(capture.responseBody, allSchemaUuids, formSchema);
666
676
  }
667
677
  return { fieldNameMap, fieldOptionsMap, allSchemaUuids };
668
678
  }
669
679
  /**
670
- * Walks a response body collecting every UUID-shaped string that appears at
671
- * a `FieldId` or `OptionId` (and its lowercase variants) path leaf, OR as the
672
- * `Id` of an object that has `OptionSourceCode`/`StringKey`/`FieldOptions`
673
- * sibling i.e. an OptionId in the form schema. These UUIDs are stable
674
- * schema anchors that must be shielded from state-threading even when
675
- * detectFormSchemaFieldNames doesn't emit a payload-mappable name for the
676
- * field (e.g. when the field's FieldName is too long for our naming heuristic).
680
+ * Walks a response body collecting UUID-shaped strings under the schema's
681
+ * field-id key, or under the option-id key of an entry in a sibling
682
+ * field-options array. These are stable schema anchors that must be shielded
683
+ * from state-threading even when detectFormSchemaFieldNames emits no
684
+ * payload-mappable name for the field (e.g. when the field name is too long for
685
+ * our naming heuristic).
686
+ *
687
+ * The wire keys come from the consumer-supplied {@link ReconFormSchema}, so a
688
+ * vendor declares its own keys with `--form-schema` rather than the engine
689
+ * hardcoding any — the inversion issue #57 asked for.
677
690
  */
678
- function walkForSchemaUuids(value, out) {
691
+ function walkForSchemaUuids(value, out, formSchema) {
679
692
  if (value === null || typeof value !== "object")
680
693
  return;
681
694
  if (Array.isArray(value)) {
682
695
  for (const item of value)
683
- walkForSchemaUuids(item, out);
696
+ walkForSchemaUuids(item, out, formSchema);
684
697
  return;
685
698
  }
686
699
  const obj = value;
687
- const fieldIdRaw = obj.FieldId;
700
+ const fieldIdRaw = obj[formSchema.fieldIdKey];
688
701
  if (typeof fieldIdRaw === "string" && UUID_REGEX.test(fieldIdRaw)) {
689
702
  out.add(fieldIdRaw);
690
703
  }
691
- const optionsRaw = obj.FieldOptions;
704
+ const optionsRaw = obj[formSchema.fieldOptionsKey];
692
705
  if (Array.isArray(optionsRaw)) {
693
706
  for (const opt of optionsRaw) {
694
707
  if (opt !== null && typeof opt === "object") {
695
- const optId = opt.Id;
708
+ const optId = opt[formSchema.optionIdKey];
696
709
  if (typeof optId === "string" && UUID_REGEX.test(optId))
697
710
  out.add(optId);
698
711
  }
699
712
  }
700
713
  }
701
- // Recurse into nested objects/arrays so nested SectionFields get walked too.
714
+ // Recurse into nested objects/arrays so nested form fields get walked too.
702
715
  for (const v of Object.values(obj))
703
- walkForSchemaUuids(v, out);
716
+ walkForSchemaUuids(v, out, formSchema);
704
717
  }
705
718
  /**
706
719
  * Closed enum of well-known cache-buster query parameter names. Stripped
@@ -723,14 +736,14 @@ function stripCacheBusterParams(url) {
723
736
  }
724
737
  /**
725
738
  * Recon-driven detection of the form-schema fetch capture. Returns the first
726
- * GET capture (in recon order) whose response body contains a SectionFields-
727
- * shaped array. Sites without such a capture get `null` and Phase B/C/D
728
- * become no-ops.
739
+ * GET capture (in recon order) whose response body contains a form-fields-
740
+ * shaped array (per the supplied schema keys). Sites without such a capture get
741
+ * `null` and Phase B/C/D become no-ops.
729
742
  *
730
743
  * Site-agnostic: identifies the fetch by structural fingerprint of the
731
744
  * response body, not by URL or site name.
732
745
  */
733
- function detectFormSchemaFetchCapture(captures, baseUrl) {
746
+ function detectFormSchemaFetchCapture(captures, baseUrl, formSchema) {
734
747
  let host;
735
748
  try {
736
749
  host = new URL(baseUrl).host;
@@ -753,52 +766,52 @@ function detectFormSchemaFetchCapture(captures, baseUrl) {
753
766
  }
754
767
  if (captureHost !== host)
755
768
  continue;
756
- if (TELEMETRY_URL_PATTERNS.some((p) => capture.url.includes(p)))
769
+ if ((0, capture_filters_1.telemetryUrlPatterns)().some((p) => capture.url.includes(p)))
757
770
  continue;
758
- if (responseContainsSectionFields(capture.responseBody)) {
771
+ if (responseContainsSectionFields(capture.responseBody, formSchema)) {
759
772
  return { capture, index: i };
760
773
  }
761
774
  }
762
775
  return null;
763
776
  }
764
- function responseContainsSectionFields(value) {
777
+ function responseContainsSectionFields(value, formSchema) {
765
778
  if (value === null || typeof value !== "object")
766
779
  return false;
767
780
  if (Array.isArray(value)) {
768
- if (looksLikeSectionFieldsArray(value))
781
+ if (looksLikeSectionFieldsArray(value, formSchema))
769
782
  return true;
770
783
  for (const item of value) {
771
- if (responseContainsSectionFields(item))
784
+ if (responseContainsSectionFields(item, formSchema))
772
785
  return true;
773
786
  }
774
787
  return false;
775
788
  }
776
789
  for (const v of Object.values(value)) {
777
- if (responseContainsSectionFields(v))
790
+ if (responseContainsSectionFields(v, formSchema))
778
791
  return true;
779
792
  }
780
793
  return false;
781
794
  }
782
- function walkForSectionFieldsArrays(value, fieldNameMap, fieldOptionsMap) {
795
+ function walkForSectionFieldsArrays(value, fieldNameMap, fieldOptionsMap, formSchema) {
783
796
  if (value === null || typeof value !== "object")
784
797
  return;
785
798
  if (Array.isArray(value)) {
786
- if (looksLikeSectionFieldsArray(value)) {
787
- assignFieldNamesFromArray(value, fieldNameMap, fieldOptionsMap);
799
+ if (looksLikeSectionFieldsArray(value, formSchema)) {
800
+ assignFieldNamesFromArray(value, fieldNameMap, fieldOptionsMap, formSchema);
788
801
  }
789
802
  for (const item of value)
790
- walkForSectionFieldsArrays(item, fieldNameMap, fieldOptionsMap);
803
+ walkForSectionFieldsArrays(item, fieldNameMap, fieldOptionsMap, formSchema);
791
804
  return;
792
805
  }
793
806
  for (const v of Object.values(value)) {
794
- walkForSectionFieldsArrays(v, fieldNameMap, fieldOptionsMap);
807
+ walkForSectionFieldsArrays(v, fieldNameMap, fieldOptionsMap, formSchema);
795
808
  }
796
809
  }
797
810
  /**
798
811
  * Structural fingerprint: array of objects, at least half of which have a
799
- * UUID-shaped FieldId AND at least one of FieldName/FieldSourceCode.
812
+ * UUID-shaped field-id AND at least one of the schema's field-name keys.
800
813
  */
801
- function looksLikeSectionFieldsArray(arr) {
814
+ function looksLikeSectionFieldsArray(arr, formSchema) {
802
815
  if (arr.length === 0)
803
816
  return false;
804
817
  let matches = 0;
@@ -806,26 +819,31 @@ function looksLikeSectionFieldsArray(arr) {
806
819
  if (item === null || typeof item !== "object")
807
820
  continue;
808
821
  const obj = item;
809
- const fieldIdRaw = obj.FieldId;
822
+ const fieldIdRaw = obj[formSchema.fieldIdKey];
810
823
  if (typeof fieldIdRaw !== "string")
811
824
  continue;
812
825
  if (!UUID_REGEX.test(fieldIdRaw))
813
826
  continue;
814
- if (typeof obj.FieldName === "string" || typeof obj.FieldSourceCode === "string") {
827
+ if (formSchema.fieldNameKeys.some((key) => typeof obj[key] === "string")) {
815
828
  matches++;
816
829
  }
817
830
  }
818
831
  return matches >= Math.max(1, Math.floor(arr.length * 0.5));
819
832
  }
820
- function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
833
+ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap, formSchema) {
821
834
  let currentPrefix = null;
822
835
  const usedNames = new Set([...fieldNameMap.values()]);
836
+ // First field-name key is the machine code (preferred, PascalCased directly);
837
+ // any later key is a human label (subject to the section-heading heuristic).
838
+ // With one key both branches collapse to the label path.
839
+ const [codeKey, ...labelKeys] = formSchema.fieldNameKeys;
840
+ const labelKey = labelKeys[0];
823
841
  for (const obj of arr) {
824
- const fieldId = obj.FieldId;
842
+ const fieldId = obj[formSchema.fieldIdKey];
825
843
  if (typeof fieldId !== "string")
826
844
  continue;
827
- const sourceCode = obj.FieldSourceCode;
828
- const name = obj.FieldName;
845
+ const sourceCode = codeKey !== undefined && labelKey !== undefined ? obj[codeKey] : undefined;
846
+ const name = labelKey !== undefined ? obj[labelKey] : obj[codeKey ?? ""];
829
847
  let semantic = null;
830
848
  if (typeof sourceCode === "string" && sourceCode.trim().length > 0) {
831
849
  semantic = sourceCodeToPascalCase(sourceCode);
@@ -833,7 +851,7 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
833
851
  }
834
852
  else if (typeof name === "string" && name.trim().length > 0 && name.length < 250) {
835
853
  const hasNoSourceCode = typeof sourceCode !== "string" || sourceCode.trim().length === 0;
836
- // Section-heading heuristic: short FieldName, no SourceCode, MOSTLY
854
+ // Section-heading heuristic: short label, no machine code, MOSTLY
837
855
  // uppercase letters (>= 70% of alphabetic chars) OR contains '#'.
838
856
  // Whole-name uppercase ratio avoids false positives like "MM/DD/YYYY"
839
857
  // appearing as a format hint inside a normal field label.
@@ -859,11 +877,11 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
859
877
  }
860
878
  fieldNameMap.set(fieldId, unique);
861
879
  usedNames.add(unique);
862
- // Capture FieldOptions when present and ALL options have non-empty
863
- // semantic Values (SystemFieldOption-tagged). Custom options with empty
864
- // Value are skipped they have no semantic label, so we can't generate
865
- // a meaningful enum and leave the field's OptionId baked.
866
- const optionsRaw = obj.FieldOptions;
880
+ // Capture the field's options when present and ALL options have non-empty
881
+ // semantic labels. Options with an empty label are skipped — with no
882
+ // semantic value we can't generate a meaningful enum, so we leave the
883
+ // field's option-id baked.
884
+ const optionsRaw = obj[formSchema.fieldOptionsKey];
867
885
  if (Array.isArray(optionsRaw) && optionsRaw.length > 0) {
868
886
  const options = [];
869
887
  let allSemantic = true;
@@ -873,8 +891,8 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
873
891
  break;
874
892
  }
875
893
  const opt = optRaw;
876
- const optId = opt.Id;
877
- const optValue = opt.Value;
894
+ const optId = opt[formSchema.optionIdKey];
895
+ const optValue = opt[formSchema.optionValueKey];
878
896
  if (typeof optId !== "string" ||
879
897
  typeof optValue !== "string" ||
880
898
  optValue.trim().length === 0) {
@@ -891,20 +909,21 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
891
909
  }
892
910
  }
893
911
  /**
894
- * Substitutes Responses[].Value literals with payload accessors based on the
912
+ * Substitutes the submitted-value literals with payload accessors based on the
895
913
  * field-name map from the form schema. Operates on the body string before
896
914
  * state interpolation so already-substituted state values (e.g. ${firstName})
897
915
  * are preserved.
898
916
  *
899
- * Closed-set substring matching: both FieldId and the literal Value come from
900
- * the generator's own input (recon).
917
+ * Closed-set substring matching: both the field-id and the submitted value come
918
+ * from the generator's own input (recon).
901
919
  */
902
- function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields) {
920
+ function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields, formSchema) {
903
921
  if (fieldNameMap.size === 0)
904
922
  return rawBody;
905
923
  let result = rawBody;
924
+ const valueMarker = `"${formSchema.responseValueKey}":"`;
906
925
  for (const [fieldId, semanticName] of fieldNameMap) {
907
- const fieldIdMarker = `"FieldId":"${fieldId}"`;
926
+ const fieldIdMarker = `"${formSchema.fieldIdKey}":"${fieldId}"`;
908
927
  let cursor = 0;
909
928
  while (true) {
910
929
  const idx = result.indexOf(fieldIdMarker, cursor);
@@ -914,7 +933,6 @@ function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields
914
933
  if (objEnd === -1)
915
934
  break;
916
935
  const segment = result.slice(idx, objEnd);
917
- const valueMarker = `"Value":"`;
918
936
  const valueIdx = segment.indexOf(valueMarker);
919
937
  if (valueIdx === -1) {
920
938
  cursor = objEnd;
@@ -940,22 +958,23 @@ function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields
940
958
  return result;
941
959
  }
942
960
  /**
943
- * Substitutes Responses[].OptionId literals with payload-driven enum lookups.
944
- * Operates on the body string before state interpolation. For each FieldId
945
- * with a captured FieldOptionsMapping, find `"FieldId":"<uuid>"` and rewrite
946
- * the matching `"OptionId":"<uuid>"` to `"OptionId":"${OPT_<Name>[payload.<Name>]}"`.
961
+ * Substitutes submitted option-id literals with payload-driven enum lookups.
962
+ * Operates on the body string before state interpolation. For each field with a
963
+ * captured FieldOptionsMapping, find the schema's field-id marker and rewrite
964
+ * the matching option-id value to `${OPT_<Name>[payload.<Name>]}`.
947
965
  *
948
- * Order-insensitive: matches `"OptionId":"<uuid>"` anywhere within the same
949
- * JSON object as the FieldId (which is between this FieldId marker and the
950
- * closing `}`). Closed-set substring matching: both FieldId and OptionId
951
- * come from the generator's own input.
966
+ * Order-insensitive: matches the option-id marker anywhere within the same
967
+ * JSON object as the field-id marker (which is between it and the closing `}`).
968
+ * Closed-set substring matching: both marker values come from the generator's
969
+ * own input.
952
970
  */
953
- function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDiscoveredOptionFields) {
971
+ function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDiscoveredOptionFields, formSchema) {
954
972
  if (fieldOptionsMap.size === 0)
955
973
  return rawBody;
956
974
  let result = rawBody;
975
+ const optionIdMarker = `"${formSchema.responseOptionIdKey}":"`;
957
976
  for (const [fieldId, mapping] of fieldOptionsMap) {
958
- const fieldIdMarker = `"FieldId":"${fieldId}"`;
977
+ const fieldIdMarker = `"${formSchema.fieldIdKey}":"${fieldId}"`;
959
978
  let cursor = 0;
960
979
  while (true) {
961
980
  const idx = result.indexOf(fieldIdMarker, cursor);
@@ -965,7 +984,6 @@ function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDisco
965
984
  if (objEnd === -1)
966
985
  break;
967
986
  const segment = result.slice(idx, objEnd);
968
- const optionIdMarker = `"OptionId":"`;
969
987
  const optionIdLocal = segment.indexOf(optionIdMarker);
970
988
  if (optionIdLocal === -1) {
971
989
  cursor = objEnd;
@@ -991,25 +1009,26 @@ function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDisco
991
1009
  return result;
992
1010
  }
993
1011
  /**
994
- * For fields whose FieldOptions have NO semantic values (CustomFieldOption
995
- * options where the recon schema's `.Value` is empty), T3's OPT_* enum
996
- * mapping can't be emitted. Instead, parameterize the OptionId slot as a
997
- * caller-supplied `<FieldName>OptionId` payload field with the recon-observed
998
- * UUID documented in a TSDoc comment.
1012
+ * For fields whose options have NO semantic labels (the schema's option-label
1013
+ * value is empty), T3's OPT_* enum mapping can't be emitted. Instead,
1014
+ * parameterize the submitted option-id slot as a caller-supplied
1015
+ * `<Name>OptionId` payload field with the recon-observed UUID documented in a
1016
+ * TSDoc comment.
999
1017
  *
1000
- * Operates on the same `"FieldId":"<uuid>"` anchored search as
1001
- * applyFormSchemaOptionIdSubstitutions, but only fires when the FieldId is
1002
- * in fieldNameMap (has a semantic name) AND NOT in fieldOptionsMap (the
1003
- * structured enum substitution didn't fire). Site-agnostic.
1018
+ * Operates on the same field-id-anchored search as
1019
+ * applyFormSchemaOptionIdSubstitutions, but only fires when the field is in
1020
+ * fieldNameMap (has a semantic name) AND NOT in fieldOptionsMap (the structured
1021
+ * enum substitution didn't fire). Site-agnostic.
1004
1022
  */
1005
- function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields) {
1023
+ function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields, formSchema) {
1006
1024
  if (fieldNameMap.size === 0)
1007
1025
  return rawBody;
1008
1026
  let result = rawBody;
1027
+ const optionIdMarker = `"${formSchema.responseOptionIdKey}":"`;
1009
1028
  for (const [fieldId, fieldName] of fieldNameMap) {
1010
1029
  if (fieldOptionsMap.has(fieldId))
1011
1030
  continue; // T3's OPT_* already handles this.
1012
- const fieldIdMarker = `"FieldId":"${fieldId}"`;
1031
+ const fieldIdMarker = `"${formSchema.fieldIdKey}":"${fieldId}"`;
1013
1032
  let cursor = 0;
1014
1033
  while (true) {
1015
1034
  const idx = result.indexOf(fieldIdMarker, cursor);
@@ -1019,7 +1038,6 @@ function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOption
1019
1038
  if (objEnd === -1)
1020
1039
  break;
1021
1040
  const segment = result.slice(idx, objEnd);
1022
- const optionIdMarker = `"OptionId":"`;
1023
1041
  const optionIdLocal = segment.indexOf(optionIdMarker);
1024
1042
  if (optionIdLocal === -1) {
1025
1043
  cursor = objEnd;
@@ -1053,13 +1071,14 @@ function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOption
1053
1071
  * embedded base64 images, etc.) that aren't candidates for state threading. */
1054
1072
  const MAX_STATE_VALUE_LENGTH = 256;
1055
1073
  /** Canonical "uninitialized" sentinel values that some REST APIs return as
1056
- * placeholders before a downstream call populates the real identifier.
1057
- * ClearCompany's `/user/create` returns these for CandidateId/ApplicationId/
1058
- * ApplyProcessId, then `/user/start` returns the real values. Indexing the
1059
- * placeholder would lock the generated plugin's `${candidateId}` binding to
1060
- * the all-zero UUID — every downstream call would then 404 with "candidate
1061
- * does not exist". Closed set, literal-string match — never expand to
1062
- * pattern-based detection (would trip the no-regex-on-open-sets rule). */
1074
+ * placeholders before a downstream call populates the real identifier. An ATS
1075
+ * whose `/user/create` returns these for CandidateId/ApplicationId/
1076
+ * ApplyProcessId, then yields the real values on `/user/start`, is the case that
1077
+ * motivated this: indexing the placeholder would lock the generated plugin's
1078
+ * `${candidateId}` binding to the all-zero UUID — every downstream call would
1079
+ * then 404 with "candidate does not exist". Closed set, literal-string match —
1080
+ * never expand to pattern-based detection (would trip the no-regex-on-open-sets
1081
+ * rule). */
1063
1082
  const PLACEHOLDER_STATE_VALUES = new Set(["00000000-0000-0000-0000-000000000000"]);
1064
1083
  /**
1065
1084
  * Splits a raw `Set-Cookie` response-header string into `name`/`value` pairs.
@@ -1155,8 +1174,9 @@ function indexStateValues(captures, shieldedUuids = new Set(), actionCaptureIndi
1155
1174
  continue;
1156
1175
  if (PLACEHOLDER_STATE_VALUES.has(value))
1157
1176
  continue;
1158
- // Schema-identifier UUIDs (FieldId, OptionId) are stable anchors that
1159
- // T2/T3 substitution depends on remaining literal in body templates.
1177
+ // Schema-identifier UUIDs (the field-id and option-id anchors) are stable
1178
+ // anchors that T2/T3 substitution depends on remaining literal in body
1179
+ // templates.
1160
1180
  // Indexing them would let state-threading rewrite the anchors and
1161
1181
  // corrupt T2/T3's already-substituted Values.
1162
1182
  if (shieldedUuids.has(value))
@@ -1484,7 +1504,7 @@ function applyPayloadKeyValueSubstitutions(template, inputBody, additionalBodies
1484
1504
  // ── base64 Content parameterization ──────────────────────────────────────────
1485
1505
  /**
1486
1506
  * Maps a site's screening-question prompts to the payload field that answers
1487
- * them, as `{ FieldName: [keyword, …] }`.
1507
+ * them, as `{ payloadField: [keyword, …] }`.
1488
1508
  *
1489
1509
  * Empty by default and supplied by the operator via `RECON_QUESTION_KEYWORDS`
1490
1510
  * (JSON) — the engine cannot know what any site asks or what a caller's payload
@@ -1693,14 +1713,12 @@ function emitErrorSignalGuards(varName, urlPath, signals) {
1693
1713
  parentSegments.reverse();
1694
1714
  out.push(` const ${parentVar} = (${varName} as { ${parentObjType} }).${parentAccessor};`);
1695
1715
  for (const { errorKey } of errorKeys) {
1696
- const label = errorKey === "ResponseValidationErrors"
1697
- ? "validation errors"
1698
- : errorKey === "DataValidationErrors"
1699
- ? "data errors"
1700
- : errorKey
1701
- .replace(/([A-Z])/g, " $1")
1702
- .trim()
1703
- .toLowerCase();
1716
+ // Humanize the wire key generically (CamelCase → "camel case"); no
1717
+ // per-vendor special cases — the engine carries no vendor's key vocabulary.
1718
+ const label = errorKey
1719
+ .replace(/([A-Z])/g, " $1")
1720
+ .trim()
1721
+ .toLowerCase();
1704
1722
  out.push(` if (${parentVar} != null && ${parentVar}.${errorKey} != null) throw new Error(\`step ${varName} ${label}: \${JSON.stringify(${parentVar}.${errorKey})}\`);`);
1705
1723
  }
1706
1724
  }
@@ -1708,7 +1726,7 @@ function emitErrorSignalGuards(varName, urlPath, signals) {
1708
1726
  }
1709
1727
  /** Exported for unit testing — lets tests drive the multipart-upload code path directly
1710
1728
  * without going through the full emitContractTs pipeline. */
1711
- function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap, outDiscoveredFields, fieldOptionsMap, outDiscoveredOptionFields, outDiscoveredRawOptionFields, outDiscoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride = new Map()) {
1729
+ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap, outDiscoveredFields, fieldOptionsMap, outDiscoveredOptionFields, outDiscoveredRawOptionFields, outDiscoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride = new Map(), formSchema = null) {
1712
1730
  // Walk the first action's request body to map each leaf string value to its
1713
1731
  // `payload.<accessor>` expression. The emit's second interpolation pass uses
1714
1732
  // this to substitute literal occurrences (e.g. "Reginald") with their
@@ -1747,7 +1765,7 @@ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap
1747
1765
  outDiscoveredFields.add("BaseUrl");
1748
1766
  }
1749
1767
  // G2: register any tenant-subdomain header values as payload-supplied fields
1750
- // (e.g. ClearCompany's `API-ShortName: "addus"` becomes `payload.ApiShortName`).
1768
+ // (e.g. an `API-ShortName: "addus"` header becomes `payload.ApiShortName`).
1751
1769
  for (const [headerName, _value] of tenantSubdomainHeaders) {
1752
1770
  outDiscoveredFields.add(headerNameToPayloadFieldName(headerName));
1753
1771
  }
@@ -1775,13 +1793,16 @@ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap
1775
1793
  const prior = actions.slice(0, i);
1776
1794
  const url = interpolateStateValues(cap.url, prior, payloadAccessorByValue);
1777
1795
  // Form-schema substitution runs first on the raw recon body so its
1778
- // FieldId-anchored matches see the original JSON. State-threading and
1779
- // payload key-value passes then run on top. OptionId substitution runs
1780
- // here too: same closed-set FieldId anchor; rewrites "OptionId":"<uuid>"
1781
- // slots to "${OPT_X[payload.X]}" lookups.
1782
- const rawBodyWithFormSubs = cap.requestPostData
1783
- ? applyRawOptionIdPayloadSubstitutions(applyFormSchemaOptionIdSubstitutions(applyFormSchemaSubstitutions(cap.requestPostData, fieldNameMap, outDiscoveredFields), fieldOptionsMap, outDiscoveredOptionFields), fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields)
1784
- : "";
1796
+ // field-id-anchored matches see the original JSON. State-threading and
1797
+ // payload key-value passes then run on top. Option-id substitution runs
1798
+ // here too: same closed-set field-id anchor; rewrites the submitted
1799
+ // option-id slots to "${OPT_X[payload.X]}" lookups.
1800
+ // Form-schema passes only fire when a `--form-schema` was supplied (which is
1801
+ // also the only way the field maps are non-empty); without one they are
1802
+ // no-ops and the raw recon body flows straight through.
1803
+ const rawBodyWithFormSubs = cap.requestPostData && formSchema !== null
1804
+ ? applyRawOptionIdPayloadSubstitutions(applyFormSchemaOptionIdSubstitutions(applyFormSchemaSubstitutions(cap.requestPostData, fieldNameMap, outDiscoveredFields, formSchema), fieldOptionsMap, outDiscoveredOptionFields, formSchema), fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields, formSchema)
1805
+ : (cap.requestPostData ?? "");
1785
1806
  let bodyTemplate = rawBodyWithFormSubs
1786
1807
  ? applyPayloadKeyValueSubstitutions(interpolateStateValues(rawBodyWithFormSubs, prior, payloadAccessorByValue), inputBody, additionalBodies, outDiscoveredAdditionalBodyKeys)
1787
1808
  : "";
@@ -2013,6 +2034,13 @@ function emitContractTs(opts) {
2013
2034
  // Multi-step plugins thread responses through many different shapes that a
2014
2035
  // single Zod schema can't cover — use z.unknown() so each per-step access
2015
2036
  // compiles cleanly. Single-endpoint plugins keep the inferred schema.
2037
+ //
2038
+ // This is deliberate, not an unfinished schema: a submission flow's terminal
2039
+ // shape is the plugin's OWN contract with its caller (e.g. { verified: boolean }),
2040
+ // a field that appears in zero captured responses. Inferring a schema from the
2041
+ // captures would emit the wrong shape with false confidence. z.unknown() plus
2042
+ // the generated `[ ] Narrow ResponseSchema` checklist item is the intended
2043
+ // hand-off to the plugin author, who alone knows that contract.
2016
2044
  const responseSchemaExpr = multiStepBody ? `z.unknown()` : inferZodSchema(responseBody);
2017
2045
  // Multi-step flows that include a multipart upload need the binary asset
2018
2046
  // on the payload. Add Resume/ResumeContentType/ResumeFilename as required
@@ -2053,7 +2081,7 @@ function emitContractTs(opts) {
2053
2081
  .join("\n")}\n})`
2054
2082
  : "";
2055
2083
  // Build per-field OPT_<Name> constant declarations + payload-schema enum
2056
- // entries from the form schema's FieldOptions. Only fields whose OptionId
2084
+ // entries from the form schema's options. Only fields whose option-id
2057
2085
  // slots were actually rewritten in the body (i.e. that appear in
2058
2086
  // discoveredOptionFields) get emitted; the rest leave their schema entries
2059
2087
  // unused. Computed BEFORE payloadSchemaExpr so the extension string is
@@ -2080,16 +2108,16 @@ function emitContractTs(opts) {
2080
2108
  .map((m) => ` ${m.semanticName}: z.enum([${m.options.map((o) => JSON.stringify(o.value)).join(", ")}]),`)
2081
2109
  .join("\n")}\n})`
2082
2110
  : "";
2083
- // Phase E raw-option payload fields: FieldOptions whose .Value strings are
2084
- // empty in the schema (CustomFieldOption) — no semantic enum is possible,
2085
- // so the caller supplies the OptionId UUID directly. The recon-observed
2086
- // UUID is documented in a TSDoc comment so callers have a starting point.
2111
+ // Phase E raw-option payload fields: options whose label strings are empty in
2112
+ // the schema — no semantic enum is possible, so the caller supplies the
2113
+ // option-id UUID directly. The recon-observed UUID is documented in a TSDoc
2114
+ // comment so callers have a starting point.
2087
2115
  const sortedRawOptionEntries = discoveredRawOptionFields
2088
2116
  ? [...discoveredRawOptionFields.entries()].sort(([a], [b]) => a.localeCompare(b))
2089
2117
  : [];
2090
2118
  const rawOptionSchemaExtension = sortedRawOptionEntries.length > 0
2091
2119
  ? `.extend({\n${sortedRawOptionEntries
2092
- .map(([name, reconUuid]) => ` /** Recon-observed: ${reconUuid}. Caller supplies the OptionId UUID for this field. */\n ${name}: z.string(),`)
2120
+ .map(([name, reconUuid]) => ` /** Recon-observed: ${reconUuid}. Caller supplies the option-id UUID for this field. */\n ${name}: z.string(),`)
2093
2121
  .join("\n")}\n})`
2094
2122
  : "";
2095
2123
  // Phase F: additional-body keys (from action POSTs beyond r0). Each gets a
@@ -2125,24 +2153,24 @@ function emitContractTs(opts) {
2125
2153
  ? `${basePayloadSchemaExpr}.extend({\n Resume: z.instanceof(Buffer),\n ResumeContentType: z.string(),\n ResumeFilename: z.string(),\n})${formFieldsExtension}${splicedFieldsExtension}${optionSchemaExtension}${rawOptionSchemaExtension}${additionalBodyKeysExtension}${answersExtension}`
2126
2154
  : `${basePayloadSchemaExpr}${formFieldsExtension}${splicedFieldsExtension}${optionSchemaExtension}${rawOptionSchemaExtension}${additionalBodyKeysExtension}${answersExtension}`;
2127
2155
  // When the payload schema uses multipartBoolean(), import the shared helper
2128
- // from @/lib/zod-multipart so the generated file resolves the reference and
2129
- // doesn't re-inline the preprocess expression per boolean field.
2156
+ // so the generated file resolves the reference and doesn't re-inline the
2157
+ // preprocess expression per boolean field.
2130
2158
  const multipartBoolImport = hasMultipartStep
2131
- ? `import { multipartBoolean } from "../lib/zod-multipart";\n`
2159
+ ? `import { multipartBoolean } from "${ENGINE_PKG}/lib/zod-multipart";\n`
2132
2160
  : "";
2133
2161
  // Content-Type must be absent from multipart fetch calls so FormData can inject the boundary.
2134
2162
  const caseInsensitiveHeadersImport = hasMultipartStep
2135
- ? `import { omitHeaderCaseInsensitive } from "../lib/case-insensitive-headers";\n`
2163
+ ? `import { omitHeaderCaseInsensitive } from "${ENGINE_PKG}/lib/case-insensitive-headers";\n`
2136
2164
  : "";
2137
2165
  // Emit identifier-shaped keys unquoted so Biome's formatter doesn't rewrite
2138
2166
  // the generated file on first lint:fix.
2139
2167
  const headersLiteral = Object.entries(baseHeaders)
2140
2168
  .map(([k, v]) => ` ${isValidJsIdentifier(k) ? k : JSON.stringify(k)}: ${JSON.stringify(v)}`)
2141
2169
  .join(",\n");
2142
- const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "../scraper/fixtures";\n` : "";
2170
+ const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "${ENGINE_PKG}/scraper/fixtures";\n` : "";
2143
2171
  const clientImport = gql
2144
- ? `import { createGraphqlClient } from "../scraper/graphql-client";`
2145
- : `import { createHttpClient } from "../scraper/http-client";`;
2172
+ ? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";`
2173
+ : `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
2146
2174
  const queryConst = gql && gqlQuery
2147
2175
  ? `\n// Lifted verbatim from recon capture — trim UI-only fields before shipping.\nconst ${pascal.toUpperCase()}_QUERY = \`${gqlQuery.trim()}\`;\n`
2148
2176
  : "";
@@ -2206,8 +2234,8 @@ import Bottleneck from "bottleneck";
2206
2234
  import { z } from "zod/v4";
2207
2235
 
2208
2236
  ${fixtureImport}${caseInsensitiveHeadersImport}${multipartBoolImport}${clientImport}
2209
- import type { BrowserSession } from "../scraper/session";
2210
- import type { SitePlugin, SitePluginContext, SitePluginResult } from "../site-plugin";
2237
+ import type { BrowserSession } from "${ENGINE_PKG}/scraper/session";
2238
+ import type { SitePlugin, SitePluginContext, SitePluginResult } from "${ENGINE_PKG}/site-plugin";
2211
2239
  import { run${pascal}BrowserFlow } from "@/sites/${siteId}/flows/browser-flow";
2212
2240
 
2213
2241
  const BASE_HEADERS: Record<string, string> = {
@@ -2333,16 +2361,47 @@ function buildManifestInstruction(instruction, field) {
2333
2361
  `{{ .request.${field} }}` +
2334
2362
  instruction.slice(m.index + m[0].length));
2335
2363
  }
2364
+ /**
2365
+ * The JSON Schema `type` keyword for a sample value. Just the keyword, not a
2366
+ * full schema: the manifest is a scaffold a human narrows, so it needs the real
2367
+ * type a caller must send (`page` is a number, `filters` an array) without
2368
+ * duplicating {@link inferZodSchema}'s recursive shape inference. `null` and
2369
+ * `undefined` fall back to `string`, the safe default for a field a caller fills.
2370
+ */
2371
+ function jsonSchemaTypeOf(value) {
2372
+ if (Array.isArray(value))
2373
+ return "array";
2374
+ if (value === null || value === undefined)
2375
+ return "string";
2376
+ const t = typeof value;
2377
+ if (t === "number")
2378
+ return "number";
2379
+ if (t === "boolean")
2380
+ return "boolean";
2381
+ if (t === "object")
2382
+ return "object";
2383
+ return "string";
2384
+ }
2336
2385
  /**
2337
2386
  * Emits a config-only plugin manifest (`<siteId>.plugin.json`) from the recon
2338
2387
  * flow, as an alternative to the `.ts` trio for browser-only sites. Reuses the
2339
2388
  * SAME `resolveStepPayloadField` splice logic as the browser-flow emitter, so
2340
2389
  * every `{{ .request.<field> }}` reference also lands in the manifest's request
2341
- * schema — the two cannot drift. The direct-HTTP hot path is intentionally
2342
- * omitted; a site that needs it keeps the `.ts` path or wires `spec.httpModule`.
2390
+ * schema — the two cannot drift.
2391
+ *
2392
+ * `recovered` carries the request contract the `.ts` path infers from real
2393
+ * captures — the first POST body's fields plus form-schema discoveries — so
2394
+ * `--emit config` no longer throws that away and emit a request schema built
2395
+ * only from the handful of flow-step splice hints.
2396
+ *
2397
+ * When the site has a direct-HTTP path (a submission flow whose `.ts` emit would
2398
+ * carry an `executeHttp`), `httpModulePath` emits a `spec.httpModule` reference
2399
+ * to a compiled module the operator drops in — the config plugin's escape hatch
2400
+ * for the imperative hot path a JSON manifest cannot express. Absent that, the
2401
+ * browser `flow` is the only execution path, and the field is omitted.
2343
2402
  */
2344
2403
  function emitConfigManifest(opts) {
2345
- const { siteId, displayName, baseUrl, flowSteps, vocabulary } = opts;
2404
+ const { siteId, displayName, baseUrl, flowSteps, vocabulary, inputBody, recoveredFields, httpModulePath, } = opts;
2346
2405
  const payloadFieldNames = new Set();
2347
2406
  const steps = flowSteps.map((step) => {
2348
2407
  const isObj = typeof step !== "string";
@@ -2358,14 +2417,31 @@ function emitConfigManifest(opts) {
2358
2417
  return rewritten;
2359
2418
  return { step: rewritten, optional, upload, submitStep };
2360
2419
  });
2361
- const requestProperties = Object.fromEntries([...payloadFieldNames].sort().map((name) => [name, { type: "string" }]));
2420
+ // The request surface, widest wins: a flow splice, a recovered form field, or
2421
+ // a key from the first POST body all name something a caller controls. Splices
2422
+ // and recovered fields are strings (the browser flow fills them as text); a
2423
+ // body key keeps its captured type so a caller sends `page: 1`, not `"1"`.
2424
+ const requestProperties = {};
2425
+ for (const name of payloadFieldNames)
2426
+ requestProperties[name] = { type: "string" };
2427
+ for (const name of recoveredFields ?? [])
2428
+ requestProperties[name] = { type: "string" };
2429
+ if (inputBody !== null && typeof inputBody === "object" && !Array.isArray(inputBody)) {
2430
+ for (const [name, value] of Object.entries(inputBody)) {
2431
+ requestProperties[name] = { type: jsonSchemaTypeOf(value) };
2432
+ }
2433
+ }
2434
+ const sortedRequestProperties = Object.fromEntries(Object.keys(requestProperties)
2435
+ .sort()
2436
+ .map((name) => [name, requestProperties[name]]));
2362
2437
  const manifest = {
2363
2438
  apiVersion: plugin_manifest_envelope_1.CONFIG_PLUGIN_API_VERSION,
2364
2439
  kind: plugin_manifest_envelope_1.CONFIG_PLUGIN_KIND,
2365
2440
  metadata: { siteId, displayName },
2366
2441
  spec: {
2367
2442
  defaultBaseUrl: baseUrl,
2368
- request: { type: "object", properties: requestProperties },
2443
+ ...(httpModulePath ? { httpModule: httpModulePath } : {}),
2444
+ request: { type: "object", properties: sortedRequestProperties },
2369
2445
  response: {
2370
2446
  type: "object",
2371
2447
  description: "TODO: declare the fields this site returns (recon leaves this empty).",
@@ -2440,10 +2516,10 @@ function emitBrowserFlowTs(opts) {
2440
2516
  import type { Stagehand } from "@browserbasehq/stagehand";
2441
2517
  import { z } from "zod/v4";
2442
2518
 
2443
- import { buildAnthropicClient } from "../lib/llm/anthropic-client";
2444
- import { getLogger } from "../lib/logging";
2445
- import { type HealingFlowStep, runHealingFlow, waitForSpaReady } from "../scraper/flow-runner";
2446
- import { guardedExtract } from "../scraper/stagehand-guard";
2519
+ import { buildAnthropicClient } from "${ENGINE_PKG}/lib/llm/anthropic-client";
2520
+ import { getLogger } from "${ENGINE_PKG}/lib/logging";
2521
+ import { type HealingFlowStep, runHealingFlow, waitForSpaReady } from "${ENGINE_PKG}/scraper/flow-runner";
2522
+ import { guardedExtract } from "${ENGINE_PKG}/scraper/stagehand-guard";
2447
2523
  import type { ${pascal}Payload, ${pascal}Response } from "@/sites/${siteId}/contract";
2448
2524
 
2449
2525
  const logger = getLogger({ name: "${siteId}-browser-flow" });
@@ -2556,17 +2632,36 @@ async function resolveVocabulary(specifier, flowSteps) {
2556
2632
  }
2557
2633
  return DEPRECATED_BUILTIN_ATS_VOCABULARY;
2558
2634
  }
2635
+ /**
2636
+ * Resolves the form-schema for this run and reports which one is in play.
2637
+ *
2638
+ * The engine carries no vendor's wire format (issue #57): a consumer whose ATS
2639
+ * exposes a form definition declares its keys with `--form-schema`. Absent one
2640
+ * (or `--form-schema none`), form-key recovery does not run and the generator
2641
+ * recovers nothing from ATS-shaped responses — the same "absence means none"
2642
+ * discipline `--vocabulary` uses.
2643
+ */
2644
+ async function resolveFormSchema(specifier) {
2645
+ if (!specifier)
2646
+ return null;
2647
+ const formSchema = await (0, load_form_schema_1.loadReconFormSchema)(specifier, process.cwd());
2648
+ logger.info(`form-schema: ${specifier === load_form_schema_1.FORM_SCHEMA_NONE ? "none (no ATS form recovery)" : `custom keys from ${specifier}`}`);
2649
+ return formSchema;
2650
+ }
2559
2651
  async function main() {
2560
2652
  const args = process.argv.slice(2);
2561
2653
  let siteId = "";
2562
2654
  let force = false;
2563
2655
  let emit = "ts";
2564
2656
  let vocabularySpecifier = "";
2657
+ let formSchemaSpecifier = "";
2565
2658
  for (let i = 0; i < args.length; i++) {
2566
2659
  if (args[i] === "--site-id" && args[i + 1])
2567
2660
  siteId = args[++i];
2568
2661
  else if (args[i] === "--vocabulary" && args[i + 1])
2569
2662
  vocabularySpecifier = args[++i];
2663
+ else if (args[i] === "--form-schema" && args[i + 1])
2664
+ formSchemaSpecifier = args[++i];
2570
2665
  else if (args[i] === "--force")
2571
2666
  force = true;
2572
2667
  else if (args[i] === "--emit" && args[i + 1]) {
@@ -2637,6 +2732,9 @@ async function main() {
2637
2732
  // module-level const would freeze at import time, which is the bug that makes
2638
2733
  // RECON_QUESTION_KEYWORDS silently inert for anyone setting it after load.
2639
2734
  const vocabulary = await resolveVocabulary(vocabularySpecifier, flowSteps);
2735
+ // Consumer-supplied wire keys for ATS form-schema recovery, or null. When
2736
+ // null the recovery functions no-op — the engine hardcodes no vendor format.
2737
+ const formSchema = await resolveFormSchema(formSchemaSpecifier);
2640
2738
  const pascal = toPascalCase(siteId);
2641
2739
  const baseUrl = deriveBaseUrl(captures);
2642
2740
  const baseHeaders = deriveRequestHeaders(captures, replays, baseUrl);
@@ -2652,15 +2750,15 @@ async function main() {
2652
2750
  const rawActionCaptures = gql
2653
2751
  ? []
2654
2752
  : collapseRedundantPatches(extractActionSequence(captures, baseUrl));
2655
- // Form-schema detection runs BEFORE state-indexing so the FieldId/OptionId
2753
+ // Form-schema detection runs BEFORE state-indexing so the field-id/option-id
2656
2754
  // UUIDs can be shielded from indexing — those UUIDs are stable schema
2657
2755
  // anchors that T2/T3 substitution depends on remaining literal in body
2658
2756
  // templates.
2659
- const { fieldNameMap, fieldOptionsMap, allSchemaUuids } = detectFormSchemaFieldNames(captures);
2660
- // Shield ALL FieldId/OptionId UUIDs that appear in any schema response, not
2757
+ const { fieldNameMap, fieldOptionsMap, allSchemaUuids } = detectFormSchemaFieldNames(captures, formSchema);
2758
+ // Shield ALL field-id/option-id UUIDs that appear in any schema response, not
2661
2759
  // just the ones that detectFormSchemaFieldNames emits a payload name for.
2662
- // Some fields have FieldNames too long for the naming heuristic (>80 chars)
2663
- // and would be skipped by fieldNameMap; their FieldIds still need shielding
2760
+ // Some fields have names too long for the naming heuristic (>80 chars) and
2761
+ // would be skipped by fieldNameMap; their field-ids still need shielding
2664
2762
  // because they appear as anchors in the T2-substituted body templates.
2665
2763
  const shieldedUuids = new Set(allSchemaUuids);
2666
2764
  // T4 — Phase B+C: detect a form-schema GET capture and insert it into the
@@ -2670,7 +2768,7 @@ async function main() {
2670
2768
  // params (recon timestamps) from the captured URL so the emitted runtime
2671
2769
  // fetch uses a clean template. Sites without a schema-fetch capture
2672
2770
  // (rawSchemaFetch === null) get unchanged behavior.
2673
- const rawSchemaFetch = gql ? null : detectFormSchemaFetchCapture(captures, baseUrl);
2771
+ const rawSchemaFetch = gql || formSchema === null ? null : detectFormSchemaFetchCapture(captures, baseUrl, formSchema);
2674
2772
  const schemaFetchCleaned = rawSchemaFetch
2675
2773
  ? { ...rawSchemaFetch.capture, url: stripCacheBusterParams(rawSchemaFetch.capture.url) }
2676
2774
  : null;
@@ -2711,9 +2809,9 @@ async function main() {
2711
2809
  const errorSignals = detectErrorSignals(actionSteps);
2712
2810
  const discoveredFormFields = new Set();
2713
2811
  const discoveredOptionFields = new Set();
2714
- // Phase E: maps FieldName-derived raw-option payload field name (e.g.
2715
- // "AreYouOverTheAgeOf18OptionId") → recon-observed OptionId UUID. Used to
2716
- // emit `<FieldName>OptionId: z.string()` payload fields with TSDoc docs.
2812
+ // Phase E: maps label-derived raw-option payload field name (e.g.
2813
+ // "AreYouOverTheAgeOf18OptionId") → recon-observed option-id UUID. Used to
2814
+ // emit `<Name>OptionId: z.string()` payload fields with TSDoc docs.
2717
2815
  const discoveredRawOptionFields = new Map();
2718
2816
  // Phase F: keys from additional action POST bodies (beyond inputBody/r0)
2719
2817
  // that get parameterized. Recorded with their value type so the contract
@@ -2750,7 +2848,7 @@ async function main() {
2750
2848
  }
2751
2849
  }
2752
2850
  const multiStepBody = isSubmissionFlow
2753
- ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders)
2851
+ ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, new Map(), formSchema)
2754
2852
  : undefined;
2755
2853
  let base64ContentHelper = "";
2756
2854
  const base64PatchOverride = new Map();
@@ -2842,7 +2940,7 @@ async function main() {
2842
2940
  }
2843
2941
  }
2844
2942
  const processedMultiStepBody = isSubmissionFlow
2845
- ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride)
2943
+ ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride, formSchema)
2846
2944
  : multiStepBody;
2847
2945
  const hasMultipartStep = actionSteps.some((s) => s.isMultipart);
2848
2946
  const headerBindings = collectHeaderBindings(actionSteps);
@@ -2861,6 +2959,12 @@ async function main() {
2861
2959
  baseUrl,
2862
2960
  flowSteps,
2863
2961
  vocabulary,
2962
+ inputBody,
2963
+ recoveredFields: [...discoveredFormFields, ...discoveredOptionFields],
2964
+ // A submission flow is the case where the `.ts` emit carries an
2965
+ // executeHttp hot path; point the manifest at where the operator drops
2966
+ // the compiled module rather than silently dropping the direct path.
2967
+ httpModulePath: isSubmissionFlow ? `./${siteId}.http.js` : undefined,
2864
2968
  }));
2865
2969
  logger.info(`wrote ${manifestPath}`);
2866
2970
  logger.info(`done — review ${manifestPath}, fill in response/extract schemas, then load via BARNACLE_PLUGINS or BARNACLE_PLUGINS_CONFIG_DIR (no compile step)`);