@enricai/barnacle 1.4.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/dist/api/errors.d.ts +2 -1
- package/dist/api/errors.d.ts.map +1 -1
- package/dist/api/errors.js +3 -2
- package/dist/api/errors.js.map +1 -1
- package/dist/lib/llm/judge.d.ts +1 -1
- package/dist/lib/llm/judge.js +1 -1
- package/dist/recon/capture-filters.d.ts +40 -0
- package/dist/recon/capture-filters.d.ts.map +1 -0
- package/dist/recon/capture-filters.js +106 -0
- package/dist/recon/capture-filters.js.map +1 -0
- package/dist/recon/form-schema.d.ts +51 -0
- package/dist/recon/form-schema.d.ts.map +1 -0
- package/dist/recon/form-schema.js +20 -0
- package/dist/recon/form-schema.js.map +1 -0
- package/dist/recon/load-form-schema.d.ts +24 -0
- package/dist/recon/load-form-schema.d.ts.map +1 -0
- package/dist/recon/load-form-schema.js +66 -0
- package/dist/recon/load-form-schema.js.map +1 -0
- package/dist/scraper/errors.d.ts +14 -14
- package/dist/scraper/errors.d.ts.map +1 -1
- package/dist/scraper/errors.js +15 -15
- package/dist/scraper/errors.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts +12 -0
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +46 -8
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/http-client.d.ts +12 -0
- package/dist/scraper/http-client.d.ts.map +1 -1
- package/dist/scraper/http-client.js +11 -15
- package/dist/scraper/http-client.js.map +1 -1
- package/dist/scraper/parse-json-response.d.ts +7 -5
- package/dist/scraper/parse-json-response.d.ts.map +1 -1
- package/dist/scraper/parse-json-response.js +9 -8
- package/dist/scraper/parse-json-response.js.map +1 -1
- package/dist/scraper/rate-limited-json-client.d.ts +8 -1
- package/dist/scraper/rate-limited-json-client.d.ts.map +1 -1
- package/dist/scraper/rate-limited-json-client.js +6 -1
- package/dist/scraper/rate-limited-json-client.js.map +1 -1
- package/dist/scraper/session-browserbase.d.ts +6 -0
- package/dist/scraper/session-browserbase.d.ts.map +1 -1
- package/dist/scraper/session-browserbase.js +6 -0
- package/dist/scraper/session-browserbase.js.map +1 -1
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +3 -4
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +69 -20
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +277 -173
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/scripts/recon-http.d.ts +5 -2
- package/dist/scripts/recon-http.d.ts.map +1 -1
- package/dist/scripts/recon-http.js +16 -4
- package/dist/scripts/recon-http.js.map +1 -1
- package/dist/testing/persona-fixture.d.ts +2 -2
- package/dist/testing/persona-fixture.d.ts.map +1 -1
- package/dist/testing/persona-fixture.js +2 -2
- package/dist/testing/persona-fixture.js.map +1 -1
- package/package.json +9 -1
- package/dist/scraper/oracle-sentinels.d.ts +0 -22
- package/dist/scraper/oracle-sentinels.d.ts.map +0 -1
- package/dist/scraper/oracle-sentinels.js +0 -40
- package/dist/scraper/oracle-sentinels.js.map +0 -1
|
@@ -20,6 +20,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
20
20
|
exports.resolveStepPayloadField = resolveStepPayloadField;
|
|
21
21
|
exports.inferZodSchemaFromSamples = inferZodSchemaFromSamples;
|
|
22
22
|
exports.selectPayloadAction = selectPayloadAction;
|
|
23
|
+
exports.extractActionSequence = extractActionSequence;
|
|
24
|
+
exports.detectFormSchemaFieldNames = detectFormSchemaFieldNames;
|
|
23
25
|
exports.indexStateValues = indexStateValues;
|
|
24
26
|
exports.compileActionSteps = compileActionSteps;
|
|
25
27
|
exports.loadQuestionPromptKeywords = loadQuestionPromptKeywords;
|
|
@@ -34,10 +36,25 @@ const errors_1 = require("../lib/errors");
|
|
|
34
36
|
const logging_1 = require("../lib/logging");
|
|
35
37
|
const plugin_api_version_1 = require("../plugins/plugin-api-version");
|
|
36
38
|
const plugin_manifest_envelope_1 = require("../plugins/plugin-manifest-envelope");
|
|
39
|
+
const capture_filters_1 = require("../recon/capture-filters");
|
|
40
|
+
const load_form_schema_1 = require("../recon/load-form-schema");
|
|
37
41
|
const load_vocabulary_1 = require("../recon/load-vocabulary");
|
|
38
42
|
const vocabulary_1 = require("../recon/vocabulary");
|
|
39
43
|
const recon_shared_1 = require("../scripts/recon-shared");
|
|
40
44
|
const logger = (0, logging_1.getScriptLogger)("recon-generate");
|
|
45
|
+
/**
|
|
46
|
+
* Engine imports in GENERATED code must be package subpaths, never the `@/`
|
|
47
|
+
* alias, and the reason is not visible from the source: `tsc-alias` rewrites by
|
|
48
|
+
* text, so it cannot tell an import this module *uses* from one it *emits as a
|
|
49
|
+
* string*. Written as `@/scraper/session`, the build silently rewrote the
|
|
50
|
+
* template literal itself — shipping `dist/` emitters that generated
|
|
51
|
+
* `../scraper/session` and left every out-of-tree consumer with TS2307.
|
|
52
|
+
* (`@/sites/...` survived only because `src/sites/` is empty, so it resolved to
|
|
53
|
+
* no file.) A bare specifier has nothing to resolve against, so the build leaves
|
|
54
|
+
* it alone. `out-of-tree-e2e.test.ts` asserts this against the BUILT dist —
|
|
55
|
+
* asserting it against the source would pass while the shipped artifact is broken.
|
|
56
|
+
*/
|
|
57
|
+
const ENGINE_PKG = "@enricai/barnacle";
|
|
41
58
|
// ── helpers ──────────────────────────────────────────────────────────────────
|
|
42
59
|
function toPascalCase(siteId) {
|
|
43
60
|
return siteId
|
|
@@ -320,9 +337,8 @@ const IGNORE_REQUEST_HEADERS = new Set([
|
|
|
320
337
|
* stateless replay phase can't thread), derive headers from the meaningful
|
|
321
338
|
* action POSTs instead — same `extractActionSequence` definition used by
|
|
322
339
|
* the submission-flow detector. This catches load-bearing site-specific
|
|
323
|
-
* headers (
|
|
324
|
-
*
|
|
325
|
-
* know about any particular site.
|
|
340
|
+
* headers (a `X-CSRF-Token`, a `Job-Boards-API-Token`, an `API-ShortName`,
|
|
341
|
+
* etc.) without the generator needing to know about any particular site.
|
|
326
342
|
*/
|
|
327
343
|
function deriveRequestHeaders(captures, replays, baseUrl) {
|
|
328
344
|
const successfulUrls = new Set(replays.filter((r) => r.success).map((r) => endpointKey(r.url)));
|
|
@@ -389,30 +405,14 @@ function firstEndpointPath(captures) {
|
|
|
389
405
|
}
|
|
390
406
|
return "/api/search";
|
|
391
407
|
}
|
|
392
|
-
// ── multi-step "submission flow" detection ────────────────────────────────────
|
|
393
|
-
//
|
|
394
|
-
// For transactional sites (apply forms, multi-step checkout, etc.) the captures
|
|
395
|
-
// form an ordered sequence of POSTs that thread state values through subsequent
|
|
396
|
-
// requests (auth tokens, candidate IDs, application IDs). Single-endpoint
|
|
397
|
-
// sites (job search, pricing APIs) have one action capture and skip this path.
|
|
398
|
-
/**
|
|
399
|
-
* Path elements we always treat as noise (analytics, logging). Site-specific
|
|
400
|
-
* trackers belong in RECON_TELEMETRY_URL_PATTERNS (comma-separated), not here —
|
|
401
|
-
* the engine must not carry any one site's ad-tech domains.
|
|
402
|
-
*/
|
|
403
|
-
const TELEMETRY_URL_PATTERNS = [
|
|
404
|
-
"/util/logging/vweb/message",
|
|
405
|
-
"/blank/page",
|
|
406
|
-
"stats.g.doubleclick.net",
|
|
407
|
-
"google-analytics.com",
|
|
408
|
-
...(process.env.RECON_TELEMETRY_URL_PATTERNS ?? "")
|
|
409
|
-
.split(",")
|
|
410
|
-
.map((p) => p.trim())
|
|
411
|
-
.filter(Boolean),
|
|
412
|
-
];
|
|
413
408
|
/**
|
|
414
409
|
* Extracts the ordered sequence of meaningful POSTs that represent the
|
|
415
|
-
* transactional flow
|
|
410
|
+
* transactional flow: same-host 2xx POSTs, minus telemetry and error-reporting
|
|
411
|
+
* sinks. Assets need no filter of their own — they arrive as GETs.
|
|
412
|
+
*
|
|
413
|
+
* Exported for tests: this predicate decides what a generated plugin will POST
|
|
414
|
+
* at a live site, and it is the only gate between a browser's incidental
|
|
415
|
+
* chatter and the emitted hot path.
|
|
416
416
|
*/
|
|
417
417
|
function extractActionSequence(captures, baseUrl) {
|
|
418
418
|
let host;
|
|
@@ -438,7 +438,7 @@ function extractActionSequence(captures, baseUrl) {
|
|
|
438
438
|
}
|
|
439
439
|
if (captureHost !== host)
|
|
440
440
|
return false;
|
|
441
|
-
if (
|
|
441
|
+
if ((0, capture_filters_1.isNoiseUrl)(capture.url))
|
|
442
442
|
return false;
|
|
443
443
|
return true;
|
|
444
444
|
});
|
|
@@ -530,7 +530,7 @@ const KNOWN_TOP_LEVEL_ERROR_KEYS = new Set(["message", "error", "errormessage"])
|
|
|
530
530
|
/**
|
|
531
531
|
* Suffixes that mark a JSON key as carrying validation/data errors when its
|
|
532
532
|
* value is non-null. Case-sensitive because real APIs use mixed-case in the
|
|
533
|
-
* exact form they ship (e.g.
|
|
533
|
+
* exact form they ship (e.g. a `ResponseValidationErrors` key).
|
|
534
534
|
*/
|
|
535
535
|
const NESTED_ERROR_KEY_SUFFIXES = ["ValidationErrors", "DataErrors", "ValidationError"];
|
|
536
536
|
/**
|
|
@@ -540,7 +540,7 @@ const NESTED_ERROR_KEY_SUFFIXES = ["ValidationErrors", "DataErrors", "Validation
|
|
|
540
540
|
* responses (so legitimate success-only fields like `Name` aren't false-
|
|
541
541
|
* flagged as errors).
|
|
542
542
|
*
|
|
543
|
-
* Site-agnostic:
|
|
543
|
+
* Site-agnostic: one ATS may use `Message`/`Sections.ResponseValidationErrors`
|
|
544
544
|
* /`Sections.DataValidationErrors`; a different ATS using `error`/`errors[]`
|
|
545
545
|
* would emit guards for those instead.
|
|
546
546
|
*/
|
|
@@ -601,7 +601,7 @@ function walkForNestedErrorKeys(value, path, candidates) {
|
|
|
601
601
|
*/
|
|
602
602
|
const UUID_REGEX = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i;
|
|
603
603
|
/**
|
|
604
|
-
* Converts a
|
|
604
|
+
* Converts a machine code like "contact.first.name" or "address.country.subdivision"
|
|
605
605
|
* to PascalCase: "ContactFirstName", "AddressCountrySubdivision". Site-agnostic:
|
|
606
606
|
* operates only on the input string. Returns null for inputs that don't
|
|
607
607
|
* produce a valid JS identifier.
|
|
@@ -614,7 +614,7 @@ function sourceCodeToPascalCase(sourceCode) {
|
|
|
614
614
|
return isValidJsIdentifier(pascal) ? pascal : null;
|
|
615
615
|
}
|
|
616
616
|
/**
|
|
617
|
-
* Converts a free-form
|
|
617
|
+
* Converts a free-form field label like "Reference #1 First Name" or "Email" to
|
|
618
618
|
* PascalCase, stripping punctuation in a way that preserves position (so
|
|
619
619
|
* "Reference #1 First Name" → "Reference1FirstName" via a section-heading
|
|
620
620
|
* prefix). Site-agnostic: operates only on input strings.
|
|
@@ -650,57 +650,70 @@ function fieldNameToPascalCase(fieldName, prefix) {
|
|
|
650
650
|
}
|
|
651
651
|
/**
|
|
652
652
|
* Recon-driven detection of form-schema captures. Scans all response bodies
|
|
653
|
-
* for arrays whose objects look like
|
|
654
|
-
*
|
|
653
|
+
* for arrays whose objects look like form fields (a UUID-shaped field-id key
|
|
654
|
+
* plus at least one of the schema's name keys). Builds field-id → PascalCase
|
|
655
|
+
* name map.
|
|
656
|
+
*
|
|
657
|
+
* Site-agnostic: identifies form-schema captures by structural fingerprint
|
|
658
|
+
* against the consumer-supplied keys, not by URL or site name. Any ATS exposing
|
|
659
|
+
* a matching schema would match.
|
|
655
660
|
*
|
|
656
|
-
*
|
|
657
|
-
*
|
|
661
|
+
* Exported for unit testing — lets tests prove the rewiring: the same field is
|
|
662
|
+
* recovered whether the supplied keys are one vendor's names or a differing
|
|
663
|
+
* vendor's, and that a null schema recovers nothing.
|
|
658
664
|
*/
|
|
659
|
-
function detectFormSchemaFieldNames(captures) {
|
|
665
|
+
function detectFormSchemaFieldNames(captures, formSchema) {
|
|
660
666
|
const fieldNameMap = new Map();
|
|
661
667
|
const fieldOptionsMap = new Map();
|
|
662
668
|
const allSchemaUuids = new Set();
|
|
669
|
+
// No form-schema declared → recover nothing. The engine carries no vendor's
|
|
670
|
+
// wire keys, so there is nothing to fingerprint ATS responses against.
|
|
671
|
+
if (formSchema === null)
|
|
672
|
+
return { fieldNameMap, fieldOptionsMap, allSchemaUuids };
|
|
663
673
|
for (const capture of captures) {
|
|
664
|
-
walkForSectionFieldsArrays(capture.responseBody, fieldNameMap, fieldOptionsMap);
|
|
665
|
-
walkForSchemaUuids(capture.responseBody, allSchemaUuids);
|
|
674
|
+
walkForSectionFieldsArrays(capture.responseBody, fieldNameMap, fieldOptionsMap, formSchema);
|
|
675
|
+
walkForSchemaUuids(capture.responseBody, allSchemaUuids, formSchema);
|
|
666
676
|
}
|
|
667
677
|
return { fieldNameMap, fieldOptionsMap, allSchemaUuids };
|
|
668
678
|
}
|
|
669
679
|
/**
|
|
670
|
-
* Walks a response body collecting
|
|
671
|
-
*
|
|
672
|
-
*
|
|
673
|
-
*
|
|
674
|
-
*
|
|
675
|
-
*
|
|
676
|
-
*
|
|
680
|
+
* Walks a response body collecting UUID-shaped strings under the schema's
|
|
681
|
+
* field-id key, or under the option-id key of an entry in a sibling
|
|
682
|
+
* field-options array. These are stable schema anchors that must be shielded
|
|
683
|
+
* from state-threading even when detectFormSchemaFieldNames emits no
|
|
684
|
+
* payload-mappable name for the field (e.g. when the field name is too long for
|
|
685
|
+
* our naming heuristic).
|
|
686
|
+
*
|
|
687
|
+
* The wire keys come from the consumer-supplied {@link ReconFormSchema}, so a
|
|
688
|
+
* vendor declares its own keys with `--form-schema` rather than the engine
|
|
689
|
+
* hardcoding any — the inversion issue #57 asked for.
|
|
677
690
|
*/
|
|
678
|
-
function walkForSchemaUuids(value, out) {
|
|
691
|
+
function walkForSchemaUuids(value, out, formSchema) {
|
|
679
692
|
if (value === null || typeof value !== "object")
|
|
680
693
|
return;
|
|
681
694
|
if (Array.isArray(value)) {
|
|
682
695
|
for (const item of value)
|
|
683
|
-
walkForSchemaUuids(item, out);
|
|
696
|
+
walkForSchemaUuids(item, out, formSchema);
|
|
684
697
|
return;
|
|
685
698
|
}
|
|
686
699
|
const obj = value;
|
|
687
|
-
const fieldIdRaw = obj.
|
|
700
|
+
const fieldIdRaw = obj[formSchema.fieldIdKey];
|
|
688
701
|
if (typeof fieldIdRaw === "string" && UUID_REGEX.test(fieldIdRaw)) {
|
|
689
702
|
out.add(fieldIdRaw);
|
|
690
703
|
}
|
|
691
|
-
const optionsRaw = obj.
|
|
704
|
+
const optionsRaw = obj[formSchema.fieldOptionsKey];
|
|
692
705
|
if (Array.isArray(optionsRaw)) {
|
|
693
706
|
for (const opt of optionsRaw) {
|
|
694
707
|
if (opt !== null && typeof opt === "object") {
|
|
695
|
-
const optId = opt.
|
|
708
|
+
const optId = opt[formSchema.optionIdKey];
|
|
696
709
|
if (typeof optId === "string" && UUID_REGEX.test(optId))
|
|
697
710
|
out.add(optId);
|
|
698
711
|
}
|
|
699
712
|
}
|
|
700
713
|
}
|
|
701
|
-
// Recurse into nested objects/arrays so nested
|
|
714
|
+
// Recurse into nested objects/arrays so nested form fields get walked too.
|
|
702
715
|
for (const v of Object.values(obj))
|
|
703
|
-
walkForSchemaUuids(v, out);
|
|
716
|
+
walkForSchemaUuids(v, out, formSchema);
|
|
704
717
|
}
|
|
705
718
|
/**
|
|
706
719
|
* Closed enum of well-known cache-buster query parameter names. Stripped
|
|
@@ -723,14 +736,14 @@ function stripCacheBusterParams(url) {
|
|
|
723
736
|
}
|
|
724
737
|
/**
|
|
725
738
|
* Recon-driven detection of the form-schema fetch capture. Returns the first
|
|
726
|
-
* GET capture (in recon order) whose response body contains a
|
|
727
|
-
* shaped array. Sites without such a capture get
|
|
728
|
-
* become no-ops.
|
|
739
|
+
* GET capture (in recon order) whose response body contains a form-fields-
|
|
740
|
+
* shaped array (per the supplied schema keys). Sites without such a capture get
|
|
741
|
+
* `null` and Phase B/C/D become no-ops.
|
|
729
742
|
*
|
|
730
743
|
* Site-agnostic: identifies the fetch by structural fingerprint of the
|
|
731
744
|
* response body, not by URL or site name.
|
|
732
745
|
*/
|
|
733
|
-
function detectFormSchemaFetchCapture(captures, baseUrl) {
|
|
746
|
+
function detectFormSchemaFetchCapture(captures, baseUrl, formSchema) {
|
|
734
747
|
let host;
|
|
735
748
|
try {
|
|
736
749
|
host = new URL(baseUrl).host;
|
|
@@ -753,52 +766,52 @@ function detectFormSchemaFetchCapture(captures, baseUrl) {
|
|
|
753
766
|
}
|
|
754
767
|
if (captureHost !== host)
|
|
755
768
|
continue;
|
|
756
|
-
if (
|
|
769
|
+
if ((0, capture_filters_1.telemetryUrlPatterns)().some((p) => capture.url.includes(p)))
|
|
757
770
|
continue;
|
|
758
|
-
if (responseContainsSectionFields(capture.responseBody)) {
|
|
771
|
+
if (responseContainsSectionFields(capture.responseBody, formSchema)) {
|
|
759
772
|
return { capture, index: i };
|
|
760
773
|
}
|
|
761
774
|
}
|
|
762
775
|
return null;
|
|
763
776
|
}
|
|
764
|
-
function responseContainsSectionFields(value) {
|
|
777
|
+
function responseContainsSectionFields(value, formSchema) {
|
|
765
778
|
if (value === null || typeof value !== "object")
|
|
766
779
|
return false;
|
|
767
780
|
if (Array.isArray(value)) {
|
|
768
|
-
if (looksLikeSectionFieldsArray(value))
|
|
781
|
+
if (looksLikeSectionFieldsArray(value, formSchema))
|
|
769
782
|
return true;
|
|
770
783
|
for (const item of value) {
|
|
771
|
-
if (responseContainsSectionFields(item))
|
|
784
|
+
if (responseContainsSectionFields(item, formSchema))
|
|
772
785
|
return true;
|
|
773
786
|
}
|
|
774
787
|
return false;
|
|
775
788
|
}
|
|
776
789
|
for (const v of Object.values(value)) {
|
|
777
|
-
if (responseContainsSectionFields(v))
|
|
790
|
+
if (responseContainsSectionFields(v, formSchema))
|
|
778
791
|
return true;
|
|
779
792
|
}
|
|
780
793
|
return false;
|
|
781
794
|
}
|
|
782
|
-
function walkForSectionFieldsArrays(value, fieldNameMap, fieldOptionsMap) {
|
|
795
|
+
function walkForSectionFieldsArrays(value, fieldNameMap, fieldOptionsMap, formSchema) {
|
|
783
796
|
if (value === null || typeof value !== "object")
|
|
784
797
|
return;
|
|
785
798
|
if (Array.isArray(value)) {
|
|
786
|
-
if (looksLikeSectionFieldsArray(value)) {
|
|
787
|
-
assignFieldNamesFromArray(value, fieldNameMap, fieldOptionsMap);
|
|
799
|
+
if (looksLikeSectionFieldsArray(value, formSchema)) {
|
|
800
|
+
assignFieldNamesFromArray(value, fieldNameMap, fieldOptionsMap, formSchema);
|
|
788
801
|
}
|
|
789
802
|
for (const item of value)
|
|
790
|
-
walkForSectionFieldsArrays(item, fieldNameMap, fieldOptionsMap);
|
|
803
|
+
walkForSectionFieldsArrays(item, fieldNameMap, fieldOptionsMap, formSchema);
|
|
791
804
|
return;
|
|
792
805
|
}
|
|
793
806
|
for (const v of Object.values(value)) {
|
|
794
|
-
walkForSectionFieldsArrays(v, fieldNameMap, fieldOptionsMap);
|
|
807
|
+
walkForSectionFieldsArrays(v, fieldNameMap, fieldOptionsMap, formSchema);
|
|
795
808
|
}
|
|
796
809
|
}
|
|
797
810
|
/**
|
|
798
811
|
* Structural fingerprint: array of objects, at least half of which have a
|
|
799
|
-
* UUID-shaped
|
|
812
|
+
* UUID-shaped field-id AND at least one of the schema's field-name keys.
|
|
800
813
|
*/
|
|
801
|
-
function looksLikeSectionFieldsArray(arr) {
|
|
814
|
+
function looksLikeSectionFieldsArray(arr, formSchema) {
|
|
802
815
|
if (arr.length === 0)
|
|
803
816
|
return false;
|
|
804
817
|
let matches = 0;
|
|
@@ -806,26 +819,31 @@ function looksLikeSectionFieldsArray(arr) {
|
|
|
806
819
|
if (item === null || typeof item !== "object")
|
|
807
820
|
continue;
|
|
808
821
|
const obj = item;
|
|
809
|
-
const fieldIdRaw = obj.
|
|
822
|
+
const fieldIdRaw = obj[formSchema.fieldIdKey];
|
|
810
823
|
if (typeof fieldIdRaw !== "string")
|
|
811
824
|
continue;
|
|
812
825
|
if (!UUID_REGEX.test(fieldIdRaw))
|
|
813
826
|
continue;
|
|
814
|
-
if (
|
|
827
|
+
if (formSchema.fieldNameKeys.some((key) => typeof obj[key] === "string")) {
|
|
815
828
|
matches++;
|
|
816
829
|
}
|
|
817
830
|
}
|
|
818
831
|
return matches >= Math.max(1, Math.floor(arr.length * 0.5));
|
|
819
832
|
}
|
|
820
|
-
function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
|
|
833
|
+
function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap, formSchema) {
|
|
821
834
|
let currentPrefix = null;
|
|
822
835
|
const usedNames = new Set([...fieldNameMap.values()]);
|
|
836
|
+
// First field-name key is the machine code (preferred, PascalCased directly);
|
|
837
|
+
// any later key is a human label (subject to the section-heading heuristic).
|
|
838
|
+
// With one key both branches collapse to the label path.
|
|
839
|
+
const [codeKey, ...labelKeys] = formSchema.fieldNameKeys;
|
|
840
|
+
const labelKey = labelKeys[0];
|
|
823
841
|
for (const obj of arr) {
|
|
824
|
-
const fieldId = obj.
|
|
842
|
+
const fieldId = obj[formSchema.fieldIdKey];
|
|
825
843
|
if (typeof fieldId !== "string")
|
|
826
844
|
continue;
|
|
827
|
-
const sourceCode = obj
|
|
828
|
-
const name = obj
|
|
845
|
+
const sourceCode = codeKey !== undefined && labelKey !== undefined ? obj[codeKey] : undefined;
|
|
846
|
+
const name = labelKey !== undefined ? obj[labelKey] : obj[codeKey ?? ""];
|
|
829
847
|
let semantic = null;
|
|
830
848
|
if (typeof sourceCode === "string" && sourceCode.trim().length > 0) {
|
|
831
849
|
semantic = sourceCodeToPascalCase(sourceCode);
|
|
@@ -833,7 +851,7 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
|
|
|
833
851
|
}
|
|
834
852
|
else if (typeof name === "string" && name.trim().length > 0 && name.length < 250) {
|
|
835
853
|
const hasNoSourceCode = typeof sourceCode !== "string" || sourceCode.trim().length === 0;
|
|
836
|
-
// Section-heading heuristic: short
|
|
854
|
+
// Section-heading heuristic: short label, no machine code, MOSTLY
|
|
837
855
|
// uppercase letters (>= 70% of alphabetic chars) OR contains '#'.
|
|
838
856
|
// Whole-name uppercase ratio avoids false positives like "MM/DD/YYYY"
|
|
839
857
|
// appearing as a format hint inside a normal field label.
|
|
@@ -859,11 +877,11 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
|
|
|
859
877
|
}
|
|
860
878
|
fieldNameMap.set(fieldId, unique);
|
|
861
879
|
usedNames.add(unique);
|
|
862
|
-
// Capture
|
|
863
|
-
// semantic
|
|
864
|
-
//
|
|
865
|
-
//
|
|
866
|
-
const optionsRaw = obj.
|
|
880
|
+
// Capture the field's options when present and ALL options have non-empty
|
|
881
|
+
// semantic labels. Options with an empty label are skipped — with no
|
|
882
|
+
// semantic value we can't generate a meaningful enum, so we leave the
|
|
883
|
+
// field's option-id baked.
|
|
884
|
+
const optionsRaw = obj[formSchema.fieldOptionsKey];
|
|
867
885
|
if (Array.isArray(optionsRaw) && optionsRaw.length > 0) {
|
|
868
886
|
const options = [];
|
|
869
887
|
let allSemantic = true;
|
|
@@ -873,8 +891,8 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
|
|
|
873
891
|
break;
|
|
874
892
|
}
|
|
875
893
|
const opt = optRaw;
|
|
876
|
-
const optId = opt.
|
|
877
|
-
const optValue = opt.
|
|
894
|
+
const optId = opt[formSchema.optionIdKey];
|
|
895
|
+
const optValue = opt[formSchema.optionValueKey];
|
|
878
896
|
if (typeof optId !== "string" ||
|
|
879
897
|
typeof optValue !== "string" ||
|
|
880
898
|
optValue.trim().length === 0) {
|
|
@@ -891,20 +909,21 @@ function assignFieldNamesFromArray(arr, fieldNameMap, fieldOptionsMap) {
|
|
|
891
909
|
}
|
|
892
910
|
}
|
|
893
911
|
/**
|
|
894
|
-
* Substitutes
|
|
912
|
+
* Substitutes the submitted-value literals with payload accessors based on the
|
|
895
913
|
* field-name map from the form schema. Operates on the body string before
|
|
896
914
|
* state interpolation so already-substituted state values (e.g. ${firstName})
|
|
897
915
|
* are preserved.
|
|
898
916
|
*
|
|
899
|
-
* Closed-set substring matching: both
|
|
900
|
-
* the generator's own input (recon).
|
|
917
|
+
* Closed-set substring matching: both the field-id and the submitted value come
|
|
918
|
+
* from the generator's own input (recon).
|
|
901
919
|
*/
|
|
902
|
-
function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields) {
|
|
920
|
+
function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields, formSchema) {
|
|
903
921
|
if (fieldNameMap.size === 0)
|
|
904
922
|
return rawBody;
|
|
905
923
|
let result = rawBody;
|
|
924
|
+
const valueMarker = `"${formSchema.responseValueKey}":"`;
|
|
906
925
|
for (const [fieldId, semanticName] of fieldNameMap) {
|
|
907
|
-
const fieldIdMarker = `"
|
|
926
|
+
const fieldIdMarker = `"${formSchema.fieldIdKey}":"${fieldId}"`;
|
|
908
927
|
let cursor = 0;
|
|
909
928
|
while (true) {
|
|
910
929
|
const idx = result.indexOf(fieldIdMarker, cursor);
|
|
@@ -914,7 +933,6 @@ function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields
|
|
|
914
933
|
if (objEnd === -1)
|
|
915
934
|
break;
|
|
916
935
|
const segment = result.slice(idx, objEnd);
|
|
917
|
-
const valueMarker = `"Value":"`;
|
|
918
936
|
const valueIdx = segment.indexOf(valueMarker);
|
|
919
937
|
if (valueIdx === -1) {
|
|
920
938
|
cursor = objEnd;
|
|
@@ -940,22 +958,23 @@ function applyFormSchemaSubstitutions(rawBody, fieldNameMap, outDiscoveredFields
|
|
|
940
958
|
return result;
|
|
941
959
|
}
|
|
942
960
|
/**
|
|
943
|
-
* Substitutes
|
|
944
|
-
* Operates on the body string before state interpolation. For each
|
|
945
|
-
*
|
|
946
|
-
* the matching
|
|
961
|
+
* Substitutes submitted option-id literals with payload-driven enum lookups.
|
|
962
|
+
* Operates on the body string before state interpolation. For each field with a
|
|
963
|
+
* captured FieldOptionsMapping, find the schema's field-id marker and rewrite
|
|
964
|
+
* the matching option-id value to `${OPT_<Name>[payload.<Name>]}`.
|
|
947
965
|
*
|
|
948
|
-
* Order-insensitive: matches
|
|
949
|
-
* JSON object as the
|
|
950
|
-
*
|
|
951
|
-
*
|
|
966
|
+
* Order-insensitive: matches the option-id marker anywhere within the same
|
|
967
|
+
* JSON object as the field-id marker (which is between it and the closing `}`).
|
|
968
|
+
* Closed-set substring matching: both marker values come from the generator's
|
|
969
|
+
* own input.
|
|
952
970
|
*/
|
|
953
|
-
function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDiscoveredOptionFields) {
|
|
971
|
+
function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDiscoveredOptionFields, formSchema) {
|
|
954
972
|
if (fieldOptionsMap.size === 0)
|
|
955
973
|
return rawBody;
|
|
956
974
|
let result = rawBody;
|
|
975
|
+
const optionIdMarker = `"${formSchema.responseOptionIdKey}":"`;
|
|
957
976
|
for (const [fieldId, mapping] of fieldOptionsMap) {
|
|
958
|
-
const fieldIdMarker = `"
|
|
977
|
+
const fieldIdMarker = `"${formSchema.fieldIdKey}":"${fieldId}"`;
|
|
959
978
|
let cursor = 0;
|
|
960
979
|
while (true) {
|
|
961
980
|
const idx = result.indexOf(fieldIdMarker, cursor);
|
|
@@ -965,7 +984,6 @@ function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDisco
|
|
|
965
984
|
if (objEnd === -1)
|
|
966
985
|
break;
|
|
967
986
|
const segment = result.slice(idx, objEnd);
|
|
968
|
-
const optionIdMarker = `"OptionId":"`;
|
|
969
987
|
const optionIdLocal = segment.indexOf(optionIdMarker);
|
|
970
988
|
if (optionIdLocal === -1) {
|
|
971
989
|
cursor = objEnd;
|
|
@@ -991,25 +1009,26 @@ function applyFormSchemaOptionIdSubstitutions(rawBody, fieldOptionsMap, outDisco
|
|
|
991
1009
|
return result;
|
|
992
1010
|
}
|
|
993
1011
|
/**
|
|
994
|
-
* For fields whose
|
|
995
|
-
*
|
|
996
|
-
*
|
|
997
|
-
*
|
|
998
|
-
*
|
|
1012
|
+
* For fields whose options have NO semantic labels (the schema's option-label
|
|
1013
|
+
* value is empty), T3's OPT_* enum mapping can't be emitted. Instead,
|
|
1014
|
+
* parameterize the submitted option-id slot as a caller-supplied
|
|
1015
|
+
* `<Name>OptionId` payload field with the recon-observed UUID documented in a
|
|
1016
|
+
* TSDoc comment.
|
|
999
1017
|
*
|
|
1000
|
-
* Operates on the same
|
|
1001
|
-
* applyFormSchemaOptionIdSubstitutions, but only fires when the
|
|
1002
|
-
*
|
|
1003
|
-
*
|
|
1018
|
+
* Operates on the same field-id-anchored search as
|
|
1019
|
+
* applyFormSchemaOptionIdSubstitutions, but only fires when the field is in
|
|
1020
|
+
* fieldNameMap (has a semantic name) AND NOT in fieldOptionsMap (the structured
|
|
1021
|
+
* enum substitution didn't fire). Site-agnostic.
|
|
1004
1022
|
*/
|
|
1005
|
-
function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields) {
|
|
1023
|
+
function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields, formSchema) {
|
|
1006
1024
|
if (fieldNameMap.size === 0)
|
|
1007
1025
|
return rawBody;
|
|
1008
1026
|
let result = rawBody;
|
|
1027
|
+
const optionIdMarker = `"${formSchema.responseOptionIdKey}":"`;
|
|
1009
1028
|
for (const [fieldId, fieldName] of fieldNameMap) {
|
|
1010
1029
|
if (fieldOptionsMap.has(fieldId))
|
|
1011
1030
|
continue; // T3's OPT_* already handles this.
|
|
1012
|
-
const fieldIdMarker = `"
|
|
1031
|
+
const fieldIdMarker = `"${formSchema.fieldIdKey}":"${fieldId}"`;
|
|
1013
1032
|
let cursor = 0;
|
|
1014
1033
|
while (true) {
|
|
1015
1034
|
const idx = result.indexOf(fieldIdMarker, cursor);
|
|
@@ -1019,7 +1038,6 @@ function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOption
|
|
|
1019
1038
|
if (objEnd === -1)
|
|
1020
1039
|
break;
|
|
1021
1040
|
const segment = result.slice(idx, objEnd);
|
|
1022
|
-
const optionIdMarker = `"OptionId":"`;
|
|
1023
1041
|
const optionIdLocal = segment.indexOf(optionIdMarker);
|
|
1024
1042
|
if (optionIdLocal === -1) {
|
|
1025
1043
|
cursor = objEnd;
|
|
@@ -1053,13 +1071,14 @@ function applyRawOptionIdPayloadSubstitutions(rawBody, fieldNameMap, fieldOption
|
|
|
1053
1071
|
* embedded base64 images, etc.) that aren't candidates for state threading. */
|
|
1054
1072
|
const MAX_STATE_VALUE_LENGTH = 256;
|
|
1055
1073
|
/** Canonical "uninitialized" sentinel values that some REST APIs return as
|
|
1056
|
-
* placeholders before a downstream call populates the real identifier.
|
|
1057
|
-
*
|
|
1058
|
-
* ApplyProcessId, then `/user/start
|
|
1059
|
-
* placeholder would lock the generated plugin's
|
|
1060
|
-
* the all-zero UUID — every downstream call would
|
|
1061
|
-
* does not exist". Closed set, literal-string match —
|
|
1062
|
-
* pattern-based detection (would trip the no-regex-on-open-sets
|
|
1074
|
+
* placeholders before a downstream call populates the real identifier. An ATS
|
|
1075
|
+
* whose `/user/create` returns these for CandidateId/ApplicationId/
|
|
1076
|
+
* ApplyProcessId, then yields the real values on `/user/start`, is the case that
|
|
1077
|
+
* motivated this: indexing the placeholder would lock the generated plugin's
|
|
1078
|
+
* `${candidateId}` binding to the all-zero UUID — every downstream call would
|
|
1079
|
+
* then 404 with "candidate does not exist". Closed set, literal-string match —
|
|
1080
|
+
* never expand to pattern-based detection (would trip the no-regex-on-open-sets
|
|
1081
|
+
* rule). */
|
|
1063
1082
|
const PLACEHOLDER_STATE_VALUES = new Set(["00000000-0000-0000-0000-000000000000"]);
|
|
1064
1083
|
/**
|
|
1065
1084
|
* Splits a raw `Set-Cookie` response-header string into `name`/`value` pairs.
|
|
@@ -1155,8 +1174,9 @@ function indexStateValues(captures, shieldedUuids = new Set(), actionCaptureIndi
|
|
|
1155
1174
|
continue;
|
|
1156
1175
|
if (PLACEHOLDER_STATE_VALUES.has(value))
|
|
1157
1176
|
continue;
|
|
1158
|
-
// Schema-identifier UUIDs (
|
|
1159
|
-
// T2/T3 substitution depends on remaining literal in body
|
|
1177
|
+
// Schema-identifier UUIDs (the field-id and option-id anchors) are stable
|
|
1178
|
+
// anchors that T2/T3 substitution depends on remaining literal in body
|
|
1179
|
+
// templates.
|
|
1160
1180
|
// Indexing them would let state-threading rewrite the anchors and
|
|
1161
1181
|
// corrupt T2/T3's already-substituted Values.
|
|
1162
1182
|
if (shieldedUuids.has(value))
|
|
@@ -1484,7 +1504,7 @@ function applyPayloadKeyValueSubstitutions(template, inputBody, additionalBodies
|
|
|
1484
1504
|
// ── base64 Content parameterization ──────────────────────────────────────────
|
|
1485
1505
|
/**
|
|
1486
1506
|
* Maps a site's screening-question prompts to the payload field that answers
|
|
1487
|
-
* them, as `{
|
|
1507
|
+
* them, as `{ payloadField: [keyword, …] }`.
|
|
1488
1508
|
*
|
|
1489
1509
|
* Empty by default and supplied by the operator via `RECON_QUESTION_KEYWORDS`
|
|
1490
1510
|
* (JSON) — the engine cannot know what any site asks or what a caller's payload
|
|
@@ -1693,14 +1713,12 @@ function emitErrorSignalGuards(varName, urlPath, signals) {
|
|
|
1693
1713
|
parentSegments.reverse();
|
|
1694
1714
|
out.push(` const ${parentVar} = (${varName} as { ${parentObjType} }).${parentAccessor};`);
|
|
1695
1715
|
for (const { errorKey } of errorKeys) {
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
.trim()
|
|
1703
|
-
.toLowerCase();
|
|
1716
|
+
// Humanize the wire key generically (CamelCase → "camel case"); no
|
|
1717
|
+
// per-vendor special cases — the engine carries no vendor's key vocabulary.
|
|
1718
|
+
const label = errorKey
|
|
1719
|
+
.replace(/([A-Z])/g, " $1")
|
|
1720
|
+
.trim()
|
|
1721
|
+
.toLowerCase();
|
|
1704
1722
|
out.push(` if (${parentVar} != null && ${parentVar}.${errorKey} != null) throw new Error(\`step ${varName} ${label}: \${JSON.stringify(${parentVar}.${errorKey})}\`);`);
|
|
1705
1723
|
}
|
|
1706
1724
|
}
|
|
@@ -1708,7 +1726,7 @@ function emitErrorSignalGuards(varName, urlPath, signals) {
|
|
|
1708
1726
|
}
|
|
1709
1727
|
/** Exported for unit testing — lets tests drive the multipart-upload code path directly
|
|
1710
1728
|
* without going through the full emitContractTs pipeline. */
|
|
1711
|
-
function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap, outDiscoveredFields, fieldOptionsMap, outDiscoveredOptionFields, outDiscoveredRawOptionFields, outDiscoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride = new Map()) {
|
|
1729
|
+
function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap, outDiscoveredFields, fieldOptionsMap, outDiscoveredOptionFields, outDiscoveredRawOptionFields, outDiscoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride = new Map(), formSchema = null) {
|
|
1712
1730
|
// Walk the first action's request body to map each leaf string value to its
|
|
1713
1731
|
// `payload.<accessor>` expression. The emit's second interpolation pass uses
|
|
1714
1732
|
// this to substitute literal occurrences (e.g. "Reginald") with their
|
|
@@ -1747,7 +1765,7 @@ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap
|
|
|
1747
1765
|
outDiscoveredFields.add("BaseUrl");
|
|
1748
1766
|
}
|
|
1749
1767
|
// G2: register any tenant-subdomain header values as payload-supplied fields
|
|
1750
|
-
// (e.g.
|
|
1768
|
+
// (e.g. an `API-ShortName: "addus"` header becomes `payload.ApiShortName`).
|
|
1751
1769
|
for (const [headerName, _value] of tenantSubdomainHeaders) {
|
|
1752
1770
|
outDiscoveredFields.add(headerNameToPayloadFieldName(headerName));
|
|
1753
1771
|
}
|
|
@@ -1775,13 +1793,16 @@ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap
|
|
|
1775
1793
|
const prior = actions.slice(0, i);
|
|
1776
1794
|
const url = interpolateStateValues(cap.url, prior, payloadAccessorByValue);
|
|
1777
1795
|
// Form-schema substitution runs first on the raw recon body so its
|
|
1778
|
-
//
|
|
1779
|
-
// payload key-value passes then run on top.
|
|
1780
|
-
// here too: same closed-set
|
|
1781
|
-
// slots to "${OPT_X[payload.X]}" lookups.
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1796
|
+
// field-id-anchored matches see the original JSON. State-threading and
|
|
1797
|
+
// payload key-value passes then run on top. Option-id substitution runs
|
|
1798
|
+
// here too: same closed-set field-id anchor; rewrites the submitted
|
|
1799
|
+
// option-id slots to "${OPT_X[payload.X]}" lookups.
|
|
1800
|
+
// Form-schema passes only fire when a `--form-schema` was supplied (which is
|
|
1801
|
+
// also the only way the field maps are non-empty); without one they are
|
|
1802
|
+
// no-ops and the raw recon body flows straight through.
|
|
1803
|
+
const rawBodyWithFormSubs = cap.requestPostData && formSchema !== null
|
|
1804
|
+
? applyRawOptionIdPayloadSubstitutions(applyFormSchemaOptionIdSubstitutions(applyFormSchemaSubstitutions(cap.requestPostData, fieldNameMap, outDiscoveredFields, formSchema), fieldOptionsMap, outDiscoveredOptionFields, formSchema), fieldNameMap, fieldOptionsMap, outDiscoveredRawOptionFields, formSchema)
|
|
1805
|
+
: (cap.requestPostData ?? "");
|
|
1785
1806
|
let bodyTemplate = rawBodyWithFormSubs
|
|
1786
1807
|
? applyPayloadKeyValueSubstitutions(interpolateStateValues(rawBodyWithFormSubs, prior, payloadAccessorByValue), inputBody, additionalBodies, outDiscoveredAdditionalBodyKeys)
|
|
1787
1808
|
: "";
|
|
@@ -2013,6 +2034,13 @@ function emitContractTs(opts) {
|
|
|
2013
2034
|
// Multi-step plugins thread responses through many different shapes that a
|
|
2014
2035
|
// single Zod schema can't cover — use z.unknown() so each per-step access
|
|
2015
2036
|
// compiles cleanly. Single-endpoint plugins keep the inferred schema.
|
|
2037
|
+
//
|
|
2038
|
+
// This is deliberate, not an unfinished schema: a submission flow's terminal
|
|
2039
|
+
// shape is the plugin's OWN contract with its caller (e.g. { verified: boolean }),
|
|
2040
|
+
// a field that appears in zero captured responses. Inferring a schema from the
|
|
2041
|
+
// captures would emit the wrong shape with false confidence. z.unknown() plus
|
|
2042
|
+
// the generated `[ ] Narrow ResponseSchema` checklist item is the intended
|
|
2043
|
+
// hand-off to the plugin author, who alone knows that contract.
|
|
2016
2044
|
const responseSchemaExpr = multiStepBody ? `z.unknown()` : inferZodSchema(responseBody);
|
|
2017
2045
|
// Multi-step flows that include a multipart upload need the binary asset
|
|
2018
2046
|
// on the payload. Add Resume/ResumeContentType/ResumeFilename as required
|
|
@@ -2053,7 +2081,7 @@ function emitContractTs(opts) {
|
|
|
2053
2081
|
.join("\n")}\n})`
|
|
2054
2082
|
: "";
|
|
2055
2083
|
// Build per-field OPT_<Name> constant declarations + payload-schema enum
|
|
2056
|
-
// entries from the form schema's
|
|
2084
|
+
// entries from the form schema's options. Only fields whose option-id
|
|
2057
2085
|
// slots were actually rewritten in the body (i.e. that appear in
|
|
2058
2086
|
// discoveredOptionFields) get emitted; the rest leave their schema entries
|
|
2059
2087
|
// unused. Computed BEFORE payloadSchemaExpr so the extension string is
|
|
@@ -2080,16 +2108,16 @@ function emitContractTs(opts) {
|
|
|
2080
2108
|
.map((m) => ` ${m.semanticName}: z.enum([${m.options.map((o) => JSON.stringify(o.value)).join(", ")}]),`)
|
|
2081
2109
|
.join("\n")}\n})`
|
|
2082
2110
|
: "";
|
|
2083
|
-
// Phase E raw-option payload fields:
|
|
2084
|
-
//
|
|
2085
|
-
//
|
|
2086
|
-
//
|
|
2111
|
+
// Phase E raw-option payload fields: options whose label strings are empty in
|
|
2112
|
+
// the schema — no semantic enum is possible, so the caller supplies the
|
|
2113
|
+
// option-id UUID directly. The recon-observed UUID is documented in a TSDoc
|
|
2114
|
+
// comment so callers have a starting point.
|
|
2087
2115
|
const sortedRawOptionEntries = discoveredRawOptionFields
|
|
2088
2116
|
? [...discoveredRawOptionFields.entries()].sort(([a], [b]) => a.localeCompare(b))
|
|
2089
2117
|
: [];
|
|
2090
2118
|
const rawOptionSchemaExtension = sortedRawOptionEntries.length > 0
|
|
2091
2119
|
? `.extend({\n${sortedRawOptionEntries
|
|
2092
|
-
.map(([name, reconUuid]) => ` /** Recon-observed: ${reconUuid}. Caller supplies the
|
|
2120
|
+
.map(([name, reconUuid]) => ` /** Recon-observed: ${reconUuid}. Caller supplies the option-id UUID for this field. */\n ${name}: z.string(),`)
|
|
2093
2121
|
.join("\n")}\n})`
|
|
2094
2122
|
: "";
|
|
2095
2123
|
// Phase F: additional-body keys (from action POSTs beyond r0). Each gets a
|
|
@@ -2125,24 +2153,24 @@ function emitContractTs(opts) {
|
|
|
2125
2153
|
? `${basePayloadSchemaExpr}.extend({\n Resume: z.instanceof(Buffer),\n ResumeContentType: z.string(),\n ResumeFilename: z.string(),\n})${formFieldsExtension}${splicedFieldsExtension}${optionSchemaExtension}${rawOptionSchemaExtension}${additionalBodyKeysExtension}${answersExtension}`
|
|
2126
2154
|
: `${basePayloadSchemaExpr}${formFieldsExtension}${splicedFieldsExtension}${optionSchemaExtension}${rawOptionSchemaExtension}${additionalBodyKeysExtension}${answersExtension}`;
|
|
2127
2155
|
// When the payload schema uses multipartBoolean(), import the shared helper
|
|
2128
|
-
//
|
|
2129
|
-
//
|
|
2156
|
+
// so the generated file resolves the reference and doesn't re-inline the
|
|
2157
|
+
// preprocess expression per boolean field.
|
|
2130
2158
|
const multipartBoolImport = hasMultipartStep
|
|
2131
|
-
? `import { multipartBoolean } from "
|
|
2159
|
+
? `import { multipartBoolean } from "${ENGINE_PKG}/lib/zod-multipart";\n`
|
|
2132
2160
|
: "";
|
|
2133
2161
|
// Content-Type must be absent from multipart fetch calls so FormData can inject the boundary.
|
|
2134
2162
|
const caseInsensitiveHeadersImport = hasMultipartStep
|
|
2135
|
-
? `import { omitHeaderCaseInsensitive } from "
|
|
2163
|
+
? `import { omitHeaderCaseInsensitive } from "${ENGINE_PKG}/lib/case-insensitive-headers";\n`
|
|
2136
2164
|
: "";
|
|
2137
2165
|
// Emit identifier-shaped keys unquoted so Biome's formatter doesn't rewrite
|
|
2138
2166
|
// the generated file on first lint:fix.
|
|
2139
2167
|
const headersLiteral = Object.entries(baseHeaders)
|
|
2140
2168
|
.map(([k, v]) => ` ${isValidJsIdentifier(k) ? k : JSON.stringify(k)}: ${JSON.stringify(v)}`)
|
|
2141
2169
|
.join(",\n");
|
|
2142
|
-
const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "
|
|
2170
|
+
const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "${ENGINE_PKG}/scraper/fixtures";\n` : "";
|
|
2143
2171
|
const clientImport = gql
|
|
2144
|
-
? `import { createGraphqlClient } from "
|
|
2145
|
-
: `import { createHttpClient } from "
|
|
2172
|
+
? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";`
|
|
2173
|
+
: `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
|
|
2146
2174
|
const queryConst = gql && gqlQuery
|
|
2147
2175
|
? `\n// Lifted verbatim from recon capture — trim UI-only fields before shipping.\nconst ${pascal.toUpperCase()}_QUERY = \`${gqlQuery.trim()}\`;\n`
|
|
2148
2176
|
: "";
|
|
@@ -2206,8 +2234,8 @@ import Bottleneck from "bottleneck";
|
|
|
2206
2234
|
import { z } from "zod/v4";
|
|
2207
2235
|
|
|
2208
2236
|
${fixtureImport}${caseInsensitiveHeadersImport}${multipartBoolImport}${clientImport}
|
|
2209
|
-
import type { BrowserSession } from "
|
|
2210
|
-
import type { SitePlugin, SitePluginContext, SitePluginResult } from "
|
|
2237
|
+
import type { BrowserSession } from "${ENGINE_PKG}/scraper/session";
|
|
2238
|
+
import type { SitePlugin, SitePluginContext, SitePluginResult } from "${ENGINE_PKG}/site-plugin";
|
|
2211
2239
|
import { run${pascal}BrowserFlow } from "@/sites/${siteId}/flows/browser-flow";
|
|
2212
2240
|
|
|
2213
2241
|
const BASE_HEADERS: Record<string, string> = {
|
|
@@ -2333,16 +2361,47 @@ function buildManifestInstruction(instruction, field) {
|
|
|
2333
2361
|
`{{ .request.${field} }}` +
|
|
2334
2362
|
instruction.slice(m.index + m[0].length));
|
|
2335
2363
|
}
|
|
2364
|
+
/**
|
|
2365
|
+
* The JSON Schema `type` keyword for a sample value. Just the keyword, not a
|
|
2366
|
+
* full schema: the manifest is a scaffold a human narrows, so it needs the real
|
|
2367
|
+
* type a caller must send (`page` is a number, `filters` an array) without
|
|
2368
|
+
* duplicating {@link inferZodSchema}'s recursive shape inference. `null` and
|
|
2369
|
+
* `undefined` fall back to `string`, the safe default for a field a caller fills.
|
|
2370
|
+
*/
|
|
2371
|
+
function jsonSchemaTypeOf(value) {
|
|
2372
|
+
if (Array.isArray(value))
|
|
2373
|
+
return "array";
|
|
2374
|
+
if (value === null || value === undefined)
|
|
2375
|
+
return "string";
|
|
2376
|
+
const t = typeof value;
|
|
2377
|
+
if (t === "number")
|
|
2378
|
+
return "number";
|
|
2379
|
+
if (t === "boolean")
|
|
2380
|
+
return "boolean";
|
|
2381
|
+
if (t === "object")
|
|
2382
|
+
return "object";
|
|
2383
|
+
return "string";
|
|
2384
|
+
}
|
|
2336
2385
|
/**
|
|
2337
2386
|
* Emits a config-only plugin manifest (`<siteId>.plugin.json`) from the recon
|
|
2338
2387
|
* flow, as an alternative to the `.ts` trio for browser-only sites. Reuses the
|
|
2339
2388
|
* SAME `resolveStepPayloadField` splice logic as the browser-flow emitter, so
|
|
2340
2389
|
* every `{{ .request.<field> }}` reference also lands in the manifest's request
|
|
2341
|
-
* schema — the two cannot drift.
|
|
2342
|
-
*
|
|
2390
|
+
* schema — the two cannot drift.
|
|
2391
|
+
*
|
|
2392
|
+
* `recovered` carries the request contract the `.ts` path infers from real
|
|
2393
|
+
* captures — the first POST body's fields plus form-schema discoveries — so
|
|
2394
|
+
* `--emit config` no longer throws that away and emit a request schema built
|
|
2395
|
+
* only from the handful of flow-step splice hints.
|
|
2396
|
+
*
|
|
2397
|
+
* When the site has a direct-HTTP path (a submission flow whose `.ts` emit would
|
|
2398
|
+
* carry an `executeHttp`), `httpModulePath` emits a `spec.httpModule` reference
|
|
2399
|
+
* to a compiled module the operator drops in — the config plugin's escape hatch
|
|
2400
|
+
* for the imperative hot path a JSON manifest cannot express. Absent that, the
|
|
2401
|
+
* browser `flow` is the only execution path, and the field is omitted.
|
|
2343
2402
|
*/
|
|
2344
2403
|
function emitConfigManifest(opts) {
|
|
2345
|
-
const { siteId, displayName, baseUrl, flowSteps, vocabulary } = opts;
|
|
2404
|
+
const { siteId, displayName, baseUrl, flowSteps, vocabulary, inputBody, recoveredFields, httpModulePath, } = opts;
|
|
2346
2405
|
const payloadFieldNames = new Set();
|
|
2347
2406
|
const steps = flowSteps.map((step) => {
|
|
2348
2407
|
const isObj = typeof step !== "string";
|
|
@@ -2358,14 +2417,31 @@ function emitConfigManifest(opts) {
|
|
|
2358
2417
|
return rewritten;
|
|
2359
2418
|
return { step: rewritten, optional, upload, submitStep };
|
|
2360
2419
|
});
|
|
2361
|
-
|
|
2420
|
+
// The request surface, widest wins: a flow splice, a recovered form field, or
|
|
2421
|
+
// a key from the first POST body all name something a caller controls. Splices
|
|
2422
|
+
// and recovered fields are strings (the browser flow fills them as text); a
|
|
2423
|
+
// body key keeps its captured type so a caller sends `page: 1`, not `"1"`.
|
|
2424
|
+
const requestProperties = {};
|
|
2425
|
+
for (const name of payloadFieldNames)
|
|
2426
|
+
requestProperties[name] = { type: "string" };
|
|
2427
|
+
for (const name of recoveredFields ?? [])
|
|
2428
|
+
requestProperties[name] = { type: "string" };
|
|
2429
|
+
if (inputBody !== null && typeof inputBody === "object" && !Array.isArray(inputBody)) {
|
|
2430
|
+
for (const [name, value] of Object.entries(inputBody)) {
|
|
2431
|
+
requestProperties[name] = { type: jsonSchemaTypeOf(value) };
|
|
2432
|
+
}
|
|
2433
|
+
}
|
|
2434
|
+
const sortedRequestProperties = Object.fromEntries(Object.keys(requestProperties)
|
|
2435
|
+
.sort()
|
|
2436
|
+
.map((name) => [name, requestProperties[name]]));
|
|
2362
2437
|
const manifest = {
|
|
2363
2438
|
apiVersion: plugin_manifest_envelope_1.CONFIG_PLUGIN_API_VERSION,
|
|
2364
2439
|
kind: plugin_manifest_envelope_1.CONFIG_PLUGIN_KIND,
|
|
2365
2440
|
metadata: { siteId, displayName },
|
|
2366
2441
|
spec: {
|
|
2367
2442
|
defaultBaseUrl: baseUrl,
|
|
2368
|
-
|
|
2443
|
+
...(httpModulePath ? { httpModule: httpModulePath } : {}),
|
|
2444
|
+
request: { type: "object", properties: sortedRequestProperties },
|
|
2369
2445
|
response: {
|
|
2370
2446
|
type: "object",
|
|
2371
2447
|
description: "TODO: declare the fields this site returns (recon leaves this empty).",
|
|
@@ -2440,10 +2516,10 @@ function emitBrowserFlowTs(opts) {
|
|
|
2440
2516
|
import type { Stagehand } from "@browserbasehq/stagehand";
|
|
2441
2517
|
import { z } from "zod/v4";
|
|
2442
2518
|
|
|
2443
|
-
import { buildAnthropicClient } from "
|
|
2444
|
-
import { getLogger } from "
|
|
2445
|
-
import { type HealingFlowStep, runHealingFlow, waitForSpaReady } from "
|
|
2446
|
-
import { guardedExtract } from "
|
|
2519
|
+
import { buildAnthropicClient } from "${ENGINE_PKG}/lib/llm/anthropic-client";
|
|
2520
|
+
import { getLogger } from "${ENGINE_PKG}/lib/logging";
|
|
2521
|
+
import { type HealingFlowStep, runHealingFlow, waitForSpaReady } from "${ENGINE_PKG}/scraper/flow-runner";
|
|
2522
|
+
import { guardedExtract } from "${ENGINE_PKG}/scraper/stagehand-guard";
|
|
2447
2523
|
import type { ${pascal}Payload, ${pascal}Response } from "@/sites/${siteId}/contract";
|
|
2448
2524
|
|
|
2449
2525
|
const logger = getLogger({ name: "${siteId}-browser-flow" });
|
|
@@ -2556,17 +2632,36 @@ async function resolveVocabulary(specifier, flowSteps) {
|
|
|
2556
2632
|
}
|
|
2557
2633
|
return DEPRECATED_BUILTIN_ATS_VOCABULARY;
|
|
2558
2634
|
}
|
|
2635
|
+
/**
|
|
2636
|
+
* Resolves the form-schema for this run and reports which one is in play.
|
|
2637
|
+
*
|
|
2638
|
+
* The engine carries no vendor's wire format (issue #57): a consumer whose ATS
|
|
2639
|
+
* exposes a form definition declares its keys with `--form-schema`. Absent one
|
|
2640
|
+
* (or `--form-schema none`), form-key recovery does not run and the generator
|
|
2641
|
+
* recovers nothing from ATS-shaped responses — the same "absence means none"
|
|
2642
|
+
* discipline `--vocabulary` uses.
|
|
2643
|
+
*/
|
|
2644
|
+
async function resolveFormSchema(specifier) {
|
|
2645
|
+
if (!specifier)
|
|
2646
|
+
return null;
|
|
2647
|
+
const formSchema = await (0, load_form_schema_1.loadReconFormSchema)(specifier, process.cwd());
|
|
2648
|
+
logger.info(`form-schema: ${specifier === load_form_schema_1.FORM_SCHEMA_NONE ? "none (no ATS form recovery)" : `custom keys from ${specifier}`}`);
|
|
2649
|
+
return formSchema;
|
|
2650
|
+
}
|
|
2559
2651
|
async function main() {
|
|
2560
2652
|
const args = process.argv.slice(2);
|
|
2561
2653
|
let siteId = "";
|
|
2562
2654
|
let force = false;
|
|
2563
2655
|
let emit = "ts";
|
|
2564
2656
|
let vocabularySpecifier = "";
|
|
2657
|
+
let formSchemaSpecifier = "";
|
|
2565
2658
|
for (let i = 0; i < args.length; i++) {
|
|
2566
2659
|
if (args[i] === "--site-id" && args[i + 1])
|
|
2567
2660
|
siteId = args[++i];
|
|
2568
2661
|
else if (args[i] === "--vocabulary" && args[i + 1])
|
|
2569
2662
|
vocabularySpecifier = args[++i];
|
|
2663
|
+
else if (args[i] === "--form-schema" && args[i + 1])
|
|
2664
|
+
formSchemaSpecifier = args[++i];
|
|
2570
2665
|
else if (args[i] === "--force")
|
|
2571
2666
|
force = true;
|
|
2572
2667
|
else if (args[i] === "--emit" && args[i + 1]) {
|
|
@@ -2637,6 +2732,9 @@ async function main() {
|
|
|
2637
2732
|
// module-level const would freeze at import time, which is the bug that makes
|
|
2638
2733
|
// RECON_QUESTION_KEYWORDS silently inert for anyone setting it after load.
|
|
2639
2734
|
const vocabulary = await resolveVocabulary(vocabularySpecifier, flowSteps);
|
|
2735
|
+
// Consumer-supplied wire keys for ATS form-schema recovery, or null. When
|
|
2736
|
+
// null the recovery functions no-op — the engine hardcodes no vendor format.
|
|
2737
|
+
const formSchema = await resolveFormSchema(formSchemaSpecifier);
|
|
2640
2738
|
const pascal = toPascalCase(siteId);
|
|
2641
2739
|
const baseUrl = deriveBaseUrl(captures);
|
|
2642
2740
|
const baseHeaders = deriveRequestHeaders(captures, replays, baseUrl);
|
|
@@ -2652,15 +2750,15 @@ async function main() {
|
|
|
2652
2750
|
const rawActionCaptures = gql
|
|
2653
2751
|
? []
|
|
2654
2752
|
: collapseRedundantPatches(extractActionSequence(captures, baseUrl));
|
|
2655
|
-
// Form-schema detection runs BEFORE state-indexing so the
|
|
2753
|
+
// Form-schema detection runs BEFORE state-indexing so the field-id/option-id
|
|
2656
2754
|
// UUIDs can be shielded from indexing — those UUIDs are stable schema
|
|
2657
2755
|
// anchors that T2/T3 substitution depends on remaining literal in body
|
|
2658
2756
|
// templates.
|
|
2659
|
-
const { fieldNameMap, fieldOptionsMap, allSchemaUuids } = detectFormSchemaFieldNames(captures);
|
|
2660
|
-
// Shield ALL
|
|
2757
|
+
const { fieldNameMap, fieldOptionsMap, allSchemaUuids } = detectFormSchemaFieldNames(captures, formSchema);
|
|
2758
|
+
// Shield ALL field-id/option-id UUIDs that appear in any schema response, not
|
|
2661
2759
|
// just the ones that detectFormSchemaFieldNames emits a payload name for.
|
|
2662
|
-
// Some fields have
|
|
2663
|
-
//
|
|
2760
|
+
// Some fields have names too long for the naming heuristic (>80 chars) and
|
|
2761
|
+
// would be skipped by fieldNameMap; their field-ids still need shielding
|
|
2664
2762
|
// because they appear as anchors in the T2-substituted body templates.
|
|
2665
2763
|
const shieldedUuids = new Set(allSchemaUuids);
|
|
2666
2764
|
// T4 — Phase B+C: detect a form-schema GET capture and insert it into the
|
|
@@ -2670,7 +2768,7 @@ async function main() {
|
|
|
2670
2768
|
// params (recon timestamps) from the captured URL so the emitted runtime
|
|
2671
2769
|
// fetch uses a clean template. Sites without a schema-fetch capture
|
|
2672
2770
|
// (rawSchemaFetch === null) get unchanged behavior.
|
|
2673
|
-
const rawSchemaFetch = gql ? null : detectFormSchemaFetchCapture(captures, baseUrl);
|
|
2771
|
+
const rawSchemaFetch = gql || formSchema === null ? null : detectFormSchemaFetchCapture(captures, baseUrl, formSchema);
|
|
2674
2772
|
const schemaFetchCleaned = rawSchemaFetch
|
|
2675
2773
|
? { ...rawSchemaFetch.capture, url: stripCacheBusterParams(rawSchemaFetch.capture.url) }
|
|
2676
2774
|
: null;
|
|
@@ -2711,9 +2809,9 @@ async function main() {
|
|
|
2711
2809
|
const errorSignals = detectErrorSignals(actionSteps);
|
|
2712
2810
|
const discoveredFormFields = new Set();
|
|
2713
2811
|
const discoveredOptionFields = new Set();
|
|
2714
|
-
// Phase E: maps
|
|
2715
|
-
// "AreYouOverTheAgeOf18OptionId") → recon-observed
|
|
2716
|
-
// emit `<
|
|
2812
|
+
// Phase E: maps label-derived raw-option payload field name (e.g.
|
|
2813
|
+
// "AreYouOverTheAgeOf18OptionId") → recon-observed option-id UUID. Used to
|
|
2814
|
+
// emit `<Name>OptionId: z.string()` payload fields with TSDoc docs.
|
|
2717
2815
|
const discoveredRawOptionFields = new Map();
|
|
2718
2816
|
// Phase F: keys from additional action POST bodies (beyond inputBody/r0)
|
|
2719
2817
|
// that get parameterized. Recorded with their value type so the contract
|
|
@@ -2750,7 +2848,7 @@ async function main() {
|
|
|
2750
2848
|
}
|
|
2751
2849
|
}
|
|
2752
2850
|
const multiStepBody = isSubmissionFlow
|
|
2753
|
-
? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders)
|
|
2851
|
+
? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, new Map(), formSchema)
|
|
2754
2852
|
: undefined;
|
|
2755
2853
|
let base64ContentHelper = "";
|
|
2756
2854
|
const base64PatchOverride = new Map();
|
|
@@ -2842,7 +2940,7 @@ async function main() {
|
|
|
2842
2940
|
}
|
|
2843
2941
|
}
|
|
2844
2942
|
const processedMultiStepBody = isSubmissionFlow
|
|
2845
|
-
? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride)
|
|
2943
|
+
? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride, formSchema)
|
|
2846
2944
|
: multiStepBody;
|
|
2847
2945
|
const hasMultipartStep = actionSteps.some((s) => s.isMultipart);
|
|
2848
2946
|
const headerBindings = collectHeaderBindings(actionSteps);
|
|
@@ -2861,6 +2959,12 @@ async function main() {
|
|
|
2861
2959
|
baseUrl,
|
|
2862
2960
|
flowSteps,
|
|
2863
2961
|
vocabulary,
|
|
2962
|
+
inputBody,
|
|
2963
|
+
recoveredFields: [...discoveredFormFields, ...discoveredOptionFields],
|
|
2964
|
+
// A submission flow is the case where the `.ts` emit carries an
|
|
2965
|
+
// executeHttp hot path; point the manifest at where the operator drops
|
|
2966
|
+
// the compiled module rather than silently dropping the direct path.
|
|
2967
|
+
httpModulePath: isSubmissionFlow ? `./${siteId}.http.js` : undefined,
|
|
2864
2968
|
}));
|
|
2865
2969
|
logger.info(`wrote ${manifestPath}`);
|
|
2866
2970
|
logger.info(`done — review ${manifestPath}, fill in response/extract schemas, then load via BARNACLE_PLUGINS or BARNACLE_PLUGINS_CONFIG_DIR (no compile step)`);
|