@enricai/barnacle 1.12.17 → 1.12.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1211
- package/dist/scraper/errors.d.ts +7 -7
- package/dist/scraper/errors.js +7 -7
- package/dist/scraper/fake-timer-advance.d.ts +35 -8
- package/dist/scraper/fake-timer-advance.d.ts.map +1 -1
- package/dist/scraper/fake-timer-advance.js +29 -9
- package/dist/scraper/fake-timer-advance.js.map +1 -1
- package/dist/scraper/http-client.d.ts +1 -1
- package/dist/scraper/http-client.js +1 -1
- package/dist/scraper/pool.d.ts +3 -3
- package/dist/scraper/pool.js +5 -5
- package/dist/scripts/recon-browser.d.ts +2 -1
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +2 -1
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +30 -3
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +212 -43
- package/dist/scripts/recon-generate.js.map +1 -1
- package/dist/scripts/recon-heal.d.ts +3 -3
- package/dist/scripts/recon-heal.js +4 -4
- package/dist/scripts/recon-heal.js.map +1 -1
- package/dist/server.js +1 -1
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
|
@@ -33,6 +33,7 @@ exports.selectPayloadAction = selectPayloadAction;
|
|
|
33
33
|
exports.selectReturnAction = selectReturnAction;
|
|
34
34
|
exports.selectEffectiveResponseBody = selectEffectiveResponseBody;
|
|
35
35
|
exports.extractEntryUrlParams = extractEntryUrlParams;
|
|
36
|
+
exports.gatherResponseBodySamples = gatherResponseBodySamples;
|
|
36
37
|
exports.selectPrimaryGraphQLOperation = selectPrimaryGraphQLOperation;
|
|
37
38
|
exports.firstEndpointCapture = firstEndpointCapture;
|
|
38
39
|
exports.firstEndpointPath = firstEndpointPath;
|
|
@@ -550,7 +551,7 @@ function inferZodSchemaFromSamples(samples, depth = 0, indent = "", opts = {}) {
|
|
|
550
551
|
}
|
|
551
552
|
if (kind === "object") {
|
|
552
553
|
const objects = nonNull;
|
|
553
|
-
const keys = [...new Set(objects.flatMap((o) => Object.keys(o)))];
|
|
554
|
+
const keys = [...new Set(objects.flatMap((o) => Object.keys(o)))].filter((k) => !(opts.looseServerResponse && k === "__typename"));
|
|
554
555
|
if (keys.length === 0)
|
|
555
556
|
return wrap("z.record(z.string(), z.unknown())");
|
|
556
557
|
const inner = `${indent} `;
|
|
@@ -569,7 +570,8 @@ function inferZodSchemaFromSamples(samples, depth = 0, indent = "", opts = {}) {
|
|
|
569
570
|
return `${inner}${isValidJsIdentifier(k) ? k : JSON.stringify(k)}: ${optional}`;
|
|
570
571
|
})
|
|
571
572
|
.join(",\n");
|
|
572
|
-
|
|
573
|
+
const objectExpr = `z.object({\n${fields},\n${indent}})${opts.looseServerResponse ? ".loose()" : ""}`;
|
|
574
|
+
return wrap(objectExpr);
|
|
573
575
|
}
|
|
574
576
|
return wrap("z.unknown()");
|
|
575
577
|
}
|
|
@@ -687,8 +689,35 @@ function selectEffectiveResponseBody(isSubmissionFlow, actionSteps, replayRespon
|
|
|
687
689
|
return replayResponseBody;
|
|
688
690
|
return selectReturnAction(actionSteps)?.capture.responseBody ?? replayResponseBody;
|
|
689
691
|
}
|
|
690
|
-
|
|
691
|
-
|
|
692
|
+
/**
|
|
693
|
+
* A capture's own URL is not guaranteed parseable (see the try/catch in
|
|
694
|
+
* `deriveBaseUrl` and `firstEndpointCapture` below), so host-provenance
|
|
695
|
+
* filtering must not throw on a malformed one — it just never matches.
|
|
696
|
+
*/
|
|
697
|
+
function captureHostname(url) {
|
|
698
|
+
try {
|
|
699
|
+
return new URL(url).hostname;
|
|
700
|
+
}
|
|
701
|
+
catch {
|
|
702
|
+
return "";
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* `baseUrl` itself is what {@link registrableDomain}'s fallback would be
|
|
707
|
+
* derived FROM, so at this point in generation the registrable-domain gate
|
|
708
|
+
* doesn't exist yet: when the flow declares no `ownBackendHostnames`, the
|
|
709
|
+
* best available signal is {@link isNoiseUrl}'s conservative exclusion of
|
|
710
|
+
* known third-party asset/tracking hosts (the same fallback
|
|
711
|
+
* `selectAuxFixtureCandidates` uses regardless of host-provenance data).
|
|
712
|
+
*/
|
|
713
|
+
function deriveBaseUrl(captures, ownBackendHostnames) {
|
|
714
|
+
const candidates = captures.filter((c) => {
|
|
715
|
+
if (ownBackendHostnames.length > 0) {
|
|
716
|
+
return (0, capture_filters_1.isAllowedFixtureHost)(captureHostname(c.url), ownBackendHostnames, null);
|
|
717
|
+
}
|
|
718
|
+
return !(0, capture_filters_1.isNoiseUrl)(c.url);
|
|
719
|
+
});
|
|
720
|
+
for (const c of candidates) {
|
|
692
721
|
try {
|
|
693
722
|
const u = new URL(c.url);
|
|
694
723
|
return `${u.protocol}//${u.host}`;
|
|
@@ -805,20 +834,94 @@ function deriveRequestHeaders(captures, replays, baseUrl, submitPatterns = null)
|
|
|
805
834
|
function isGraphQL(captures) {
|
|
806
835
|
return captures.some((c) => c.operationName !== null);
|
|
807
836
|
}
|
|
808
|
-
|
|
809
|
-
|
|
837
|
+
/**
|
|
838
|
+
* The single own-backend capture the raw GraphQL fallback (no
|
|
839
|
+
* `selectPrimaryGraphQLOperation` winner) resolves its query, endpoint, and
|
|
840
|
+
* operationName from — all three MUST trace back to this same capture, not
|
|
841
|
+
* three independent array scans that could each land on a different one.
|
|
842
|
+
*/
|
|
843
|
+
function firstGraphQLCapture(captures, ownBackendHostnames = [], fallbackDomain = null) {
|
|
844
|
+
const hasHostProvenance = ownBackendHostnames.length > 0 || fallbackDomain !== null;
|
|
845
|
+
return (captures.find((c) => c.query &&
|
|
846
|
+
(!hasHostProvenance ||
|
|
847
|
+
(0, capture_filters_1.isAllowedFixtureHost)(captureHostname(c.url), ownBackendHostnames, fallbackDomain))) ?? null);
|
|
848
|
+
}
|
|
849
|
+
/**
|
|
850
|
+
* Strips leading blank lines and `#`-comment lines (GraphQL comments run
|
|
851
|
+
* from `#` to end of line) so operation-signature regexes anchored at the
|
|
852
|
+
* `query`/`mutation` keyword still match documents preceded by a comment.
|
|
853
|
+
*/
|
|
854
|
+
function stripLeadingGraphQLComments(query) {
|
|
855
|
+
return query.replace(/^(?:[ \t]*(?:#[^\n]*)?\r?\n)*/, "");
|
|
810
856
|
}
|
|
811
857
|
/**
|
|
812
858
|
* Parses `$name:` declarations out of a GraphQL operation signature, e.g.
|
|
813
|
-
* `query
|
|
859
|
+
* `query catalogSearch_Catalog($filters: String, $qualifiers: String)` yields
|
|
814
860
|
* `["filters", "qualifiers"]`.
|
|
815
861
|
*/
|
|
816
862
|
function declaredOperationVariableNames(query) {
|
|
817
|
-
const signature = /^\s*(?:query|mutation)\s+\w*\s*\(([^)]*)\)/.exec(query);
|
|
863
|
+
const signature = /^\s*(?:query|mutation)\s+\w*\s*\(([^)]*)\)/.exec(stripLeadingGraphQLComments(query));
|
|
818
864
|
if (!signature)
|
|
819
865
|
return [];
|
|
820
866
|
return Array.from(signature[1].matchAll(/\$(\w+)\s*:/g), (m) => m[1]);
|
|
821
867
|
}
|
|
868
|
+
/**
|
|
869
|
+
* Parses the operation name out of the query body itself, for captures whose
|
|
870
|
+
* top-level `operationName` field is null (an inline document with no
|
|
871
|
+
* separate operationName was still sent with a named `query`/`mutation`).
|
|
872
|
+
*/
|
|
873
|
+
function parsedOperationName(query) {
|
|
874
|
+
const signature = /^\s*(?:query|mutation)\s+(\w+)/.exec(stripLeadingGraphQLComments(query));
|
|
875
|
+
return signature?.[1] ?? null;
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Groups a capture for recurrence counting by endpoint path + operation
|
|
879
|
+
* name (falling back to the name parsed out of the query body when
|
|
880
|
+
* `operationName` is null), so operationName-less inline documents that
|
|
881
|
+
* repeat the same underlying query are recognized as recurring instead of
|
|
882
|
+
* being permanently unjoinable.
|
|
883
|
+
*/
|
|
884
|
+
function operationGroupKey(capture) {
|
|
885
|
+
const endpointPath = (() => {
|
|
886
|
+
try {
|
|
887
|
+
return new URL(capture.url).pathname;
|
|
888
|
+
}
|
|
889
|
+
catch {
|
|
890
|
+
return "";
|
|
891
|
+
}
|
|
892
|
+
})();
|
|
893
|
+
const name = capture.operationName ?? parsedOperationName(capture.query ?? "") ?? "anonymous";
|
|
894
|
+
return `${endpointPath}::${name}`;
|
|
895
|
+
}
|
|
896
|
+
/**
|
|
897
|
+
* Every 2xx capture in the run sharing `winningCapture`'s operation identity
|
|
898
|
+
* (its `operationGroupKey` for a GraphQL winner, `endpointKey` for a
|
|
899
|
+
* plain-REST one), deduplicated by response body. A single recon run can
|
|
900
|
+
* capture the same read operation multiple times (pagination, re-filtering),
|
|
901
|
+
* and each occurrence is evidence for `inferZodSchemaFromSamples` about
|
|
902
|
+
* which fields the operation actually returns every time versus only
|
|
903
|
+
* sometimes -- evidence the winning capture alone can't provide.
|
|
904
|
+
*/
|
|
905
|
+
function gatherResponseBodySamples(winningCapture, isGraphQLWinner, captures) {
|
|
906
|
+
if (!winningCapture)
|
|
907
|
+
return [null];
|
|
908
|
+
const identityOf = (c) => isGraphQLWinner ? operationGroupKey(c) : endpointKey(c.url);
|
|
909
|
+
const winningIdentity = identityOf(winningCapture);
|
|
910
|
+
const seen = new Set();
|
|
911
|
+
const samples = [];
|
|
912
|
+
for (const c of captures) {
|
|
913
|
+
if (c.status < 200 || c.status >= 300)
|
|
914
|
+
continue;
|
|
915
|
+
if (identityOf(c) !== winningIdentity)
|
|
916
|
+
continue;
|
|
917
|
+
const serialized = JSON.stringify(c.responseBody);
|
|
918
|
+
if (seen.has(serialized))
|
|
919
|
+
continue;
|
|
920
|
+
seen.add(serialized);
|
|
921
|
+
samples.push(c.responseBody);
|
|
922
|
+
}
|
|
923
|
+
return samples.length > 0 ? samples : [winningCapture.responseBody];
|
|
924
|
+
}
|
|
822
925
|
/**
|
|
823
926
|
* A declared variable is "populated" only by a non-null, non-empty-string
|
|
824
927
|
* value — an operation can declare a filter input that every capture leaves
|
|
@@ -850,8 +953,19 @@ function isPopulatedVariableValue(value) {
|
|
|
850
953
|
* Select answer (e.g. a device-type dropdown) never contributes a spurious
|
|
851
954
|
* facet match to the ranking.
|
|
852
955
|
*/
|
|
853
|
-
function selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, env = process.env) {
|
|
854
|
-
|
|
956
|
+
function selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, env = process.env, ownBackendHostnames = [], fallbackDomain = null) {
|
|
957
|
+
// Callers with no host-provenance data (the exported function's unit
|
|
958
|
+
// tests) pass neither ownBackendHostnames nor fallbackDomain — in that
|
|
959
|
+
// case isAllowedFixtureHost would reject every candidate, so the gate
|
|
960
|
+
// only applies once the caller has actually resolved a notion of "own
|
|
961
|
+
// backend" to check against.
|
|
962
|
+
const hasHostProvenance = ownBackendHostnames.length > 0 || fallbackDomain !== null;
|
|
963
|
+
const candidates = captures.filter((c) => c.status >= 200 &&
|
|
964
|
+
c.status < 300 &&
|
|
965
|
+
c.query !== null &&
|
|
966
|
+
!/^\s*mutation\b/.test(c.query) &&
|
|
967
|
+
(!hasHostProvenance ||
|
|
968
|
+
(0, capture_filters_1.isAllowedFixtureHost)(captureHostname(c.url), ownBackendHostnames, fallbackDomain)));
|
|
855
969
|
if (candidates.length === 0)
|
|
856
970
|
return null;
|
|
857
971
|
const knownFieldValues = buildKnownFieldValues(flowSteps, vocabulary, env);
|
|
@@ -892,22 +1006,24 @@ function selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, env = pr
|
|
|
892
1006
|
}
|
|
893
1007
|
return Object.values(c.variables).some((value) => spliceFacetsIntoStringVariable(value, payloadFieldList) !== null);
|
|
894
1008
|
};
|
|
1009
|
+
// A facet-spliceable candidate must win over any non-facet candidate
|
|
1010
|
+
// regardless of size/recurrence/phase — those signals only decide among
|
|
1011
|
+
// candidates of the same facet-spliceability tier, never across tiers.
|
|
1012
|
+
const facetCandidates = candidates.filter((c) => facetSpliceable(c));
|
|
1013
|
+
const scoringPool = facetCandidates.length > 0 ? facetCandidates : candidates;
|
|
895
1014
|
const operationNameCounts = new Map();
|
|
896
|
-
for (const c of
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
operationNameCounts.set(c.operationName, (operationNameCounts.get(c.operationName) ?? 0) + 1);
|
|
1015
|
+
for (const c of scoringPool) {
|
|
1016
|
+
const key = operationGroupKey(c);
|
|
1017
|
+
operationNameCounts.set(key, (operationNameCounts.get(key) ?? 0) + 1);
|
|
900
1018
|
}
|
|
901
|
-
const maxSize = Math.max(...
|
|
902
|
-
const maxFieldMatch = Math.max(...
|
|
1019
|
+
const maxSize = Math.max(...scoringPool.map(responseSize), 1);
|
|
1020
|
+
const maxFieldMatch = Math.max(...scoringPool.map(fieldMatchCount), 1);
|
|
903
1021
|
const maxRecurrence = Math.max(...Array.from(operationNameCounts.values()), 1);
|
|
904
|
-
const scored =
|
|
1022
|
+
const scored = scoringPool.map((capture) => {
|
|
905
1023
|
const sizeScore = responseSize(capture) / maxSize;
|
|
906
1024
|
const fieldScore = fieldMatchCount(capture) / maxFieldMatch;
|
|
907
1025
|
const phaseScore = capture.phase !== "home" ? 1 : 0;
|
|
908
|
-
const recurrenceScore = capture
|
|
909
|
-
? (operationNameCounts.get(capture.operationName) ?? 0) / maxRecurrence
|
|
910
|
-
: 0;
|
|
1026
|
+
const recurrenceScore = (operationNameCounts.get(operationGroupKey(capture)) ?? 0) / maxRecurrence;
|
|
911
1027
|
const facetScore = facetSpliceable(capture) ? 1 : 0;
|
|
912
1028
|
// Field correlation carries the heaviest weight so a smaller facet-matching
|
|
913
1029
|
// operation outranks a larger decoy — size alone must not decide this.
|
|
@@ -929,9 +1045,8 @@ function selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, env = pr
|
|
|
929
1045
|
return firstEndpointPath(candidates);
|
|
930
1046
|
}
|
|
931
1047
|
})();
|
|
932
|
-
const
|
|
933
|
-
|
|
934
|
-
: [winner.capture];
|
|
1048
|
+
const winnerGroupKey = operationGroupKey(winner.capture);
|
|
1049
|
+
const sharedCaptures = candidates.filter((c) => operationGroupKey(c) === winnerGroupKey);
|
|
935
1050
|
const declaredVariableNames = declaredOperationVariableNames(winner.capture.query ?? "");
|
|
936
1051
|
const unpopulatedDeclaredVariables = declaredVariableNames.filter((name) => !sharedCaptures.some((c) => isPopulatedVariableValue(c.variables?.[name])));
|
|
937
1052
|
return { capture: winner.capture, endpointPath, unpopulatedDeclaredVariables };
|
|
@@ -941,8 +1056,16 @@ function selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, env = pr
|
|
|
941
1056
|
* derives a path string from, so non-GraphQL flows can also read that
|
|
942
1057
|
* capture's own `.responseBody` instead of an array-order-first replay.
|
|
943
1058
|
*/
|
|
944
|
-
function firstEndpointCapture(captures) {
|
|
945
|
-
|
|
1059
|
+
function firstEndpointCapture(captures, ownBackendHostnames = [], fallbackDomain = null) {
|
|
1060
|
+
// Callers with no host-provenance data (unit tests exercising the
|
|
1061
|
+
// chronological-first fallback in isolation) pass neither argument -- in
|
|
1062
|
+
// that case isAllowedFixtureHost would reject every candidate, so the
|
|
1063
|
+
// gate only applies once the caller has actually resolved a notion of
|
|
1064
|
+
// "own backend" to check against. Mirrors selectPrimaryGraphQLOperation.
|
|
1065
|
+
const hasHostProvenance = ownBackendHostnames.length > 0 || fallbackDomain !== null;
|
|
1066
|
+
const allowed = (c) => !hasHostProvenance ||
|
|
1067
|
+
(0, capture_filters_1.isAllowedFixtureHost)(captureHostname(c.url), ownBackendHostnames, fallbackDomain);
|
|
1068
|
+
const nonGetCaptures = captures.filter((c) => c.method !== "GET" && allowed(c));
|
|
946
1069
|
for (const c of nonGetCaptures) {
|
|
947
1070
|
try {
|
|
948
1071
|
new URL(c.url);
|
|
@@ -952,7 +1075,7 @@ function firstEndpointCapture(captures) {
|
|
|
952
1075
|
// skip
|
|
953
1076
|
}
|
|
954
1077
|
}
|
|
955
|
-
for (const c of captures) {
|
|
1078
|
+
for (const c of captures.filter(allowed)) {
|
|
956
1079
|
try {
|
|
957
1080
|
new URL(c.url);
|
|
958
1081
|
return c;
|
|
@@ -963,9 +1086,17 @@ function firstEndpointCapture(captures) {
|
|
|
963
1086
|
}
|
|
964
1087
|
return null;
|
|
965
1088
|
}
|
|
966
|
-
function
|
|
1089
|
+
function safeUrlPathname(url) {
|
|
967
1090
|
try {
|
|
968
|
-
|
|
1091
|
+
return new URL(url).pathname;
|
|
1092
|
+
}
|
|
1093
|
+
catch {
|
|
1094
|
+
return "/api/search";
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
function firstEndpointPath(captures, ownBackendHostnames = [], fallbackDomain = null) {
|
|
1098
|
+
try {
|
|
1099
|
+
const capture = firstEndpointCapture(captures, ownBackendHostnames, fallbackDomain);
|
|
969
1100
|
return capture ? new URL(capture.url).pathname : "/api/search";
|
|
970
1101
|
}
|
|
971
1102
|
catch {
|
|
@@ -1079,8 +1210,6 @@ function extractGraphQLActionSequence(captures, submitPatterns = null) {
|
|
|
1079
1210
|
return captures
|
|
1080
1211
|
.map((capture, index) => ({ capture, index }))
|
|
1081
1212
|
.filter(({ capture }) => {
|
|
1082
|
-
if (capture.operationName === null)
|
|
1083
|
-
return false;
|
|
1084
1213
|
if (capture.status < 200 || capture.status >= 300)
|
|
1085
1214
|
return false;
|
|
1086
1215
|
if ((0, capture_filters_1.isNoiseUrl)(capture.url))
|
|
@@ -3398,7 +3527,7 @@ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap
|
|
|
3398
3527
|
// validates any individual call. Without this override, HttpRequestInit.schema
|
|
3399
3528
|
// would default to the client's z.unknown() and narrowing the caller-facing
|
|
3400
3529
|
// contract would enforce that narrowed shape on every call in the chain.
|
|
3401
|
-
const schemaExpr = inferZodSchema(cap.responseBody);
|
|
3530
|
+
const schemaExpr = inferZodSchema(cap.responseBody, 0, "", { looseServerResponse: true });
|
|
3402
3531
|
rendered.push({ url, method: cap.method, headersExpr, bodyArg, schemaExpr });
|
|
3403
3532
|
}
|
|
3404
3533
|
// Identifier scan against the rendered text — captures `${foo}`, `${foo.bar}`,
|
|
@@ -3879,7 +4008,7 @@ function buildContractChecklist(opts) {
|
|
|
3879
4008
|
].filter((line) => line !== "");
|
|
3880
4009
|
}
|
|
3881
4010
|
function emitContractTs(opts) {
|
|
3882
|
-
const { siteId, pascal, baseUrl, baseHeaders, minTime, safeRps, hasRateLimitProbeData = false, responseBody, gql, gqlQuery, endpointPath, gqlOperationName, gqlVariables, auxFiles, multiStepBody, omitExecuteHttp = false, isSubmissionFlow = false, inputBody, hasMultipartStep = false, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, discoveredStructuredKeys, payloadFieldNames, headerBindings = [], } = opts;
|
|
4011
|
+
const { siteId, pascal, baseUrl, baseHeaders, minTime, safeRps, hasRateLimitProbeData = false, responseBody, responseBodySamples = [responseBody], gql, gqlQuery, endpointPath, gqlOperationName, gqlVariables, auxFiles, multiStepBody, omitExecuteHttp = false, isSubmissionFlow = false, inputBody, hasMultipartStep = false, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, discoveredStructuredKeys, payloadFieldNames, headerBindings = [], } = opts;
|
|
3883
4012
|
// This is the CLIENT-level schema — createHttpClient's default, and the
|
|
3884
4013
|
// plugin's caller-facing contract (what executeHttp's return value promises
|
|
3885
4014
|
// its own caller). It does NOT validate any individual call in a multi-step
|
|
@@ -3908,7 +4037,10 @@ function emitContractTs(opts) {
|
|
|
3908
4037
|
? `z.object({ verified: z.boolean() })`
|
|
3909
4038
|
: omitExecuteHttp
|
|
3910
4039
|
? `z.unknown()`
|
|
3911
|
-
:
|
|
4040
|
+
: inferZodSchemaFromSamples(responseBodySamples, 0, "", {
|
|
4041
|
+
conditionalFieldNames,
|
|
4042
|
+
looseServerResponse: true,
|
|
4043
|
+
});
|
|
3912
4044
|
// Multi-step flows that include a multipart upload need the binary asset
|
|
3913
4045
|
// on the payload. ApplicantContactSchema (via ApplicantResumeSchema) already
|
|
3914
4046
|
// declares Resume/ResumeContentType/ResumeFilename, so submission flows
|
|
@@ -4150,7 +4282,7 @@ function emitContractTs(opts) {
|
|
|
4150
4282
|
? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";`
|
|
4151
4283
|
: `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
|
|
4152
4284
|
const queryConst = !omitExecuteHttp && gql && gqlQuery
|
|
4153
|
-
? `\n// Lifted verbatim from recon capture
|
|
4285
|
+
? `\n// Lifted verbatim from recon capture. The adjacent response schema is drift-tolerant by construction (dropped __typename, .loose() objects), so this query text is not hand-trimmed.\nconst ${pascal.toUpperCase()}_QUERY = \`${gqlQuery.trim()}\`;\n`
|
|
4154
4286
|
: "";
|
|
4155
4287
|
const gqlCacheBlock = omitExecuteHttp
|
|
4156
4288
|
? ""
|
|
@@ -4923,7 +5055,15 @@ async function main() {
|
|
|
4923
5055
|
// null the recovery functions no-op — the engine hardcodes no vendor format.
|
|
4924
5056
|
const formSchema = await resolveFormSchema(formSchemaSpecifier);
|
|
4925
5057
|
const pascal = toPascalCase(siteId);
|
|
4926
|
-
|
|
5058
|
+
// Read the flow's declared own-backend hosts BEFORE deriving baseUrl, so
|
|
5059
|
+
// deriveBaseUrl itself can gate on them: a third-party host that happens
|
|
5060
|
+
// to capture first (a feature-flag SDK, an analytics vendor) must never
|
|
5061
|
+
// become the generated hot path's base URL. baseUrl's own
|
|
5062
|
+
// registrable-domain fallback is necessarily derived FROM baseUrl, so it
|
|
5063
|
+
// isn't available to deriveBaseUrl yet -- see deriveBaseUrl's own doc
|
|
5064
|
+
// comment for how it copes when ownBackendHostnames is empty.
|
|
5065
|
+
const ownBackendHostnames = (0, recon_shared_1.readOwnBackendHostnames)(flowFile);
|
|
5066
|
+
const baseUrl = deriveBaseUrl(captures, ownBackendHostnames);
|
|
4927
5067
|
// Gate on the flow's declared own-backend hosts (or the registrable-domain
|
|
4928
5068
|
// fallback of baseUrl) via the same predicate recon-http.ts applies at
|
|
4929
5069
|
// write time, so a stale aux/ directory from before that filter existed
|
|
@@ -4931,7 +5071,6 @@ async function main() {
|
|
|
4931
5071
|
// third party's JSON into a generated plugin's fixtures/. A file with no
|
|
4932
5072
|
// manifest entry is unverifiable provenance, not proven safe, so it is
|
|
4933
5073
|
// excluded rather than assumed to have passed the write-time filter.
|
|
4934
|
-
const ownBackendHostnames = (0, recon_shared_1.readOwnBackendHostnames)(flowFile);
|
|
4935
5074
|
const fallbackDomain = baseUrl.length > 0 ? (0, capture_filters_1.registrableDomain)(new URL(baseUrl).hostname) : null;
|
|
4936
5075
|
const auxFiles = auxManifest
|
|
4937
5076
|
.filter((entry) => {
|
|
@@ -4967,21 +5106,42 @@ async function main() {
|
|
|
4967
5106
|
// (further down) read the same computed sequence instead of calling the
|
|
4968
5107
|
// extractor twice.
|
|
4969
5108
|
const graphqlActionSequence = gql ? extractGraphQLActionSequence(captures, submitPatterns) : [];
|
|
4970
|
-
const
|
|
4971
|
-
|
|
4972
|
-
? selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary)
|
|
5109
|
+
const primaryGraphQLOperation = gql && graphqlActionSequence.length === 0
|
|
5110
|
+
? selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, process.env, ownBackendHostnames, fallbackDomain)
|
|
4973
5111
|
: null;
|
|
4974
5112
|
if (primaryGraphQLOperation && primaryGraphQLOperation.unpopulatedDeclaredVariables.length > 0) {
|
|
4975
5113
|
const operationLabel = primaryGraphQLOperation.capture.operationName ?? "(anonymous)";
|
|
4976
5114
|
logger.warn(`WARN primary GraphQL operation '${operationLabel}' declares filter variable(s) ${primaryGraphQLOperation.unpopulatedDeclaredVariables.join(", ")} that are never populated in any capture — the generated executeHttp's payload field(s) for ${primaryGraphQLOperation.unpopulatedDeclaredVariables.join(", ")} have no wiring target and will replay the captured frozen value instead of the caller's input`);
|
|
4977
5115
|
}
|
|
4978
|
-
|
|
4979
|
-
|
|
5116
|
+
// When there's no primaryGraphQLOperation winner but the flow is still
|
|
5117
|
+
// GraphQL, the query/endpointPath/responseBody/operationName fallbacks
|
|
5118
|
+
// must all trace back to the SAME capture (firstGraphQLCapture) rather
|
|
5119
|
+
// than resolving independently — a non-GraphQL-shaped own-backend
|
|
5120
|
+
// capture could otherwise win the endpoint/body fallback while an
|
|
5121
|
+
// unrelated capture supplies the query text.
|
|
5122
|
+
const fallbackGraphQLCapture = gql
|
|
5123
|
+
? firstGraphQLCapture(captures, ownBackendHostnames, fallbackDomain)
|
|
5124
|
+
: null;
|
|
5125
|
+
const gqlQuery = primaryGraphQLOperation?.capture.query ?? fallbackGraphQLCapture?.query ?? null;
|
|
5126
|
+
const endpointPath = primaryGraphQLOperation?.endpointPath ??
|
|
5127
|
+
(fallbackGraphQLCapture
|
|
5128
|
+
? safeUrlPathname(fallbackGraphQLCapture.url)
|
|
5129
|
+
: firstEndpointPath(captures, ownBackendHostnames, fallbackDomain));
|
|
4980
5130
|
// Derived from the primary operation's own Phase-1 capture, never from
|
|
4981
5131
|
// replay array order -- a replay's body reflects whichever endpoint fired
|
|
4982
5132
|
// first, not necessarily the primary operation, and only exists once
|
|
4983
5133
|
// recon:http has run.
|
|
4984
|
-
const
|
|
5134
|
+
const winningCapture = primaryGraphQLOperation?.capture ??
|
|
5135
|
+
fallbackGraphQLCapture ??
|
|
5136
|
+
firstEndpointCapture(captures, ownBackendHostnames, fallbackDomain);
|
|
5137
|
+
const responseBody = winningCapture?.responseBody ?? null;
|
|
5138
|
+
// Every 2xx capture sharing the winning capture's operation identity, not
|
|
5139
|
+
// just the one that happened to win selection -- a paginated/re-filtered
|
|
5140
|
+
// re-fire of the same operation can omit a field the winning capture
|
|
5141
|
+
// happened to have (or vice versa), and inferZodSchemaFromSamples needs
|
|
5142
|
+
// that presence evidence across occurrences to mark the field .optional()
|
|
5143
|
+
// instead of required from a single observation.
|
|
5144
|
+
const responseBodySamples = gatherResponseBodySamples(winningCapture, primaryGraphQLOperation !== null || fallbackGraphQLCapture !== null, captures);
|
|
4985
5145
|
// Detect a multi-step submission flow (transactional sites like apply forms,
|
|
4986
5146
|
// checkout, etc.). When the action sequence has 2+ POSTs, switch the
|
|
4987
5147
|
// contract template to emit a state-threaded executeHttp.
|
|
@@ -5257,10 +5417,19 @@ async function main() {
|
|
|
5257
5417
|
safeRps,
|
|
5258
5418
|
hasRateLimitProbeData,
|
|
5259
5419
|
responseBody: effectiveResponseBody,
|
|
5420
|
+
// selectEffectiveResponseBody already resolves a submission flow's own
|
|
5421
|
+
// single terminal capture -- the multi-sample evidence gathered above
|
|
5422
|
+
// describes the (possibly different) read-flow winning capture and
|
|
5423
|
+
// doesn't apply once a submission flow has picked its own return call.
|
|
5424
|
+
responseBodySamples: isSubmissionFlow ? [effectiveResponseBody] : responseBodySamples,
|
|
5260
5425
|
gql,
|
|
5261
5426
|
gqlQuery,
|
|
5262
5427
|
endpointPath,
|
|
5263
|
-
gqlOperationName: primaryGraphQLOperation
|
|
5428
|
+
gqlOperationName: primaryGraphQLOperation
|
|
5429
|
+
? (primaryGraphQLOperation.capture.operationName ??
|
|
5430
|
+
parsedOperationName(primaryGraphQLOperation.capture.query ?? ""))
|
|
5431
|
+
: (fallbackGraphQLCapture?.operationName ??
|
|
5432
|
+
parsedOperationName(fallbackGraphQLCapture?.query ?? "")),
|
|
5264
5433
|
gqlVariables: primaryGraphQLOperation?.capture.variables ?? null,
|
|
5265
5434
|
auxFiles,
|
|
5266
5435
|
multiStepBody,
|