@enricai/barnacle 1.12.69 → 1.12.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scraper/browser-click-expr.d.ts +3 -4
- package/dist/scraper/browser-click-expr.d.ts.map +1 -1
- package/dist/scraper/browser-click-expr.js +3 -4
- package/dist/scraper/browser-click-expr.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts +27 -6
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +332 -61
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scraper/frame-target.d.ts +12 -0
- package/dist/scraper/frame-target.d.ts.map +1 -1
- package/dist/scraper/frame-target.js +1 -0
- package/dist/scraper/frame-target.js.map +1 -1
- package/dist/scraper/submit-control.d.ts +30 -2
- package/dist/scraper/submit-control.d.ts.map +1 -1
- package/dist/scraper/submit-control.js +79 -20
- package/dist/scraper/submit-control.js.map +1 -1
- package/dist/scraper/target-resolution-diagnostic.d.ts +53 -0
- package/dist/scraper/target-resolution-diagnostic.d.ts.map +1 -0
- package/dist/scraper/target-resolution-diagnostic.js +153 -0
- package/dist/scraper/target-resolution-diagnostic.js.map +1 -0
- package/dist/scripts/recon-browser.d.ts +32 -2
- package/dist/scripts/recon-browser.d.ts.map +1 -1
- package/dist/scripts/recon-browser.js +3 -1
- package/dist/scripts/recon-browser.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +6 -1
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +191 -70
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +1 -1
- package/dist/scraper/deep-query.d.ts +0 -44
- package/dist/scraper/deep-query.d.ts.map +0 -1
- package/dist/scraper/deep-query.js +0 -93
- package/dist/scraper/deep-query.js.map +0 -1
|
@@ -1142,8 +1142,16 @@ function deriveRequestHeaders(captures, replays, baseUrl, submitPatterns = null,
|
|
|
1142
1142
|
}
|
|
1143
1143
|
return baseline;
|
|
1144
1144
|
}
|
|
1145
|
-
|
|
1146
|
-
|
|
1145
|
+
/**
|
|
1146
|
+
* Classifies the flow as GraphQL only from own-backend-host traffic, so a
|
|
1147
|
+
* third-party host (chat widget, analytics SDK) making real GraphQL calls
|
|
1148
|
+
* never flips a plain REST/JSON backend's classification.
|
|
1149
|
+
*/
|
|
1150
|
+
function isGraphQL(captures, ownBackendHostnames = [], fallbackDomain = null) {
|
|
1151
|
+
const hasHostProvenance = ownBackendHostnames.length > 0 || fallbackDomain !== null;
|
|
1152
|
+
return captures.some((c) => (!hasHostProvenance ||
|
|
1153
|
+
(0, capture_filters_1.isAllowedFixtureHost)(captureHostname(c.url), ownBackendHostnames, fallbackDomain)) &&
|
|
1154
|
+
(c.operationName !== null || (0, capture_filters_1.parsedOperationName)(c.query ?? "") !== null));
|
|
1147
1155
|
}
|
|
1148
1156
|
/**
|
|
1149
1157
|
* The single own-backend capture the raw GraphQL fallback (no
|
|
@@ -1225,9 +1233,12 @@ function operationGroupKey(capture) {
|
|
|
1225
1233
|
* never independently resolve a different primary.
|
|
1226
1234
|
*/
|
|
1227
1235
|
function computeEmittedPrimaryAnchor(gql, endpointPath, gqlOperationName, gqlQuery) {
|
|
1228
|
-
|
|
1236
|
+
// Mirrors emitContractTs's own isGqlEmission gate: `gql` alone can be true
|
|
1237
|
+
// with a null gqlQuery (see gqlQuery's `?? null` fallback), in which case
|
|
1238
|
+
// emitContractTs falls through to REST emission — this anchor must agree.
|
|
1239
|
+
if (!gql || gqlQuery === null)
|
|
1229
1240
|
return null;
|
|
1230
|
-
const name = gqlOperationName ?? (0, capture_filters_1.parsedOperationName)(gqlQuery
|
|
1241
|
+
const name = gqlOperationName ?? (0, capture_filters_1.parsedOperationName)(gqlQuery) ?? "anonymous";
|
|
1231
1242
|
return `${endpointPath}::${name}`;
|
|
1232
1243
|
}
|
|
1233
1244
|
/**
|
|
@@ -1584,8 +1595,26 @@ function resolveManifestActionSequence(runRoot, captures) {
|
|
|
1584
1595
|
* Exported for tests: this predicate decides what a generated plugin will POST
|
|
1585
1596
|
* at a live site, and it is the only gate between a browser's incidental
|
|
1586
1597
|
* chatter and the emitted hot path.
|
|
1587
|
-
|
|
1588
|
-
|
|
1598
|
+
*
|
|
1599
|
+
* `applyFinalAnchorNarrowing` lets a caller that will itself truncate the
|
|
1600
|
+
* result via {@link truncateActionSequenceAtSubmitPattern} opt out of the
|
|
1601
|
+
* final structural-relevance anchor pass, which can drop a genuine earlier
|
|
1602
|
+
* chain step that shares no token with the terminal submit endpoint.
|
|
1603
|
+
*/
|
|
1604
|
+
function extractActionSequence(captures, submitPatterns = null, foldReturnSpec = null, ownBackendHostnames = [], fallbackDomain = null,
|
|
1605
|
+
// The final structural-relevance anchor pass below (guarded on
|
|
1606
|
+
// hasSubmitEndpointAnchor) is tuned for isolating a submission from
|
|
1607
|
+
// same-URL incidental chrome, not for pruning a genuine multi-step chain:
|
|
1608
|
+
// an earlier plain single-word step (e.g. `/lodging/hold/`) that shares no
|
|
1609
|
+
// compound token with the terminal submit endpoint reads as "unrelated" to
|
|
1610
|
+
// it and gets dropped, even though it's a real prior step whose body the
|
|
1611
|
+
// caller still needs. truncateActionSequenceAtSubmitPattern's own
|
|
1612
|
+
// index-slice already does the right, less-aggressive narrowing (keep
|
|
1613
|
+
// everything up to and including the last submit/fold match) — so a caller
|
|
1614
|
+
// that immediately truncates this result opts out with `false` to keep
|
|
1615
|
+
// earlier chain steps intact while still getting the structural-isolation
|
|
1616
|
+
// exemption above.
|
|
1617
|
+
applyFinalAnchorNarrowing = true) {
|
|
1589
1618
|
const matchesFoldReturn = compileFoldReturnEndpointMatcher(foldReturnSpec);
|
|
1590
1619
|
// Callers with no host-provenance data (the exported function's unit
|
|
1591
1620
|
// tests) pass neither ownBackendHostnames nor fallbackDomain — in that
|
|
@@ -1607,6 +1636,14 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
|
|
|
1607
1636
|
// the submission from same-URL chrome" behavior).
|
|
1608
1637
|
const hasSubmitEndpointAnchor = hasHostProvenance && submitPatterns !== null && submitPatterns.endpoint !== null;
|
|
1609
1638
|
const matchesSubmit = compileSubmitMatcher(hasSubmitEndpointAnchor ? { endpoint: null, body: submitPatterns.body } : submitPatterns);
|
|
1639
|
+
// Compiled once, ahead of structural isolation below, so a capture matching
|
|
1640
|
+
// the flow's own declared submitEndpointPattern can be exempted there the
|
|
1641
|
+
// same way a declared foldReturn endpointPattern already is — both are
|
|
1642
|
+
// author-declared, both are authoritative by construction, so the isolation
|
|
1643
|
+
// pass (which exists to catch INCIDENTAL same-host noise the flow never
|
|
1644
|
+
// named) must not treat one differently from the other.
|
|
1645
|
+
const declaredSubmitEndpointRx = submitPatterns?.endpoint != null ? new RegExp(submitPatterns.endpoint) : null;
|
|
1646
|
+
const matchesDeclaredSubmitEndpoint = (capture) => declaredSubmitEndpointRx?.test(capture.url) ?? false;
|
|
1610
1647
|
const hostGated = captures
|
|
1611
1648
|
.map((capture, index) => ({ capture, index }))
|
|
1612
1649
|
.filter(({ capture }) => {
|
|
@@ -1630,9 +1667,14 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
|
|
|
1630
1667
|
// (e.g. a marketing/promotions endpoint fired incidentally by a page load,
|
|
1631
1668
|
// admitted above because it too is a 2xx own-backend POST). Drop a
|
|
1632
1669
|
// structurally-isolated member of the pool first — a compound-path capture
|
|
1633
|
-
// (e.g. `/site-banner`) sharing no token with anything else host-gated in
|
|
1634
|
-
//
|
|
1635
|
-
//
|
|
1670
|
+
// (e.g. `/site-banner`) sharing no token with anything else host-gated in
|
|
1671
|
+
// — unless it matches the flow's own declared submitEndpointPattern (see
|
|
1672
|
+
// the exemption below): a densely-tokenized real submission path (e.g.
|
|
1673
|
+
// `available-sailings`) can look exactly as isolated as incidental noise
|
|
1674
|
+
// by path shape alone, and dropping it here would strip it out before the
|
|
1675
|
+
// structural-relevance anchor pass further down ever gets to treat it as
|
|
1676
|
+
// authoritative. Guarded to pools of 3+: a real chain's own steps are
|
|
1677
|
+
// usually plain single-word
|
|
1636
1678
|
// paths with no tokens of their own (never flagged, see
|
|
1637
1679
|
// isStructurallyIsolatedCapture), so this only ever removes a capture that
|
|
1638
1680
|
// is BOTH compound-path AND unrelated to every other admitted capture —
|
|
@@ -1663,11 +1705,14 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
|
|
|
1663
1705
|
.filter((h, j) => denselyNameSpaced ? safeUrlPathname(h.capture.url) !== path : j !== i)
|
|
1664
1706
|
.map((h) => safeUrlPathname(h.capture.url));
|
|
1665
1707
|
// A capture admitted via the flow's own declared foldReturnSpec
|
|
1666
|
-
// endpointPattern (the drill-down target)
|
|
1667
|
-
//
|
|
1708
|
+
// endpointPattern (the drill-down target), or matching the flow's
|
|
1709
|
+
// own declared submitEndpointPattern, is authoritative by
|
|
1710
|
+
// construction — the flow author named it explicitly — so both are
|
|
1668
1711
|
// exempt from token-overlap isolation, which exists to catch
|
|
1669
1712
|
// INCIDENTAL same-host captures the flow never declared.
|
|
1670
|
-
return matchesFoldReturn(capture) ||
|
|
1713
|
+
return (matchesFoldReturn(capture) ||
|
|
1714
|
+
matchesDeclaredSubmitEndpoint(capture) ||
|
|
1715
|
+
!(0, capture_filters_1.isStructurallyIsolatedCapture)(path, otherPaths));
|
|
1671
1716
|
})
|
|
1672
1717
|
: hostGated;
|
|
1673
1718
|
// Narrow further using structural relevance, anchored on whichever
|
|
@@ -1677,15 +1722,14 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
|
|
|
1677
1722
|
// authoritative reference, so this pass is a no-op — mirroring
|
|
1678
1723
|
// compileSubmitMatcher's own null-pattern passthrough — rather than
|
|
1679
1724
|
// guessing at a reference set.
|
|
1680
|
-
if (!hasSubmitEndpointAnchor)
|
|
1725
|
+
if (!hasSubmitEndpointAnchor || !applyFinalAnchorNarrowing)
|
|
1681
1726
|
return structurallyGated;
|
|
1682
|
-
const endpointRx = new RegExp(submitPatterns.endpoint);
|
|
1683
1727
|
const referencePaths = structurallyGated
|
|
1684
|
-
.filter(({ capture }) =>
|
|
1728
|
+
.filter(({ capture }) => matchesDeclaredSubmitEndpoint(capture))
|
|
1685
1729
|
.map(({ capture }) => safeUrlPathname(capture.url));
|
|
1686
1730
|
if (referencePaths.length === 0)
|
|
1687
1731
|
return structurallyGated;
|
|
1688
|
-
return structurallyGated.filter(({ capture }) =>
|
|
1732
|
+
return structurallyGated.filter(({ capture }) => matchesDeclaredSubmitEndpoint(capture) ||
|
|
1689
1733
|
(0, capture_filters_1.isStructurallyRelevantCapture)(safeUrlPathname(capture.url), referencePaths));
|
|
1690
1734
|
}
|
|
1691
1735
|
/**
|
|
@@ -8602,6 +8646,65 @@ function compileFoldReturnResultsMatcher(spec) {
|
|
|
8602
8646
|
const rawResultsPath = spec.resultsPath.split(".");
|
|
8603
8647
|
return (capture) => objectItemsAtPath(capture.responseBody, resolveDeclaredArrayPath(rawResultsPath, capture.responseBody)) !== null;
|
|
8604
8648
|
}
|
|
8649
|
+
/** Response-side counterpart of {@link buildRequestValueIndex}, restricted to
|
|
8650
|
+
* the endpoint-matching drill candidates: maps each response leaf value to the
|
|
8651
|
+
* ascending drill indices carrying it, so the response-only fallback can find
|
|
8652
|
+
* the drills a primary's join values could resolve against without testing
|
|
8653
|
+
* every primary/drill pair. Built eagerly per call, like
|
|
8654
|
+
* {@link buildRequestValueIndex}; the per-capture leaf sets it reads are
|
|
8655
|
+
* memoized, so repeated calls only re-pay the indexing. */
|
|
8656
|
+
function buildDrillResponseValueIndex(actions, drillStepIndices) {
|
|
8657
|
+
const index = new Map();
|
|
8658
|
+
for (const drillStepIndex of drillStepIndices) {
|
|
8659
|
+
for (const value of collectResponseLeafValues(actions[drillStepIndex].capture)) {
|
|
8660
|
+
const indices = index.get(value);
|
|
8661
|
+
if (indices)
|
|
8662
|
+
indices.push(drillStepIndex);
|
|
8663
|
+
else
|
|
8664
|
+
index.set(value, [drillStepIndex]);
|
|
8665
|
+
}
|
|
8666
|
+
}
|
|
8667
|
+
return index;
|
|
8668
|
+
}
|
|
8669
|
+
/** Descending drill indices after a primary that can possibly resolve: a drill
|
|
8670
|
+
* resolves only via a request-carried join value (some candidate entry index at
|
|
8671
|
+
* or before it) or a response-carried one, so every other drill is skipped
|
|
8672
|
+
* without changing which one wins. `unpruned` (no joinFields, where any drill
|
|
8673
|
+
* trivially matches) keeps the full freshest-first scan. */
|
|
8674
|
+
function collectDrillStepIndicesWorthTrying(allDrillStepIndices, firstDrillOffset, drillResponseValueIndex, joinValues, candidateEntryIndicesDescending, unpruned) {
|
|
8675
|
+
if (unpruned)
|
|
8676
|
+
return allDrillStepIndices.slice(firstDrillOffset).reverse();
|
|
8677
|
+
const worthTrying = new Set();
|
|
8678
|
+
const earliestEntryIndex = candidateEntryIndicesDescending.at(-1);
|
|
8679
|
+
if (earliestEntryIndex !== undefined) {
|
|
8680
|
+
const requestStart = Math.max(firstDrillOffset, firstIndexGreaterThan(allDrillStepIndices, earliestEntryIndex - 1));
|
|
8681
|
+
for (let i = requestStart; i < allDrillStepIndices.length; i++) {
|
|
8682
|
+
worthTrying.add(allDrillStepIndices[i]);
|
|
8683
|
+
}
|
|
8684
|
+
}
|
|
8685
|
+
const firstEligibleDrillStepIndex = allDrillStepIndices[firstDrillOffset];
|
|
8686
|
+
for (const value of joinValues) {
|
|
8687
|
+
for (const drillStepIndex of drillResponseValueIndex.get(value) ?? []) {
|
|
8688
|
+
if (firstEligibleDrillStepIndex !== undefined &&
|
|
8689
|
+
drillStepIndex >= firstEligibleDrillStepIndex) {
|
|
8690
|
+
worthTrying.add(drillStepIndex);
|
|
8691
|
+
}
|
|
8692
|
+
}
|
|
8693
|
+
}
|
|
8694
|
+
return [...worthTrying].sort((a, b) => b - a);
|
|
8695
|
+
}
|
|
8696
|
+
function firstIndexGreaterThan(sortedAscending, value) {
|
|
8697
|
+
let low = 0;
|
|
8698
|
+
let high = sortedAscending.length;
|
|
8699
|
+
while (low < high) {
|
|
8700
|
+
const mid = (low + high) >>> 1;
|
|
8701
|
+
if (sortedAscending[mid] <= value)
|
|
8702
|
+
low = mid + 1;
|
|
8703
|
+
else
|
|
8704
|
+
high = mid;
|
|
8705
|
+
}
|
|
8706
|
+
return low;
|
|
8707
|
+
}
|
|
8605
8708
|
function buildFoldPlanFromSpec(actions, spec,
|
|
8606
8709
|
// See detectDrillDownFoldPlan's identical parameter — constrains this
|
|
8607
8710
|
// function's own primaryStepIndex loop to the same anchor identity.
|
|
@@ -8633,6 +8736,14 @@ restrictToDrillEndpointKey = null) {
|
|
|
8633
8736
|
// paying for anything else, instead of scanning its own drill candidates
|
|
8634
8737
|
// one by one only to discover none of them can ever match.
|
|
8635
8738
|
const requestValueIndex = buildRequestValueIndex(actions);
|
|
8739
|
+
// Endpoint matching never depends on the primary, so it is evaluated once
|
|
8740
|
+
// here; testing every later action per primary made the scan quadratic.
|
|
8741
|
+
const allMatchingDrillStepIndices = actions.flatMap((action, index) => matchesFoldReturnEndpoint(action.capture) &&
|
|
8742
|
+
(restrictToDrillEndpointKey === null ||
|
|
8743
|
+
endpointKey(action.capture.url) === restrictToDrillEndpointKey)
|
|
8744
|
+
? [index]
|
|
8745
|
+
: []);
|
|
8746
|
+
const drillResponseValueIndex = buildDrillResponseValueIndex(actions, allMatchingDrillStepIndices);
|
|
8636
8747
|
let freshestPlan = null;
|
|
8637
8748
|
for (let primaryStepIndex = 0; primaryStepIndex < actions.length; primaryStepIndex++) {
|
|
8638
8749
|
if (exactPrimaryStepIndex !== null && primaryStepIndex !== exactPrimaryStepIndex) {
|
|
@@ -8648,42 +8759,32 @@ restrictToDrillEndpointKey = null) {
|
|
|
8648
8759
|
continue;
|
|
8649
8760
|
const joinValues = collectPrimaryJoinValues(primaryItems, spec.joinFields);
|
|
8650
8761
|
const candidateEntryIndicesDescending = collectCandidateEntryIndicesDescending(requestValueIndex, joinValues, primaryStepIndex);
|
|
8651
|
-
//
|
|
8652
|
-
//
|
|
8653
|
-
//
|
|
8654
|
-
//
|
|
8655
|
-
//
|
|
8656
|
-
// ties in the original last-write-wins scan first, falling through to
|
|
8657
|
-
// the next-freshest only when a candidate fails to resolve, instead of
|
|
8658
|
-
// resolving every earlier occurrence just to have it overwritten.
|
|
8762
|
+
// Drills are tried from the freshest (highest) index down so the
|
|
8763
|
+
// expensive backward-walk + computeFoldChain resolution runs on the
|
|
8764
|
+
// candidate that would win ties in the original last-write-wins scan
|
|
8765
|
+
// first, falling through to the next-freshest only when a candidate
|
|
8766
|
+
// fails to resolve.
|
|
8659
8767
|
//
|
|
8660
|
-
//
|
|
8661
|
-
//
|
|
8662
|
-
//
|
|
8663
|
-
//
|
|
8664
|
-
//
|
|
8665
|
-
//
|
|
8666
|
-
// the
|
|
8667
|
-
|
|
8668
|
-
|
|
8669
|
-
|
|
8670
|
-
if (matchesFoldReturnEndpoint(actions[drillStepIndex].capture) &&
|
|
8671
|
-
(restrictToDrillEndpointKey === null ||
|
|
8672
|
-
endpointKey(actions[drillStepIndex].capture.url) === restrictToDrillEndpointKey)) {
|
|
8673
|
-
matchingDrillStepIndices.push(drillStepIndex);
|
|
8674
|
-
}
|
|
8675
|
-
}
|
|
8676
|
-
if (candidateEntryIndicesDescending.length === 0 && matchingDrillStepIndices.length === 0) {
|
|
8768
|
+
// A primary with no request-carried join value must still reach
|
|
8769
|
+
// {@link resolveSpecMatchedPrimaryItemIndexFromResponse}'s response-only
|
|
8770
|
+
// fallback, so an empty `candidateEntryIndicesDescending` alone never
|
|
8771
|
+
// skips it; the drills tried are instead pruned to those whose response
|
|
8772
|
+
// carries one of this primary's join values (see
|
|
8773
|
+
// {@link collectDrillStepIndicesWorthTrying}), because testing every
|
|
8774
|
+
// primary/drill pair is quadratic in the capture count.
|
|
8775
|
+
const firstDrillOffset = firstIndexGreaterThan(allMatchingDrillStepIndices, primaryStepIndex);
|
|
8776
|
+
if (candidateEntryIndicesDescending.length === 0 &&
|
|
8777
|
+
firstDrillOffset === allMatchingDrillStepIndices.length) {
|
|
8677
8778
|
continue;
|
|
8678
8779
|
}
|
|
8780
|
+
const drillStepIndicesToTry = collectDrillStepIndicesWorthTrying(allMatchingDrillStepIndices, firstDrillOffset, drillResponseValueIndex, joinValues, candidateEntryIndicesDescending, spec.joinFields.length === 0);
|
|
8679
8781
|
// Shared across every matching-drill candidate tried below for THIS
|
|
8680
8782
|
// primaryStepIndex — see resolveSpecMatchedPrimaryItemIndexAlongChain's
|
|
8681
8783
|
// docstring on why a fresh cache per primary (not per drill candidate)
|
|
8682
8784
|
// is what collapses the backward-walk from O(candidates * chain length)
|
|
8683
8785
|
// to O(chain length).
|
|
8684
8786
|
const matchedItemIndexCache = new Map();
|
|
8685
|
-
for (
|
|
8686
|
-
const drillStepIndex = matchingDrillStepIndices[i];
|
|
8787
|
+
for (const drillStepIndex of drillStepIndicesToTry) {
|
|
8687
8788
|
const drill = actions[drillStepIndex];
|
|
8688
8789
|
// Widened to a flat (non-array) object response the same way the
|
|
8689
8790
|
// structural heuristic is (see findAllObjectArrayFieldsOrWholeObject):
|
|
@@ -9005,31 +9106,30 @@ primaryIdentityAnchor = null) {
|
|
|
9005
9106
|
specPlan,
|
|
9006
9107
|
];
|
|
9007
9108
|
}
|
|
9008
|
-
// Keyed by the (primaryStepIndex, primaryArrayPath) pair, not
|
|
9009
|
-
//
|
|
9010
|
-
//
|
|
9011
|
-
//
|
|
9012
|
-
//
|
|
9013
|
-
//
|
|
9014
|
-
//
|
|
9015
|
-
//
|
|
9016
|
-
//
|
|
9017
|
-
//
|
|
9018
|
-
//
|
|
9019
|
-
//
|
|
9020
|
-
//
|
|
9021
|
-
|
|
9109
|
+
// Keyed by the (primaryStepIndex, primaryArrayPath) pair, not raw step
|
|
9110
|
+
// indices: that pair is the only thing that identifies whether the spec
|
|
9111
|
+
// describes a primary a structural plan ALREADY owns. A structural
|
|
9112
|
+
// target's own `drillStepIndex` (or any index in its replay chain)
|
|
9113
|
+
// coinciding with a step the spec's own resolution happens to touch is
|
|
9114
|
+
// not, on its own, evidence of a real conflict — an unrelated,
|
|
9115
|
+
// independent primary/drill pair can legitimately enter through (or
|
|
9116
|
+
// replay) the exact same underlying capture the spec's chain does (e.g.
|
|
9117
|
+
// a shared session/auth bootstrap call reused by two unrelated
|
|
9118
|
+
// features), and both targets replaying it per their own items is
|
|
9119
|
+
// exactly what {@link resolveSpecMatchedPrimaryItemIndexAlongChain}'s
|
|
9120
|
+
// multi-hop resolution expects — see the shared-bootstrap-hop-noise
|
|
9121
|
+
// regression test. Excluding on raw index overlap (as an earlier version
|
|
9122
|
+
// of this check did, for BOTH the spec's `drillStepIndex` and its whole
|
|
9123
|
+
// `chain`) silently dropped a real, resolvable declared join whenever
|
|
9124
|
+
// some unrelated noisy structural plan merely touched the same index by
|
|
9125
|
+
// coincidence, which is exactly the "declared fold rejected" failure
|
|
9126
|
+
// mode this function exists to prevent.
|
|
9022
9127
|
const consumedPrimarySteps = new Set();
|
|
9023
9128
|
for (const plan of structuralPlansWithPerOccurrenceOverrides) {
|
|
9024
9129
|
const planPrimaryEndpointKey = endpointKey(actions[plan.primaryStepIndex].capture.url);
|
|
9025
9130
|
consumedPrimarySteps.add(`${planPrimaryEndpointKey}:${JSON.stringify(plan.primaryArrayPath)}`);
|
|
9026
|
-
for (const target of plan.targets) {
|
|
9027
|
-
consumedIndices.add(target.drillStepIndex);
|
|
9028
|
-
}
|
|
9029
9131
|
}
|
|
9030
|
-
const specConsumesOnlyItsOwnIndices = !
|
|
9031
|
-
!consumedPrimarySteps.has(`${specPrimaryEndpointKey}:${JSON.stringify(specPlan.primaryArrayPath)}`) &&
|
|
9032
|
-
specPlan.targets.every((target) => target.chain.every((chainIndex) => !consumedIndices.has(chainIndex)));
|
|
9132
|
+
const specConsumesOnlyItsOwnIndices = !consumedPrimarySteps.has(`${specPrimaryEndpointKey}:${JSON.stringify(specPlan.primaryArrayPath)}`);
|
|
9033
9133
|
if (specConsumesOnlyItsOwnIndices)
|
|
9034
9134
|
return [...structuralPlansWithPerOccurrenceOverrides, specPlan];
|
|
9035
9135
|
// The spec's primary endpoint/array is already consumed by a structural
|
|
@@ -9677,6 +9777,11 @@ function buildContractChecklist(opts) {
|
|
|
9677
9777
|
function emitContractTs(opts) {
|
|
9678
9778
|
const { siteId, displayName, pascal, baseUrl, baseHeaders, minTime, safeRps, hasRateLimitProbeData = false, responseBody, responseBodySamples = [responseBody], gql, gqlQuery, endpointPath, gqlOperationName, gqlVariables, allCaptures = [], auxFiles, multiStepBody, omitExecuteHttp = false, isSubmissionFlow = false, inputBody, hasMultipartStep = false, actionSteps = [], foldReturnSpec = null, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, discoveredStructuredKeys, valueConstraints = value_constraints_1.EMPTY_VALUE_CONSTRAINTS, payloadFieldNames, optionalPayloadFieldNames = new Set(), headerBindings = [], unpopulatedDeclaredVariables = [], fallbackGateSpec = {}, } = opts;
|
|
9679
9779
|
const { browserFallbackGate, httpTimeoutMs } = fallbackGateSpec;
|
|
9780
|
+
/** Single source of truth for every GraphQL-emission decision below
|
|
9781
|
+
* (clientImport, gqlCacheBlock, queryConst, and both executeHttpBody
|
|
9782
|
+
* fetch-call sites) — `gql` alone can be true with a null `gqlQuery`,
|
|
9783
|
+
* which would emit a `${PASCAL}_QUERY` reference with no declaration. */
|
|
9784
|
+
const isGqlEmission = gql && gqlQuery !== null;
|
|
9680
9785
|
/** Rendered `meta.browserFallbackGate` literal: `false` verbatim, a
|
|
9681
9786
|
* list-derived arrow-function predicate matching against `error.name`, or
|
|
9682
9787
|
* "" (omitted) when the flow declared no gate — preserving byte-identical
|
|
@@ -10134,15 +10239,15 @@ function emitContractTs(opts) {
|
|
|
10134
10239
|
// `noUnusedVariables`.
|
|
10135
10240
|
const clientImport = omitExecuteHttp
|
|
10136
10241
|
? ""
|
|
10137
|
-
:
|
|
10242
|
+
: isGqlEmission
|
|
10138
10243
|
? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";${needsFoldHttpClient ? `\nimport { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";` : ""}`
|
|
10139
10244
|
: `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
|
|
10140
|
-
const queryConst = !omitExecuteHttp &&
|
|
10245
|
+
const queryConst = !omitExecuteHttp && isGqlEmission
|
|
10141
10246
|
? `\n// Lifted verbatim from recon capture. The adjacent response schema is drift-tolerant by construction (dropped __typename, .loose() objects), so this query text is not hand-trimmed.\nconst ${pascal.toUpperCase()}_QUERY = \`${gqlQuery.trim()}\`;\n`
|
|
10142
10247
|
: "";
|
|
10143
10248
|
const gqlCacheBlock = omitExecuteHttp
|
|
10144
10249
|
? ""
|
|
10145
|
-
:
|
|
10250
|
+
: isGqlEmission
|
|
10146
10251
|
? `
|
|
10147
10252
|
type GqlFn = (operationName: string, query: string, variables: Record<string, unknown>) => Promise<${pascal}Response>;
|
|
10148
10253
|
|
|
@@ -10491,7 +10596,7 @@ const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottlenec
|
|
|
10491
10596
|
signal: paginationSignal,
|
|
10492
10597
|
foldMergeLines: paginatedFoldMergeLines,
|
|
10493
10598
|
valueConstraints,
|
|
10494
|
-
fetchCall:
|
|
10599
|
+
fetchCall: isGqlEmission
|
|
10495
10600
|
? {
|
|
10496
10601
|
kind: "gql",
|
|
10497
10602
|
gqlOperationNameExpr,
|
|
@@ -10499,7 +10604,7 @@ const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottlenec
|
|
|
10499
10604
|
}
|
|
10500
10605
|
: { kind: "rest", endpointPath },
|
|
10501
10606
|
})
|
|
10502
|
-
:
|
|
10607
|
+
: isGqlEmission
|
|
10503
10608
|
? ` const data = await getGql(context.baseUrl)(${gqlOperationNameExpr}, ${pascal.toUpperCase()}_QUERY, ${gqlVariablesExpr});
|
|
10504
10609
|
${dataFoldMergeBlock} return { data };`
|
|
10505
10610
|
: ` const data = await httpClient(\`\${context.baseUrl}${endpointPath}\`, {
|
|
@@ -11591,7 +11696,7 @@ async function main() {
|
|
|
11591
11696
|
const minTime = deriveMinTime(rateLimits);
|
|
11592
11697
|
const hasRateLimitProbeData = rateLimits.some((f) => f.safeRps !== null);
|
|
11593
11698
|
const safeRps = rateLimits.find((f) => f.safeRps !== null)?.safeRps ?? Math.floor(1000 / minTime);
|
|
11594
|
-
const gql = isGraphQL(activeCaptures);
|
|
11699
|
+
const gql = isGraphQL(activeCaptures, ownBackendHostnames, fallbackDomain);
|
|
11595
11700
|
// Hoisted so both the primary-operation gate below and rawActionCaptures
|
|
11596
11701
|
// (further down) read the same computed sequence instead of calling the
|
|
11597
11702
|
// extractor twice. Computed unfiltered (submitPatterns: null) — a
|
|
@@ -11669,9 +11774,25 @@ async function main() {
|
|
|
11669
11774
|
// keeps that whole chain; the gap between this and the unfiltered
|
|
11670
11775
|
// sequence is logged below for visibility, but the declared pattern is
|
|
11671
11776
|
// never overridden by the richer unfiltered sequence.
|
|
11777
|
+
//
|
|
11778
|
+
// Truncated against a pool RE-EXTRACTED with the declared pattern passed
|
|
11779
|
+
// through (not against unfilteredHeuristicActionCaptures, which is built
|
|
11780
|
+
// with submitPatterns forced to null purely to get an unbiased baseline
|
|
11781
|
+
// for the disagreement check below). extractActionSequence's own
|
|
11782
|
+
// noise/structural-isolation passes have no visibility into the declared
|
|
11783
|
+
// pattern when it's withheld, so on an archive noisy enough to make a
|
|
11784
|
+
// genuine submission capture look structurally isolated, that capture
|
|
11785
|
+
// can be dropped before truncation ever sees it — a real match then
|
|
11786
|
+
// reads as "0 captures" even though the archive contains plenty. Passing
|
|
11787
|
+
// submitPatterns through lets extractActionSequence's own submit-anchor
|
|
11788
|
+
// narrowing (REST) / per-capture submit match (GraphQL) keep the
|
|
11789
|
+
// genuine captures amid that noise, so truncation has them to find.
|
|
11790
|
+
const patternedExtractedActionCaptures = gql
|
|
11791
|
+
? dedupRedundantSameOperationCaptures(extractGraphQLActionSequence(activeCaptures, submitPatterns, foldReturnSpec, ownBackendHostnames, fallbackDomain), primaryGraphQLOperation)
|
|
11792
|
+
: collapseRedundantSameEndpointCaptures(collapseRedundantPatches(extractActionSequence(activeCaptures, submitPatterns, foldReturnSpec, ownBackendHostnames, fallbackDomain, false)));
|
|
11672
11793
|
const patternedHeuristicActionCaptures = submitPatterns.endpoint === null && submitPatterns.body === null
|
|
11673
11794
|
? unfilteredHeuristicActionCaptures
|
|
11674
|
-
: truncateActionSequenceAtSubmitPattern(
|
|
11795
|
+
: truncateActionSequenceAtSubmitPattern(patternedExtractedActionCaptures, submitPatterns, foldReturnSpec);
|
|
11675
11796
|
const patternUndercounts = patternedHeuristicActionCaptures.length < unfilteredHeuristicActionCaptures.length;
|
|
11676
11797
|
if (patternUndercounts && requireSubmitEndpointMatch) {
|
|
11677
11798
|
// Distinct wording from the non-required case below: this pattern is never
|