@enricai/barnacle 1.12.47 → 1.12.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/recon/capture-filters.d.ts +48 -5
- package/dist/recon/capture-filters.d.ts.map +1 -1
- package/dist/recon/capture-filters.js +99 -8
- package/dist/recon/capture-filters.js.map +1 -1
- package/dist/scraper/captcha-callback-capture.d.ts +6 -0
- package/dist/scraper/captcha-callback-capture.d.ts.map +1 -1
- package/dist/scraper/captcha-callback-capture.js +49 -1
- package/dist/scraper/captcha-callback-capture.js.map +1 -1
- package/dist/scraper/flow-runner.d.ts +44 -0
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +72 -7
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scripts/recon-generate-multicall-fixture.d.ts +17 -0
- package/dist/scripts/recon-generate-multicall-fixture.d.ts.map +1 -1
- package/dist/scripts/recon-generate-multicall-fixture.js +54 -0
- package/dist/scripts/recon-generate-multicall-fixture.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +13 -1
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +154 -11
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +1 -1
|
@@ -1744,6 +1744,110 @@ function collapseRedundantPatches(actions) {
|
|
|
1744
1744
|
return lastPatchByPath.get(path) === i;
|
|
1745
1745
|
});
|
|
1746
1746
|
}
|
|
1747
|
+
/** Request field name shapes that identify a paged/offset-style re-query --
|
|
1748
|
+
* the REST analog of {@link PAGE_SIZE_KEY_PATTERN}/{@link SKIP_KEY_PATTERN},
|
|
1749
|
+
* loosened to match a single paging cursor field on its own (a plain `page`
|
|
1750
|
+
* counter, unpaired with an explicit page-size key) since that is the common
|
|
1751
|
+
* REST shape, unlike GraphQL's paired variables convention. */
|
|
1752
|
+
const PAGINATION_FIELD_NAME_PATTERN = /^(page|pagenum|pagenumber|pageindex|pageno|offset|skip|start|cursor)$/i;
|
|
1753
|
+
/** Every query-string and (when JSON-object-shaped) request-body field on a
|
|
1754
|
+
* capture, merged into one comparable map -- REST pagination/facet state can
|
|
1755
|
+
* live in either depending on the endpoint's own convention. */
|
|
1756
|
+
function captureRequestFields(capture) {
|
|
1757
|
+
const fields = {};
|
|
1758
|
+
try {
|
|
1759
|
+
const url = new URL(capture.url);
|
|
1760
|
+
for (const [key, value] of url.searchParams)
|
|
1761
|
+
fields[key] = value;
|
|
1762
|
+
}
|
|
1763
|
+
catch {
|
|
1764
|
+
// Relative/invalid URLs carry no query-string signal to merge in.
|
|
1765
|
+
}
|
|
1766
|
+
if (capture.requestPostData) {
|
|
1767
|
+
try {
|
|
1768
|
+
const parsed = JSON.parse(capture.requestPostData);
|
|
1769
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
1770
|
+
Object.assign(fields, parsed);
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
catch {
|
|
1774
|
+
// A non-JSON body carries no per-field signal to merge in.
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1777
|
+
return fields;
|
|
1778
|
+
}
|
|
1779
|
+
/**
|
|
1780
|
+
* A same-endpoint capture group qualifies for collapsing when every capture
|
|
1781
|
+
* resolves to the same {@link responseShapeKey} (ruling out a mutation POST,
|
|
1782
|
+
* whose response is a single mutated object rather than an array, and any
|
|
1783
|
+
* group whose members diverge in response shape) AND its request fields vary
|
|
1784
|
+
* in at most one field once known non-semantic noise keys ({@link
|
|
1785
|
+
* CACHE_BUSTER_QUERY_KEYS}) are excluded from consideration, and that
|
|
1786
|
+
* remaining field is pagination-shaped -- a paged listing/facet re-query --
|
|
1787
|
+
* or vary in no field at all -- a polled toggles/feature-flag endpoint
|
|
1788
|
+
* re-fired with an identical request. A group
|
|
1789
|
+
* whose members vary in a non-pagination field (e.g. a per-item drill's
|
|
1790
|
+
* item-id body field) is left untouched: that variance carries the distinct
|
|
1791
|
+
* per-item state the existing fold-chain mechanism (`target.chain` in
|
|
1792
|
+
* `emitMultiStepExecuteHttp`) already hoists correctly once resolved, and
|
|
1793
|
+
* collapsing it here would erase the very state that hoisting depends on.
|
|
1794
|
+
*/
|
|
1795
|
+
function isRedundantSameEndpointGroup(group) {
|
|
1796
|
+
const shapeKey = responseShapeKey(group[0].capture);
|
|
1797
|
+
if (shapeKey === null)
|
|
1798
|
+
return false;
|
|
1799
|
+
if (!group.every((a) => responseShapeKey(a.capture) === shapeKey))
|
|
1800
|
+
return false;
|
|
1801
|
+
const fieldSets = group.map((a) => captureRequestFields(a.capture));
|
|
1802
|
+
const allKeys = new Set();
|
|
1803
|
+
for (const fields of fieldSets) {
|
|
1804
|
+
for (const key of Object.keys(fields))
|
|
1805
|
+
allKeys.add(key);
|
|
1806
|
+
}
|
|
1807
|
+
const varyingKeys = [...allKeys].filter((key) => {
|
|
1808
|
+
if (CACHE_BUSTER_QUERY_KEYS.has(key))
|
|
1809
|
+
return false;
|
|
1810
|
+
const values = new Set(fieldSets.map((fields) => JSON.stringify(fields[key])));
|
|
1811
|
+
return values.size > 1;
|
|
1812
|
+
});
|
|
1813
|
+
if (varyingKeys.length === 0)
|
|
1814
|
+
return true;
|
|
1815
|
+
return varyingKeys.length === 1 && PAGINATION_FIELD_NAME_PATTERN.test(varyingKeys[0]);
|
|
1816
|
+
}
|
|
1817
|
+
/**
|
|
1818
|
+
* REST counterpart of {@link dedupRedundantSameOperationCaptures}: collapses
|
|
1819
|
+
* each same-endpoint (method + {@link endpointKey}) group that qualifies per
|
|
1820
|
+
* {@link isRedundantSameEndpointGroup} down to its FIRST occurrence. A paged
|
|
1821
|
+
* listing re-fired across pages/facets, or a polled toggles endpoint re-fired
|
|
1822
|
+
* with an identical request, carries no distinct state for downstream steps
|
|
1823
|
+
* to thread; every other same-endpoint group (including a per-item drill
|
|
1824
|
+
* varying by item id) is left exactly as extracted. Kept representative is
|
|
1825
|
+
* the first occurrence, not the last, because a later step's fold/join (see
|
|
1826
|
+
* `buildFoldPlanFromSpec`) resolves its item-level join values against
|
|
1827
|
+
* whichever page's response survives — page 1 is what a browsing/drill flow
|
|
1828
|
+
* actually saw and drilled into first, so it is the occurrence downstream
|
|
1829
|
+
* join values are captured against, not the endpoint's final paged state.
|
|
1830
|
+
*/
|
|
1831
|
+
function collapseRedundantSameEndpointCaptures(actions) {
|
|
1832
|
+
const positionsByGroup = new Map();
|
|
1833
|
+
actions.forEach((a, i) => {
|
|
1834
|
+
const key = `${a.capture.method} ${endpointKey(a.capture.url)}`;
|
|
1835
|
+
const positions = positionsByGroup.get(key) ?? [];
|
|
1836
|
+
positions.push(i);
|
|
1837
|
+
positionsByGroup.set(key, positions);
|
|
1838
|
+
});
|
|
1839
|
+
const drop = new Set();
|
|
1840
|
+
for (const positions of positionsByGroup.values()) {
|
|
1841
|
+
if (positions.length < 2)
|
|
1842
|
+
continue;
|
|
1843
|
+
const group = positions.map((i) => actions[i]);
|
|
1844
|
+
if (!isRedundantSameEndpointGroup(group))
|
|
1845
|
+
continue;
|
|
1846
|
+
for (const position of positions.slice(1))
|
|
1847
|
+
drop.add(position);
|
|
1848
|
+
}
|
|
1849
|
+
return actions.filter((_, i) => !drop.has(i));
|
|
1850
|
+
}
|
|
1747
1851
|
/**
|
|
1748
1852
|
* Recursively walks a JSON value and yields every string leaf, paired with its
|
|
1749
1853
|
* JSON path. Numbers/booleans/nulls are skipped — only string leaves are
|
|
@@ -6005,7 +6109,7 @@ function detectDrillDownFoldPlan(actions) {
|
|
|
6005
6109
|
// A re-queried primary (same endpoint hit more than once, per
|
|
6006
6110
|
// findRequeriedActions) can have MULTIPLE occurrences that each
|
|
6007
6111
|
// independently thread a join key into the SAME later drill-down —
|
|
6008
|
-
// e.g. two "
|
|
6112
|
+
// e.g. two "list-items" calls that both happen to contain the
|
|
6009
6113
|
// item the drill-down looks up. selectReturnAction/selectPayloadAction
|
|
6010
6114
|
// already establish freshest-wins for this exact re-queried-primary
|
|
6011
6115
|
// case, so the plan must anchor on the LAST such occurrence, not the
|
|
@@ -8434,15 +8538,29 @@ function resolvedPrimaryResponseCapture(result) {
|
|
|
8434
8538
|
return result.winningCapture;
|
|
8435
8539
|
return selectReturnAction(result.actionSteps)?.capture ?? null;
|
|
8436
8540
|
}
|
|
8437
|
-
/** True when `
|
|
8438
|
-
*
|
|
8439
|
-
* {@link
|
|
8440
|
-
*
|
|
8441
|
-
|
|
8442
|
-
|
|
8443
|
-
if (typeof
|
|
8541
|
+
/** True when `fieldName` is an own key of `value` at any nesting depth up to
|
|
8542
|
+
* `maxDepth`, walking plain objects and arrays — the same recursion the
|
|
8543
|
+
* schema-inference walk ({@link inferZodSchema}, bounded by
|
|
8544
|
+
* {@link DEFAULT_MAX_INFER_DEPTH}) uses to discover fields in the first
|
|
8545
|
+
* place, so attribution can't miss a field the inference walk found. */
|
|
8546
|
+
function hasOwnFieldAtAnyDepth(value, fieldName, maxDepth) {
|
|
8547
|
+
if (maxDepth < 0 || typeof value !== "object" || value === null)
|
|
8444
8548
|
return false;
|
|
8445
|
-
|
|
8549
|
+
if (Array.isArray(value)) {
|
|
8550
|
+
return value.some((item) => hasOwnFieldAtAnyDepth(item, fieldName, maxDepth - 1));
|
|
8551
|
+
}
|
|
8552
|
+
const record = value;
|
|
8553
|
+
if (Object.hasOwn(record, fieldName))
|
|
8554
|
+
return true;
|
|
8555
|
+
return Object.values(record).some((child) => hasOwnFieldAtAnyDepth(child, fieldName, maxDepth - 1));
|
|
8556
|
+
}
|
|
8557
|
+
/** True when `capture`'s response JSON carries the field's source anywhere in
|
|
8558
|
+
* its nested shape — the same "does this capture's response carry the field"
|
|
8559
|
+
* signal {@link assertRequiredUrlFieldsReferenced} implicitly relies on,
|
|
8560
|
+
* applied per capture instead of to the merged emitted code, walked to the
|
|
8561
|
+
* same depth the schema-inference walk supports. */
|
|
8562
|
+
function captureOwnsTopLevelField(capture, fieldName) {
|
|
8563
|
+
return hasOwnFieldAtAnyDepth(capture.responseBody, fieldName, DEFAULT_MAX_INFER_DEPTH);
|
|
8446
8564
|
}
|
|
8447
8565
|
/**
|
|
8448
8566
|
* Builds the "field(s) ... capture(s) ..." detail the intent requires a
|
|
@@ -8467,11 +8585,23 @@ function describeOffendingFieldSources(fields, pool) {
|
|
|
8467
8585
|
* The narrowing pass's core relevance decision, isolated for direct
|
|
8468
8586
|
* testing: every capture in `resolvedPool` — other than `primaryCapture`,
|
|
8469
8587
|
* which must never be dropped — whose own top-level response JSON owns at
|
|
8470
|
-
* least one of `offendingFields
|
|
8588
|
+
* least one of `offendingFields`, PLUS any other same-host capture whose
|
|
8589
|
+
* path is in the same structural family ({@link isSamePathFamily}) as one
|
|
8590
|
+
* of those field-owning captures. This is the actual "is this capture
|
|
8471
8591
|
* structurally part of the resolved chain or incidental noise" call;
|
|
8472
8592
|
* {@link healUnreferencedUrlFieldsOnce} only wires it into the regenerate
|
|
8473
8593
|
* retry loop.
|
|
8474
8594
|
*
|
|
8595
|
+
* The family-broadening step exists because a noise endpoint's alternate
|
|
8596
|
+
* path/query-string variant (e.g. a `/default` GET variant of a POST the
|
|
8597
|
+
* field check already caught) commonly carries none of the fields that
|
|
8598
|
+
* flagged its sibling — it is caught by structural isolation alone, the
|
|
8599
|
+
* same signal {@link isStructurallyIsolatedCapture} uses for pool-relative
|
|
8600
|
+
* isolation, generalized here to "related to an already-known noise path"
|
|
8601
|
+
* rather than "isolated from the whole pool". Without it, that variant
|
|
8602
|
+
* survives to the emitted contract as its own hard-coded call, requiring a
|
|
8603
|
+
* required-URL-field trigger of its own it may never have.
|
|
8604
|
+
*
|
|
8475
8605
|
* Exported for tests: this predicate decides which captures the required-
|
|
8476
8606
|
* URL-field guard's self-heal excludes before regenerating.
|
|
8477
8607
|
*/
|
|
@@ -8485,6 +8615,19 @@ function identifyNoiseCapturesForFields(offendingFields, resolvedPool, primaryCa
|
|
|
8485
8615
|
noiseCaptures.add(capture);
|
|
8486
8616
|
}
|
|
8487
8617
|
}
|
|
8618
|
+
const noiseUrls = [...noiseCaptures].map((capture) => ({
|
|
8619
|
+
path: safeUrlPathname(capture.url),
|
|
8620
|
+
hostname: captureHostname(capture.url),
|
|
8621
|
+
}));
|
|
8622
|
+
for (const { capture } of resolvedPool) {
|
|
8623
|
+
if (capture === primaryCapture || noiseCaptures.has(capture))
|
|
8624
|
+
continue;
|
|
8625
|
+
const path = safeUrlPathname(capture.url);
|
|
8626
|
+
const hostname = captureHostname(capture.url);
|
|
8627
|
+
const isSameHostFamilyMatch = noiseUrls.some((noise) => noise.hostname === hostname && (0, capture_filters_1.isSamePathFamily)(path, noise.path));
|
|
8628
|
+
if (isSameHostFamilyMatch)
|
|
8629
|
+
noiseCaptures.add(capture);
|
|
8630
|
+
}
|
|
8488
8631
|
return noiseCaptures;
|
|
8489
8632
|
}
|
|
8490
8633
|
/**
|
|
@@ -8822,7 +8965,7 @@ async function main() {
|
|
|
8822
8965
|
// heuristic extraction finds.
|
|
8823
8966
|
const unfilteredHeuristicActionCaptures = gql
|
|
8824
8967
|
? dedupRedundantSameOperationCaptures(graphqlActionSequence, primaryGraphQLOperation)
|
|
8825
|
-
: collapseRedundantPatches(extractActionSequence(activeCaptures, null, foldReturnSpec, ownBackendHostnames, fallbackDomain));
|
|
8968
|
+
: collapseRedundantSameEndpointCaptures(collapseRedundantPatches(extractActionSequence(activeCaptures, null, foldReturnSpec, ownBackendHostnames, fallbackDomain)));
|
|
8826
8969
|
// A flow-declared submitEndpointPattern is authoritative: it may match only
|
|
8827
8970
|
// the final step's URL (the natural way to describe "the button that
|
|
8828
8971
|
// finishes the wizard") even though the earlier steps of the same chain
|