@crawlee/core 3.15.4-beta.27 → 3.15.4-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.15.4-beta.
|
|
3
|
+
"version": "3.15.4-beta.29",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.15.4-beta.
|
|
63
|
-
"@crawlee/types": "3.15.4-beta.
|
|
64
|
-
"@crawlee/utils": "3.15.4-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.15.4-beta.29",
|
|
63
|
+
"@crawlee/types": "3.15.4-beta.29",
|
|
64
|
+
"@crawlee/utils": "3.15.4-beta.29",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
67
67
|
"csv-stringify": "^6.2.0",
|
|
@@ -83,5 +83,5 @@
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
},
|
|
86
|
-
"gitHead": "
|
|
86
|
+
"gitHead": "aca71047da8f373e86b18479a047b92424b1bd32"
|
|
87
87
|
}
|
|
@@ -48,8 +48,6 @@ import { RequestProvider } from './request_provider';
|
|
|
48
48
|
* await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
|
|
49
49
|
* ```
|
|
50
50
|
* @category Sources
|
|
51
|
-
*
|
|
52
|
-
* @deprecated RequestQueue v1 is deprecated and will be removed in the future. Please use {@link RequestQueue} instead.
|
|
53
51
|
*/
|
|
54
52
|
declare class RequestQueue extends RequestProvider {
|
|
55
53
|
private queryQueueHeadPromise?;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,OAAO,KAAK,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAmBrD
|
|
1
|
+
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,OAAO,KAAK,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAmBrD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,cAAM,YAAa,SAAQ,eAAe;IACtC,OAAO,CAAC,qBAAqB,CAAC,CAMb;IAEjB,OAAO,CAAC,UAAU,CAAqB;IAEvC;;OAEG;gBACS,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAkC;IAYrF;;OAEG;IACI,eAAe,IAAI,MAAM;IAIhC;;;;;;;;;;;;;;;;OAgBG;IACY,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;cAgEvE,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9D;;;;;;;;;;OAUG;cACa,qBAAqB,CACjC,iBAAiB,UAAQ,EACzB,KAAK,SAA8E,EACnF,SAAS,SAAI,GACd,OAAO,CAAC,OAAO,CAAC;IAmGJ,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAqB7C;;;;;OAKG;IACY,cAAc,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;IA2BpF;;OAEG;IACY,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,yBAAyB,GAAG,IAAI,CAAC;cAQ3E,MAAM,IAAI,IAAI;IAMjC;;;;;;;;;;;;;;;OAeG;WACmB,IAAI,CAAC,GAAG,IAAI,EAAE,UAAU,CAAC,OAAO,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;CAGtG;AAED,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,CAAC"}
|
|
@@ -59,8 +59,6 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000;
|
|
|
59
59
|
* await queue.addRequest({ url: 'http://example.com/foo/bar' }, { forefront: true });
|
|
60
60
|
* ```
|
|
61
61
|
* @category Sources
|
|
62
|
-
*
|
|
63
|
-
* @deprecated RequestQueue v1 is deprecated and will be removed in the future. Please use {@link RequestQueue} instead.
|
|
64
62
|
*/
|
|
65
63
|
class RequestQueue extends request_provider_1.RequestProvider {
|
|
66
64
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":";;;AAAA,mDAA2D;AAI3D,0CAA6D;AAE7D,oDAAiD;AAEjD,uDAAuD;AAEvD,yDAAqD;AACrD,mCAOiB;AAEjB,MAAM,mBAAmB,GAAG,OAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC
|
|
1
|
+
{"version":3,"file":"request_queue.js","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":";;;AAAA,mDAA2D;AAI3D,0CAA6D;AAE7D,oDAAiD;AAEjD,uDAAuD;AAEvD,yDAAqD;AACrD,mCAOiB;AAEjB,MAAM,mBAAmB,GAAG,OAAS,CAAC;AAEtC;;;;GAIG;AACH,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,MAAM,YAAa,SAAQ,kCAAe;IAWtC;;OAEG;IACH,YAAY,OAA+B,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QACjF,KAAK,CACD;YACI,GAAG,OAAO;YACV,SAAS,EAAE,cAAc;YACzB,8BAA8B,EAAE,2BAA2B;YAC3D,mBAAmB,EAAE,mBAAmB;SAC3C,EACD,MAAM,CACT,CAAC;QAtBE;;;;mBAMI,IAAI;WAAC;QAET;;;;mBAAa,IAAI,GAAG,EAAU;WAAC;IAevC,CAAC;IAED;;OAEG;IACI,eAAe;QAClB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACM,KAAK,CAAC,gBAAgB;QAC3B,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QAEtD,oCAAoC;QACpC,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,mCAAmC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7F,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;gBAC5E,aAAa;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC;gBAC9C,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,CAAC;aAC1E,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QAE/B,IAAI,OAAuB,CAAC;QAC5B,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,qFAAqF;YACrF,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACtC,MAAM,CAAC,CAAC;QACZ,CAAC;QAED,6HAA6H;QAE7H,yIAAyI;QACzI,6EAA6E;QAC7E,uGAAuG;QACvG,2FAA2F;QAC3F,0FAA0F;QAC1F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0EAA0E,EAAE;gBACvF,aAAa;aAChB,CAAC,CAAC;YACH,UAAU,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAC1C,CAAC,EAAE,wCAAgC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,8FAA8F;QAC9F,wGAAwG;QACxG,4GAA4G;QAC5G,sDAAsD;QACtD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iEAAiE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;YACrG,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;YAC3D,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEkB,KAAK,CAAC,oBAAoB;QACzC,oCAAoC;QACpC,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACvC,CAAC;IAED;;;;;;;;;;OAUG;IACO,KAAK,CAAC,qBAAqB,CACjC,iBAAiB,GAAG,KAAK,EACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,yBAAiB,EAAE,6BAAqB,CAAC,EACnF,SAAS,GAAG,CAAC;QAEb,uDAAuD;QACvD,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,sCAAsC;QACtC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC9B,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAElC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,MAAM;iBACnC,QAAQ,CAAC,EAAE,KAAK,EAAE,CAAC;iBACnB,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,kBAAkB,EAAE,EAAE,EAAE;gBACrD,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE;oBAC3C,uFAAuF;oBACvF,IACI,CAAC,SAAS;wBACV,CAAC,SAAS;wBACV,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;wBAC9B,IAAI,CAAC,4BAA4B,CAAC,GAAG,CAAC,SAAU,CAAC;wBAEjD,OAAO;oBAEX,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;oBACnD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAA,oBAAY,EAAC,SAAS,CAAC,CAAC,EAAE,SAAS,IAAI,KAAK,CAAC;oBACrF,IAAI,CAAC,aAAa,CAAC,IAAA,oBAAY,EAAC,SAAS,CAAC,EAAE;wBACxC,SAAS;wBACT,iBAAiB,EAAE,KAAK;wBACxB,iBAAiB,EAAE,IAAI;wBACvB,SAAS;wBACT,SAAS;qBACZ,CAAC,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEH,mGAAmG;gBACnG,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAElC,OAAO;oBACH,eAAe,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK;oBACtC,SAAS,EAAE,KAAK;oBAChB,eAAe,EAAE,IAAI,IAAI,CAAC,eAAe,CAAC;oBAC1C,cAAc;oBACd,kBAAkB;iBACrB,CAAC;YACN,CAAC,CAAC,CAAC;QACX,CAAC;QAED,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,GACrF,MAAM,IAAI,CAAC,qBAAqB,CAAC;QAErC,4DAA4D;QAE5D,2DAA2D;QAC3D,wFAAwF;QACxF,kEAAkE;QAClE,0DAA0D;QAC1D,kDAAkD;QAElD,uFAAuF;QACvF,IAAI,SAAS,IAAI,qCAA4B,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uDAAuD,qCAA4B,GAAG,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,2BAA2B,GAC7B,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,eAAe,IAAI,SAAS,GAAG,qCAA4B,CAAC;QAEpG,6DAA6D;QAC7D,iGAAiG;QACjG,kFAAkF;QAClF,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,CAAC,eAAe,IAAI,2CAAmC,CAAC;QACvG,MAAM,mBAAmB,GAAG,CAAC,kBAAkB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,mBAAmB,CAAC;QACtG,uFAAuF;QACvF,MAAM,0BAA0B,GAAG,iBAAiB,IAAI,CAAC,oBAAoB,IAAI,CAAC,mBAAmB,CAAC;QAEtG,6DAA6D;QAC7D,IAAI,CAAC,2BAA2B,IAAI,CAAC,0BAA0B;YAAE,OAAO,IAAI,CAAC;QAE7E,yGAAyG;QACzG,0GAA0G;QAC1G,IAAI,CAAC,2BAA2B,IAAI,SAAS,GAAG,mCAA2B;YAAE,OAAO,KAAK,CAAC;QAE1F,MAAM,SAAS,GAAG,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAExF,+DAA+D;QAC/D,IAAI,0BAA0B,EAAE,CAAC;YAC7B,MAAM,WAAW,GAAG,2CAAmC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;YAC1F,IAAI,CAAC,GAAG,CAAC,IAAI,CACT,eAAe,WAAW,oFAAoF,CACjH,CAAC;YACF,MAAM,IAAA,qBAAK,EAAC,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,2CAA2C;IAClC,KAAK,CAAC,UAAU;QACrB,IAAA,oCAAkB,GAAE,CAAC;QAErB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/D,MAAM,OAAO,GAAG,2CACZ,IAAI,CAAC,qBAAqB,GAAG,GACjC,8BAA8B,CAAC;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,CAAC,2BAA2B,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAE/E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAChG,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAmD;QAChF,IAAA,oCAAkB,GAAE,CAAC;QAErB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QAChC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,KAAK,CAAC;QAE9C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;QAEnD,yHAAyH;QACzH,2HAA2H;QAC3H,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAG,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,+DAA+D,EAAE;oBAC5E,SAAS,EAAE,OAAO,CAAC,EAAE;iBACxB,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YAED,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;YAEpC,qEAAqE;YACrE,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,EAAG,EAAE,SAAS,CAAC,CAAC;QAC7D,CAAC,EAAE,wCAAgC,CAAC,CAAC;QAErC,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QAC9C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAEpD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;QAEpC,OAAO,GAAG,CAAC;IACf,CAAC;IAEkB,MAAM;QACrB,KAAK,CAAC,MAAM,EAAE,CAAC;QAEf,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,GAAG,IAA6C;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAA0B,CAAC;IACxD,CAAC;CACJ;AAEwB,sCAAc"}
|