@crawlee/utils 3.16.1-beta.41 → 3.16.1-beta.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/internals/iterables.d.ts
CHANGED
|
@@ -60,7 +60,7 @@ export declare function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterabl
|
|
|
60
60
|
* }
|
|
61
61
|
* ```
|
|
62
62
|
*/
|
|
63
|
-
export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number): AsyncIterable<T[]>;
|
|
63
|
+
export declare function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number | (() => number)): AsyncIterable<T[]>;
|
|
64
64
|
/**
|
|
65
65
|
* An async iterator that also supports peeking at the next value without consuming it.
|
|
66
66
|
* Extends both AsyncIterator and AsyncIterable interfaces.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterables.d.ts","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAUlE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAM5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAuB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAErG;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EACzC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"iterables.d.ts","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAUlE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,CAM5E;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAuB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAErG;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EACzC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,GACnC,aAAa,CAAC,CAAC,EAAE,CAAC,CA6BpB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF;;;;;OAKG;IACH,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC,CAAE,SAAQ,aAAa,CAAC,CAAC,CAAC;IAC9D,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,qBAAqB,CAAC,CAAC,CAAC,CA+D3G;AAGD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAuB,mBAAmB,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,CAgChG"}
|
package/internals/iterables.js
CHANGED
|
@@ -85,18 +85,27 @@ async function* asyncifyIterable(iterable) {
|
|
|
85
85
|
* ```
|
|
86
86
|
*/
|
|
87
87
|
async function* chunkedAsyncIterable(iterable, chunkSize) {
|
|
88
|
-
|
|
88
|
+
const getChunkSize = typeof chunkSize === 'function' ? chunkSize : () => chunkSize;
|
|
89
|
+
if (typeof chunkSize === 'number' && chunkSize < 1) {
|
|
89
90
|
throw new Error(`Chunk size must be a positive number (${(0, node_util_1.inspect)(chunkSize)}) received`);
|
|
90
91
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
92
|
+
const iterator = Symbol.asyncIterator in iterable
|
|
93
|
+
? iterable[Symbol.asyncIterator]()
|
|
94
|
+
: iterable[Symbol.iterator]();
|
|
95
|
+
while (true) {
|
|
96
|
+
const currentSize = getChunkSize();
|
|
97
|
+
if (currentSize < 1)
|
|
98
|
+
break;
|
|
99
|
+
const chunk = [];
|
|
100
|
+
for (let i = 0; i < currentSize; i++) {
|
|
101
|
+
const next = await iterator.next();
|
|
102
|
+
if (next.done) {
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
chunk.push(next.value);
|
|
97
106
|
}
|
|
98
|
-
|
|
99
|
-
|
|
107
|
+
if (chunk.length === 0)
|
|
108
|
+
break;
|
|
100
109
|
yield chunk;
|
|
101
110
|
}
|
|
102
111
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":";;AAeA,gCAUC;AAeD,0CAMC;AAgBD,4CAEC;AAqBD,
|
|
1
|
+
{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":";;AAeA,gCAUC;AAeD,0CAMC;AAgBD,4CAEC;AAqBD,oDAgCC;AA+CD,sDA+DC;AAqBD,kDAgCC;AAxRD,yCAAoC;AAEpC;;;;;;;;;;;;GAYG;AACH,SAAgB,UAAU,CAAI,KAAc;IACxC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,eAAe,CAAI,KAAc;IAC7C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AACrE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACI,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAI,QAAwC;IAC/E,KAAK,CAAC,CAAC,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,QAAwC,EACxC,SAAkC;IAElC,MAAM,YAAY,GAAG,OAAO,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;IAEnF,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yCAAyC,IAAA,mBAAO,EAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC7F,CAAC;IAED,MAAM,QAAQ,GACV,MAAM,CAAC,aAAa,IAAI,QAAQ;QAC5B,CAAC,CAAE,QAA6B,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QACxD,CAAC,CAAE,QAAwB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;IAEvD,OAAO,IAAI,EAAE,CAAC;QACV,MAAM,WAAW,GAAG,YAAY,EAAE,CAAC;QACnC,IAAI,WAAW,GAAG,CAAC;YAAE,MAAM;QAE3B,MAAM,KAAK,GAAQ,EAAE,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACZ,MAAM;YACV,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM;QAC9B,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAyBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAgB,qBAAqB,CAAI,QAAwC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpE,IAAI,WAAoD,CAAC;IACzD,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,MAAM,gBAAgB,GAA6B;QAC/C,KAAK,CAAC,IAAI;YACN,0DAA0D;YAC1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC;gBAC3B,WAAW,GAAG,SAAS,CAAC;gBAExB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,WAAW,GAAG,IAAI,CAAC;oBACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC5C,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAChD,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC5C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACN,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YAC5D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAElE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;KACJ,CAAC;IAEF,OAAO;QACH,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,gBAAgB,CAAC;QAC5B,CAAC;KACJ,CAAC;AACN,CAAC;AAED,gDAAgD;AAChD;;;;;;;;;;;;;;;;;GAiBG;AACI,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAI,GAAG,SAA6B;IAC1E,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,IAAI,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAClC,MAAM,KAAK,GAAmB,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpD,KAAK,UAAU,OAAO,CAAC,aAA+B,EAAE,KAAa;QACjE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACH,KAAK;YACL,MAAM;SACT,CAAC;IACN,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,IAAI,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9B,KAAK,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,MAAM,CAAC,KAAK,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;YAAS,CAAC;QACP,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,kFAAkF;YAClF,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAAE,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzF,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/utils",
|
|
3
|
-
"version": "3.16.1-beta.
|
|
3
|
+
"version": "3.16.1-beta.43",
|
|
4
4
|
"description": "A set of shared utilities that can be used by crawlers",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/log": "^2.4.0",
|
|
51
51
|
"@apify/ps-tree": "^1.2.0",
|
|
52
|
-
"@crawlee/types": "3.16.1-beta.
|
|
52
|
+
"@crawlee/types": "3.16.1-beta.43",
|
|
53
53
|
"@types/sax": "^1.2.7",
|
|
54
54
|
"cheerio": "1.0.0-rc.12",
|
|
55
55
|
"file-type": "^21.3.1",
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "db74fcbe6f33aa2867ee6a2a99fc47a9a9633bba"
|
|
74
74
|
}
|