@crawlee/http 3.13.5-beta.5 → 3.13.5-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.mjs
CHANGED
|
@@ -6,6 +6,7 @@ export const AutoscaledPool = mod.AutoscaledPool;
|
|
|
6
6
|
export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = mod.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS;
|
|
7
7
|
export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
|
|
8
8
|
export const BasicCrawler = mod.BasicCrawler;
|
|
9
|
+
export const ByteCounterStream = mod.ByteCounterStream;
|
|
9
10
|
export const Configuration = mod.Configuration;
|
|
10
11
|
export const CookieParseError = mod.CookieParseError;
|
|
11
12
|
export const CrawlerExtension = mod.CrawlerExtension;
|
|
@@ -30,6 +31,7 @@ export const LoggerJson = mod.LoggerJson;
|
|
|
30
31
|
export const LoggerText = mod.LoggerText;
|
|
31
32
|
export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
|
|
32
33
|
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
34
|
+
export const MinimumSpeedStream = mod.MinimumSpeedStream;
|
|
33
35
|
export const MissingRouteError = mod.MissingRouteError;
|
|
34
36
|
export const NonRetryableError = mod.NonRetryableError;
|
|
35
37
|
export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
1
2
|
import type { Dictionary } from '@crawlee/types';
|
|
2
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
3
4
|
import type { Request } from 'got-scraping';
|
|
@@ -24,6 +25,31 @@ JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData,
|
|
|
24
25
|
}
|
|
25
26
|
export type FileDownloadRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
26
27
|
JSONData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData, JSONData>>;
|
|
28
|
+
/**
|
|
29
|
+
* Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
|
|
30
|
+
* This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
|
|
31
|
+
* If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
|
|
32
|
+
* it will throw an error.
|
|
33
|
+
*
|
|
34
|
+
* Can be used e.g. to abort a download if the network speed is too slow.
|
|
35
|
+
* @returns Transform stream that monitors the speed of the incoming data.
|
|
36
|
+
*/
|
|
37
|
+
export declare function MinimumSpeedStream({ minSpeedKbps, historyLengthMs, checkProgressInterval: checkProgressIntervalMs, }: {
|
|
38
|
+
minSpeedKbps: number;
|
|
39
|
+
historyLengthMs?: number;
|
|
40
|
+
checkProgressInterval?: number;
|
|
41
|
+
}): Transform;
|
|
42
|
+
/**
|
|
43
|
+
* Creates a transform stream that logs the progress of the incoming data.
|
|
44
|
+
* This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
|
|
45
|
+
*
|
|
46
|
+
* Can be used e.g. to log the progress of a download.
|
|
47
|
+
* @returns Transform stream logging the progress of the incoming data.
|
|
48
|
+
*/
|
|
49
|
+
export declare function ByteCounterStream({ logTransferredBytes, loggingInterval, }: {
|
|
50
|
+
logTransferredBytes: (transferredBytes: number) => void;
|
|
51
|
+
loggingInterval?: number;
|
|
52
|
+
}): Transform;
|
|
27
53
|
/**
|
|
28
54
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
29
55
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,OAAO,KAAK,EACR,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,YAAY,EACf,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,WAAW,EAAU,MAAM,UAAU,CAAC;AAE/C,MAAM,MAAM,wBAAwB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElE,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACnC,2BAA2B,EAC3B,MAAM,GAAG,kBAAkB,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CACvE,GAAG;IACA,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AAEF,KAAK,aAAa,GAAG,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IAE/B,CAAC,IAAI,CAAC,kBAAkB,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,GAAG;IAC3F,cAAc,CAAC,EAAE,KAAK,CAAC;IACvB,aAAa,CAAC,EAAE,aAAa,CAAC;CACjC,CAAC,GACF,CAAC,IAAI,CAAC,kBAAkB,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,GAAG;IAC3F,cAAc,EAAE,0BAA0B,CAAC;IAC3C,aAAa,CAAC,EAAE,KAAK,CAAC;CACzB,CAAC,CAAC;AAET,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEtE,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC;CAAG;AAE1E,MAAM,MAAM,0BAA0B,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEpE;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAsB,EACtB,qBAAqB,EAAE,uBAA6B,GACvD,EAAE;IACC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,SAAS,CA4BZ;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAsB,GACzB,EAAE;IAAE,mBAAmB,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAoBnG;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,qBAAa,YAAa,SAAQ,WAAW,CAAC,2BAA2B,CAAC;IACtE,OAAO,CAAC,aAAa,CAAC,CAAgB;gBAE1B,OAAO,GAAE,mBAAwB;cAqBpB,kBAAkB,CAAC,OAAO,EAAE,2BAA2B;YAQlE,oBAAoB;CA8DrC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,8DAEzC"}
|
|
@@ -1,10 +1,71 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.FileDownload = void 0;
|
|
4
|
+
exports.MinimumSpeedStream = MinimumSpeedStream;
|
|
5
|
+
exports.ByteCounterStream = ByteCounterStream;
|
|
4
6
|
exports.createFileRouter = createFileRouter;
|
|
7
|
+
const node_stream_1 = require("node:stream");
|
|
5
8
|
const promises_1 = require("node:stream/promises");
|
|
6
9
|
const types_1 = require("node:util/types");
|
|
7
10
|
const index_1 = require("../index");
|
|
11
|
+
/**
|
|
12
|
+
* Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
|
|
13
|
+
* This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
|
|
14
|
+
* If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
|
|
15
|
+
* it will throw an error.
|
|
16
|
+
*
|
|
17
|
+
* Can be used e.g. to abort a download if the network speed is too slow.
|
|
18
|
+
* @returns Transform stream that monitors the speed of the incoming data.
|
|
19
|
+
*/
|
|
20
|
+
function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
|
|
21
|
+
let snapshots = [];
|
|
22
|
+
const checkInterval = setInterval(() => {
|
|
23
|
+
const now = Date.now();
|
|
24
|
+
snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
|
|
25
|
+
const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
|
|
26
|
+
const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
|
|
27
|
+
if (totalBytes / 1024 / elapsed < minSpeedKbps) {
|
|
28
|
+
clearInterval(checkInterval);
|
|
29
|
+
stream.emit('error', new Error(`Stream speed too slow, aborting...`));
|
|
30
|
+
}
|
|
31
|
+
}, checkProgressIntervalMs);
|
|
32
|
+
const stream = new node_stream_1.Transform({
|
|
33
|
+
transform: (chunk, _, callback) => {
|
|
34
|
+
snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
|
|
35
|
+
callback(null, chunk);
|
|
36
|
+
},
|
|
37
|
+
final: (callback) => {
|
|
38
|
+
clearInterval(checkInterval);
|
|
39
|
+
callback();
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
return stream;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Creates a transform stream that logs the progress of the incoming data.
|
|
46
|
+
* This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
|
|
47
|
+
*
|
|
48
|
+
* Can be used e.g. to log the progress of a download.
|
|
49
|
+
* @returns Transform stream logging the progress of the incoming data.
|
|
50
|
+
*/
|
|
51
|
+
function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
|
|
52
|
+
let transferredBytes = 0;
|
|
53
|
+
let lastLogTime = Date.now();
|
|
54
|
+
return new node_stream_1.Transform({
|
|
55
|
+
transform: (chunk, _, callback) => {
|
|
56
|
+
transferredBytes += chunk.length;
|
|
57
|
+
if (Date.now() - lastLogTime > loggingInterval) {
|
|
58
|
+
lastLogTime = Date.now();
|
|
59
|
+
logTransferredBytes(transferredBytes);
|
|
60
|
+
}
|
|
61
|
+
callback(null, chunk);
|
|
62
|
+
},
|
|
63
|
+
flush: (callback) => {
|
|
64
|
+
logTransferredBytes(transferredBytes);
|
|
65
|
+
callback();
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
}
|
|
8
69
|
/**
|
|
9
70
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
10
71
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":";;;AAsEA,gDAoCC;AASD,8CAuBC;AAqKD,4CAKC;AApTD,6CAAwC;AACxC,mDAAgD;AAChD,2CAA4C;AAe5C,oCAA+C;AA4C/C;;;;;;;;GAQG;AACH,SAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,uBAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAC8D;IACpF,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,uBAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAa,YAAa,SAAQ,mBAAwC;IAGtE,YAAY,UAA+B,EAAE;QACzC,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,OAAO,OAAO,CAAC,aAAa,CAAC;QAE7B,IAAI,aAAa,EAAE,CAAC;YAChB,8DAA8D;YAC7D,OAAe,CAAC,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,IAAI,GAAG,CAAC;QACtF,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,CAAC;QAXX;;;;;WAA8B;QAalC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAA2B,CAAC;QAC3D,CAAC;QAED,+EAA+E;QAC/E,oEAAoE;QACnE,IAAY,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxD,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,OAAoC;QAC5E,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO,CAAC,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;QAC1C,CAAC;QAED,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAoC;QACnE,MAAM,EACF,GAAG,EACH,OAAO,EAAE,EAAE,GAAG,EAAE,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC1C,GAAG;YACH,OAAO,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YAC/B,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG;SACnC,CAAC,CAAC;QAEH,IAAI,eAA2C,CAAC;QAEhD,MAAM,OAAO,GAAG,GAAG,EAAE;YACjB,aAAa,CAAC,eAAgB,CAAC,CAAC;YAChC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC,CAAC;QAEF,MAAM,eAAe,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1D,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC/B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,QAAQ,CAAC,gBAAgB,CAAC;gBAEzD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;oBAClB,GAAG,CAAC,KAAK,CAAC,cAAc,WAAW,aAAa,KAAK,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAY,EAAE,EAAE;gBAC/C,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YAEH,IAAI,mBAAmB,CAAC;YAExB,IAAI,CAAC;gBACD,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACjC,OAAO,CAAC,QAAQ,GAAG,QAAe,CAAC;gBACnC,mBAAmB,GAAG,IAAI,CAAC,aAAc,CAAC,OAAc,CAAC,CAAC;YAC9D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YAED,IAAI,IAAA,iBAAS,EAAC,mBAAmB,CAAC,EAAE,CAAC;gBACjC,mBAAmB;qBACd,IAAI,CAAC,GAAG,EAAE;oBACP,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,CAAQ,EAAE,EAAE;oBAChB,OAAO,EAAE,CAAC;oBACV,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,CAAC,CAAC,CAAC;YACX,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,eAAe,EAAE,IAAA,mBAAQ,EAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAEhE,OAAO,EAAE,CAAC;IACd,CAAC;CACJ;AA9FD,oCA8FC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,cAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "3.13.5-beta.
|
|
3
|
+
"version": "3.13.5-beta.7",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,9 +55,9 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@apify/timeout": "^0.3.0",
|
|
57
57
|
"@apify/utilities": "^2.7.10",
|
|
58
|
-
"@crawlee/basic": "3.13.5-beta.
|
|
59
|
-
"@crawlee/types": "3.13.5-beta.
|
|
60
|
-
"@crawlee/utils": "3.13.5-beta.
|
|
58
|
+
"@crawlee/basic": "3.13.5-beta.7",
|
|
59
|
+
"@crawlee/types": "3.13.5-beta.7",
|
|
60
|
+
"@crawlee/utils": "3.13.5-beta.7",
|
|
61
61
|
"@types/content-type": "^1.1.5",
|
|
62
62
|
"cheerio": "1.0.0-rc.12",
|
|
63
63
|
"content-type": "^1.0.4",
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "20d955daba865f1ec9a86bd8e4ad3592d4555c7e"
|
|
79
79
|
}
|