@crawlee/http 3.16.1-beta.4 → 3.16.1-beta.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.mjs
CHANGED
|
@@ -31,6 +31,7 @@ export const LoggerJson = mod.LoggerJson;
|
|
|
31
31
|
export const LoggerText = mod.LoggerText;
|
|
32
32
|
export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
|
|
33
33
|
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
34
|
+
export const MemoryLoadSignal = mod.MemoryLoadSignal;
|
|
34
35
|
export const MinimumSpeedStream = mod.MinimumSpeedStream;
|
|
35
36
|
export const MissingRouteError = mod.MissingRouteError;
|
|
36
37
|
export const NonRetryableError = mod.NonRetryableError;
|
|
@@ -59,6 +60,7 @@ export const Session = mod.Session;
|
|
|
59
60
|
export const SessionError = mod.SessionError;
|
|
60
61
|
export const SessionPool = mod.SessionPool;
|
|
61
62
|
export const SitemapRequestList = mod.SitemapRequestList;
|
|
63
|
+
export const SnapshotStore = mod.SnapshotStore;
|
|
62
64
|
export const Snapshotter = mod.Snapshotter;
|
|
63
65
|
export const Statistics = mod.Statistics;
|
|
64
66
|
export const StorageManager = mod.StorageManager;
|
|
@@ -72,13 +74,17 @@ export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFr
|
|
|
72
74
|
export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
|
|
73
75
|
export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
|
|
74
76
|
export const createBasicRouter = mod.createBasicRouter;
|
|
77
|
+
export const createClientLoadSignal = mod.createClientLoadSignal;
|
|
78
|
+
export const createCpuLoadSignal = mod.createCpuLoadSignal;
|
|
75
79
|
export const createDeserialize = mod.createDeserialize;
|
|
80
|
+
export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
|
|
76
81
|
export const createFileRouter = mod.createFileRouter;
|
|
77
82
|
export const createHttpRouter = mod.createHttpRouter;
|
|
78
83
|
export const createRequestOptions = mod.createRequestOptions;
|
|
79
84
|
export const createRequests = mod.createRequests;
|
|
80
85
|
export const deserializeArray = mod.deserializeArray;
|
|
81
86
|
export const enqueueLinks = mod.enqueueLinks;
|
|
87
|
+
export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
|
|
82
88
|
export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
|
|
83
89
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
84
90
|
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
@@ -8,6 +8,7 @@ export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with
|
|
|
8
8
|
JSONData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData, JSONData>>;
|
|
9
9
|
export type StreamHandlerContext = Omit<FileDownloadCrawlingContext, 'body' | 'parseWithCheerio' | 'json' | 'addRequests' | 'contentType'> & {
|
|
10
10
|
stream: Request;
|
|
11
|
+
abortDownload: () => void;
|
|
11
12
|
};
|
|
12
13
|
type StreamHandler = (context: StreamHandlerContext) => void | Promise<void>;
|
|
13
14
|
export type FileDownloadOptions<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
@@ -124,6 +125,6 @@ export declare class FileDownload extends HttpCrawler<FileDownloadCrawlingContex
|
|
|
124
125
|
* ```
|
|
125
126
|
*/
|
|
126
127
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
127
|
-
export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("
|
|
128
|
+
export declare function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("../index").RouterHandler<Context>;
|
|
128
129
|
export {};
|
|
129
130
|
//# sourceMappingURL=file-download.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,OAAO,KAAK,EACR,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,YAAY,EACf,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,WAAW,EAAU,MAAM,UAAU,CAAC;AAE/C,MAAM,MAAM,wBAAwB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElE,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACnC,2BAA2B,EAC3B,MAAM,GAAG,kBAAkB,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CACvE,GAAG;IACA,MAAM,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAE5C,OAAO,KAAK,EACR,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,2BAA2B,EAC3B,gBAAgB,EAChB,cAAc,EACd,YAAY,EACf,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,WAAW,EAAU,MAAM,UAAU,CAAC;AAE/C,MAAM,MAAM,wBAAwB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAElE,MAAM,MAAM,oBAAoB,GAAG,IAAI,CACnC,2BAA2B,EAC3B,MAAM,GAAG,kBAAkB,GAAG,MAAM,GAAG,aAAa,GAAG,aAAa,CACvE,GAAG;IACA,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,MAAM,IAAI,CAAC;CAC7B,CAAC;AAEF,KAAK,aAAa,GAAG,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,MAAM,mBAAmB,CAC3B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IAE/B,CAAC,IAAI,CAAC,kBAAkB,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,GAAG;IAC3F,cAAc,CAAC,EAAE,KAAK,CAAC;IACvB,aAAa,CAAC,EAAE,aAAa,CAAC;CACjC,CAAC,GACF,CAAC,IAAI,CAAC,kBAAkB,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,GAAG;IAC3F,cAAc,EAAE,0BAA0B,CAAC;IAC3C,aAAa,CAAC,EAAE,KAAK,CAAC;CACzB,CAAC,CAAC;AAET,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEtE,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC;CAAG;AAE1E,MAAM,MAAM,0BAA0B,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEpE;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAsB,EACtB,qBAAqB,EAAE,uBAA6B,GACvD,EAAE;IACC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,SAAS,CA4BZ;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAsB,GACzB,EAAE;IACC,mBAAmB,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,SAAS,CAoBZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,qBAAa,YAAa,SAAQ,WAAW,CAAC,2BAA2B,CAAC;IACtE,OAAO,CAAC,aAAa,CAAC,CAAgB;gBAE1B,OAAO,GAAE,mBAAwB;cAqBpB,kBAAkB,CAAC,OAAO,EAAE,2BAA2B;YAQlE,oBAAoB;CAsFrC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,6CAEzC"}
|
|
@@ -146,6 +146,14 @@ class FileDownload extends index_1.HttpCrawler {
|
|
|
146
146
|
proxyUrl: context.proxyInfo?.url,
|
|
147
147
|
});
|
|
148
148
|
let pollingInterval;
|
|
149
|
+
let abortedByHandler = false;
|
|
150
|
+
const abortDownload = () => {
|
|
151
|
+
if (abortedByHandler || !response.stream.readable) {
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
abortedByHandler = true;
|
|
155
|
+
response.stream.destroy();
|
|
156
|
+
};
|
|
149
157
|
const cleanUp = () => {
|
|
150
158
|
clearInterval(pollingInterval);
|
|
151
159
|
response.stream.destroy();
|
|
@@ -158,6 +166,10 @@ class FileDownload extends index_1.HttpCrawler {
|
|
|
158
166
|
}
|
|
159
167
|
}, 5000);
|
|
160
168
|
response.stream.on('error', async (error) => {
|
|
169
|
+
if (abortedByHandler) {
|
|
170
|
+
resolve();
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
161
173
|
cleanUp();
|
|
162
174
|
reject(error);
|
|
163
175
|
});
|
|
@@ -165,6 +177,7 @@ class FileDownload extends index_1.HttpCrawler {
|
|
|
165
177
|
try {
|
|
166
178
|
context.stream = response.stream;
|
|
167
179
|
context.response = response;
|
|
180
|
+
context.abortDownload = abortDownload;
|
|
168
181
|
streamHandlerResult = this.streamHandler(context);
|
|
169
182
|
}
|
|
170
183
|
catch (e) {
|
|
@@ -185,7 +198,13 @@ class FileDownload extends index_1.HttpCrawler {
|
|
|
185
198
|
resolve();
|
|
186
199
|
}
|
|
187
200
|
});
|
|
188
|
-
|
|
201
|
+
const streamFinishedPromise = (0, promises_1.finished)(response.stream).catch((error) => {
|
|
202
|
+
if (abortedByHandler) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
throw error;
|
|
206
|
+
});
|
|
207
|
+
await Promise.all([downloadPromise, streamFinishedPromise]);
|
|
189
208
|
cleanUp();
|
|
190
209
|
}
|
|
191
210
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":";;;AAuEA,gDAoCC;AASD,8CA0BC;AA6LD,4CAKC;AAhVD,6CAAwC;AACxC,mDAAgD;AAChD,2CAA4C;AAe5C,oCAA+C;AA6C/C;;;;;;;;GAQG;AACH,SAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,uBAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAIzB;IACG,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,uBAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAa,YAAa,SAAQ,mBAAwC;IAGtE,YAAY,UAA+B,EAAE;QACzC,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAClC,OAAO,OAAO,CAAC,aAAa,CAAC;QAE7B,IAAI,aAAa,EAAE,CAAC;YAChB,8DAA8D;YAC7D,OAAe,CAAC,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,IAAI,GAAG,CAAC;QACtF,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,CAAC;QAXX;;;;;WAA8B;QAalC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,oBAA2B,CAAC;QAC3D,CAAC;QAED,+EAA+E;QAC/E,oEAAoE;QACnE,IAAY,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxD,CAAC;IAEkB,KAAK,CAAC,kBAAkB,CAAC,OAAoC;QAC5E,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO,CAAC,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;QAC1C,CAAC;QAED,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,OAAoC;QACnE,MAAM,EACF,GAAG,EACH,OAAO,EAAE,EAAE,GAAG,EAAE,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC1C,GAAG;YACH,OAAO,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YAC/B,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG;SACnC,CAAC,CAAC;QAEH,IAAI,eAA2C,CAAC;QAChD,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAE7B,MAAM,aAAa,GAAG,GAAG,EAAE;YACvB,IAAI,gBAAgB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAChD,OAAO;YACX,CAAC;YAED,gBAAgB,GAAG,IAAI,CAAC;YACxB,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAG,EAAE;YACjB,aAAa,CAAC,eAAgB,CAAC,CAAC;YAChC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC,CAAC;QAEF,MAAM,eAAe,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1D,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC/B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,QAAQ,CAAC,gBAAgB,CAAC;gBAEzD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;oBAClB,GAAG,CAAC,KAAK,CAAC,cAAc,WAAW,aAAa,KAAK,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAY,EAAE,EAAE;gBAC/C,IAAI,gBAAgB,EAAE,CAAC;oBACnB,OAAO,EAAE,CAAC;oBACV,OAAO;gBACX,CAAC;gBAED,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;YAEH,IAAI,mBAAmB,CAAC;YAExB,IAAI,CAAC;gBACD,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACjC,OAAO,CAAC,QAAQ,GAAG,QAAe,CAAC;gBAClC,OAAe,CAAC,aAAa,GAAG,aAAa,CAAC;gBAC/C,mBAAmB,GAAG,IAAI,CAAC,aAAc,CAAC,OAAc,CAAC,CAAC;YAC9D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,CAAC,CAAC,CAAC;YACd,CAAC;YAED,IAAI,IAAA,iBAAS,EAAC,mBAAmB,CAAC,EAAE,CAAC;gBACjC,mBAAmB;qBACd,IAAI,CAAC,GAAG,EAAE;oBACP,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,CAAQ,EAAE,EAAE;oBAChB,OAAO,EAAE,CAAC;oBACV,MAAM,CAAC,CAAC,CAAC,CAAC;gBACd,CAAC,CAAC,CAAC;YACX,CAAC;iBAAM,CAAC;gBACJ,OAAO,EAAE,CAAC;YACd,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,qBAAqB,GAAG,IAAA,mBAAQ,EAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpE,IAAI,gBAAgB,EAAE,CAAC;gBACnB,OAAO;YACX,CAAC;YAED,MAAM,KAAK,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAAC,CAAC;QAE5D,OAAO,EAAE,CAAC;IACd,CAAC;CACJ;AAtHD,oCAsHC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAgB,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,cAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "3.16.1-beta.
|
|
3
|
+
"version": "3.16.1-beta.41",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,13 +55,13 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@apify/timeout": "^0.3.0",
|
|
57
57
|
"@apify/utilities": "^2.7.10",
|
|
58
|
-
"@crawlee/basic": "3.16.1-beta.
|
|
59
|
-
"@crawlee/types": "3.16.1-beta.
|
|
60
|
-
"@crawlee/utils": "3.16.1-beta.
|
|
58
|
+
"@crawlee/basic": "3.16.1-beta.41",
|
|
59
|
+
"@crawlee/types": "3.16.1-beta.41",
|
|
60
|
+
"@crawlee/utils": "3.16.1-beta.41",
|
|
61
61
|
"@types/content-type": "^1.1.5",
|
|
62
62
|
"cheerio": "1.0.0-rc.12",
|
|
63
63
|
"content-type": "^1.0.4",
|
|
64
|
-
"got-scraping": "^4.
|
|
64
|
+
"got-scraping": "^4.2.1",
|
|
65
65
|
"iconv-lite": "^0.7.0",
|
|
66
66
|
"mime-types": "^2.1.35",
|
|
67
67
|
"ow": "^0.28.1",
|
|
@@ -75,5 +75,5 @@
|
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
},
|
|
78
|
-
"gitHead": "
|
|
78
|
+
"gitHead": "9af173bb3626e298917f66d8c3088a923f1a3e87"
|
|
79
79
|
}
|