@crawlee/core 4.0.0-beta.31 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -40,6 +40,16 @@ export declare abstract class ContextPipeline<TContextBase, TCrawlingContext ext
|
|
|
40
40
|
* @returns A new ContextPipeline instance with the added middleware
|
|
41
41
|
*/
|
|
42
42
|
abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
|
|
43
|
+
/**
|
|
44
|
+
* Chains another pipeline onto this one. The other pipeline's base context must match
|
|
45
|
+
* this pipeline's output context. Returns a new pipeline that runs this pipeline's
|
|
46
|
+
* middlewares first, then the other pipeline's middlewares.
|
|
47
|
+
*
|
|
48
|
+
* @template TFinalContext - The final context type after the chained pipeline's transformations
|
|
49
|
+
* @param other - The pipeline to append after this one
|
|
50
|
+
* @returns A new ContextPipeline combining both pipelines' middlewares
|
|
51
|
+
*/
|
|
52
|
+
abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
|
|
43
53
|
/**
|
|
44
54
|
* Executes the middleware pipeline and passes the final context to a consumer function.
|
|
45
55
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context_pipeline.d.ts","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"context_pipeline.d.ts","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAYhD;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB,CAAC,gBAAgB,EAAE,yBAAyB;IAC1E,6DAA6D;IAC7D,MAAM,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,SAAS,CAAC,yBAAyB,CAAC,CAAC;IAC5E,4EAA4E;IAC5E,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,GAAG,yBAAyB,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;CACzG;AAED;;;;;;;;;GASG;AACH,8BAAsB,eAAe,CAAC,YAAY,EAAE,gBAAgB,SAAS,YAAY;IACrF;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,YAAY,KAAK,eAAe,CAAC,YAAY,EAAE,YAAY,CAAC;IAI1E;;;;;;;;;OASG;IACH,QAAQ,CAAC,OAAO,CAAC,yBAAyB,EACtC,UAAU,EAAE,iBAAiB,CAAC,gBAAgB,EAAE,yBAAyB,CAAC,GAC3E,eAAe,CAAC,YAAY,EAAE,gBAAgB,GAAG,yBAAyB,CAAC;IAE9E;;;;;;;;OAQG;IACH,QAAQ,CAAC,KAAK,CAAC,aAAa,SAAS,gBAAgB,EACjD,KAAK,EAAE,eAAe,CAAC,gBAAgB,EAAE,aAAa,CAAC,GACxD,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC;IAE/C;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,IAAI,CACT,eAAe,EAAE,YAAY,EAC7B,oBAAoB,EAAE,CAAC,YAAY,EAAE,gBAAgB,KAAK,SAAS,CAAC,OAAO,CAAC,GAC7E,OAAO,CAAC,IAAI,CAAC;CACnB"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import log from '@apify/log';
|
|
1
2
|
import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
|
|
2
3
|
/**
|
|
3
4
|
* Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
|
|
@@ -38,6 +39,14 @@ class ContextPipelineImpl extends ContextPipeline {
|
|
|
38
39
|
compose(middleware) {
|
|
39
40
|
return new ContextPipelineImpl(middleware, this);
|
|
40
41
|
}
|
|
42
|
+
chain(other) {
|
|
43
|
+
const otherMiddlewares = Array.from(other.middlewareChain()).reverse();
|
|
44
|
+
let result = this;
|
|
45
|
+
for (const middleware of otherMiddlewares) {
|
|
46
|
+
result = result.compose(middleware);
|
|
47
|
+
}
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
41
50
|
*middlewareChain() {
|
|
42
51
|
let step = this;
|
|
43
52
|
while (step !== undefined) {
|
|
@@ -56,7 +65,20 @@ class ContextPipelineImpl extends ContextPipeline {
|
|
|
56
65
|
for (const { action, cleanup } of middlewares) {
|
|
57
66
|
try {
|
|
58
67
|
const contextExtension = await action(crawlingContext);
|
|
59
|
-
|
|
68
|
+
const extensionNames = [
|
|
69
|
+
...Object.getOwnPropertyNames(contextExtension),
|
|
70
|
+
...Object.getOwnPropertySymbols(contextExtension),
|
|
71
|
+
];
|
|
72
|
+
for (const key of extensionNames) {
|
|
73
|
+
try {
|
|
74
|
+
if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
|
|
75
|
+
Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
log.debug(`Context pipeline failed to define property ${key.toString()}:`, error);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
60
82
|
if (cleanup) {
|
|
61
83
|
cleanupStack.push(cleanup);
|
|
62
84
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context_pipeline.js","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,2BAA2B,EAC3B,kCAAkC,EAClC,+BAA+B,EAC/B,mBAAmB,EACnB,YAAY,GACf,MAAM,cAAc,CAAC;AAetB;;;;;;;;;GASG;AACH,MAAM,OAAgB,eAAe;IACjC;;;;;OAKG;IACH,MAAM,CAAC,MAAM;QACT,OAAO,IAAI,mBAAmB,CAA6B,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;
|
|
1
|
+
{"version":3,"file":"context_pipeline.js","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EACH,2BAA2B,EAC3B,kCAAkC,EAClC,+BAA+B,EAC/B,mBAAmB,EACnB,YAAY,GACf,MAAM,cAAc,CAAC;AAetB;;;;;;;;;GASG;AACH,MAAM,OAAgB,eAAe;IACjC;;;;;OAKG;IACH,MAAM,CAAC,MAAM;QACT,OAAO,IAAI,mBAAmB,CAA6B,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;CAiDJ;AAED;;;GAGG;AACH,MAAM,mBAAyE,SAAQ,eAGtF;IAEe;IACA;IAFZ,YACY,UAA6D,EAC7D,MAAwD;QAEhE,KAAK,EAAE,CAAC;QAHA,eAAU,GAAV,UAAU,CAAmD;QAC7D,WAAM,GAAN,MAAM,CAAkD;IAGpE,CAAC;IAED;;OAEG;IACH,OAAO,CACH,UAA0E;QAE1E,OAAO,IAAI,mBAAmB,CAC1B,UAAiB,EACjB,IAAW,CACd,CAAC;IACN,CAAC;IAED,KAAK,CACD,KAAuD;QAEvD,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAC9B,KAAa,CAAC,eAAe,EAA2C,CAC5E,CAAC,OAAO,EAAE,CAAC;QAEZ,IAAI,MAAM,GAAuC,IAAW,CAAC;QAC7D,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE,CAAC;YACxC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,UAAiB,CAAC,CAAC;QAC/C,CAAC;QAED,OAAO,MAAsD,CAAC;IAClE,CAAC;IAEO,CAAC,eAAe;QACpB,IAAI,IAAI,GAAgE,IAAW,CAAC;QAEpF,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CACN,eAA6B,EAC7B,oBAA0E;QAE1E,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QACjE,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,IAAI,iBAAsC,CAAC;QAE3C,IAAI,CAAC;YACD,KAAK,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACD,MAAM,gBAAgB,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;oBAEvD,MAAM,cAAc,GAAG;wBACnB,GAAG,MAAM,CAAC,mBAAmB,CAAC,gBAAgB,CAAC;wBAC/C,GAAG,MAAM,CAAC,qBAAqB,CAAC,gBAAgB,CAAC;qBACpD,CAAC;oBAEF,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;wBAC/B,IAAI,CAAC;4BACD,IAAI,MAAM,CAAC,wBAAwB,CAAC,eAAe,EAAE,GAAG,CAAC,EAAE,YAAY,KAAK,KAAK,EAAE,CAAC;gCAChF,MAAM,CAAC,cAAc,CACjB,eAAe,EACf,GAAG,EACH,MAAM,CAAC,wBAAwB,CAAC,gBAAgB,EAAE,GAAG,CAAE,CAC1D,CAAC;4BACN,CAAC;wBACL,CAAC;wBAAC,OAAO,KAAU,EAAE,CAAC;4BAClB,GAAG,CAAC,KAAK,CAAC,8CAA8C,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;wBACtF,CAAC;oBACL,CAAC;oBAED,IAAI,OAAO,EAAE,CAAC;wBACV,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC/B,CAAC;gBACL,CAAC;gBAAC,OAAO,SAAkB,EAAE,CAAC;oBAC1B,IAAI,SAAS,YAAY,YAAY,EAAE,CAAC;wBACpC,MAAM,SAAS,CAAC,CAAC,qCAAqC;oBAC1D,CAAC;oBACD,IAAI,SAAS,YAAY,+BAA+B,EAAE,CAAC;wBACvD,MAAM,SAAS,CAAC;oBACpB,CAAC;oBAED,MAAM,IAAI,kCAAkC,CAAC,SAAS,CAAC,CAAC;gBAC5D,CAAC;YACL,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,oBAAoB,CAAC,eAAmC,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,SAAkB,EAAE,CAAC;gBAC1B,IAAI,SAAS,YAAY,YAAY,EAAE,CAAC;oBACpC,iBAAiB,GAAG,SAAS,CAAC;oBAC9B,MAAM,SAAS,CAAC,CAAC,qCAAqC;gBAC1D,CAAC;gBACD,iBAAiB,GAAG,SAAS,CAAC;gBAC9B,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAC7C,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC;gBACD,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC3C,MAAM,OAAO,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAAC,OAAO,SAAkB,EAAE,CAAC;gBAC1B,6CAA6C;gBAC7C,MAAM,IAAI,2BAA2B,CAAC,SAAS,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;IACL,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.32",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/memory-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/memory-storage": "4.0.0-beta.32",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.32",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.32",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
61
61
|
"csv-stringify": "^6.5.2",
|
|
@@ -76,5 +76,5 @@
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
},
|
|
79
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "86c266f094f0ac2bcf8c376512e16b677279a614"
|
|
80
80
|
}
|