@crawlee/core 3.0.1-beta.2 → 3.0.1-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/snapshotter.d.ts +8 -8
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/configuration.d.ts +3 -29
- package/configuration.d.ts.map +1 -1
- package/configuration.js +2 -1
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +43 -0
- package/cookie_utils.d.ts.map +1 -0
- package/cookie_utils.js +118 -0
- package/cookie_utils.js.map +1 -0
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -47
- package/crawlers/crawler_utils.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +5 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +10 -14
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/index.mjs +5 -2
- package/package.json +4 -5
- package/router.d.ts.map +1 -1
- package/router.js +2 -1
- package/router.js.map +1 -1
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +3 -19
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +6 -52
- package/session_pool/session.js.map +1 -1
- package/storages/request_list.d.ts +2 -2
- package/storages/request_list.d.ts.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/session_pool/session_utils.d.ts +0 -11
- package/session_pool/session_utils.d.ts.map +0 -1
- package/session_pool/session_utils.js +0 -22
- package/session_pool/session_utils.js.map +0 -1
|
@@ -47,12 +47,12 @@ export interface SnapshotterOptions {
|
|
|
47
47
|
/** @internal */
|
|
48
48
|
config?: Configuration;
|
|
49
49
|
}
|
|
50
|
-
|
|
50
|
+
interface MemorySnapshot {
|
|
51
51
|
createdAt: Date;
|
|
52
52
|
isOverloaded: boolean;
|
|
53
53
|
usedBytes?: number;
|
|
54
|
-
}
|
|
55
|
-
|
|
54
|
+
}
|
|
55
|
+
interface CpuSnapshot {
|
|
56
56
|
createdAt: Date;
|
|
57
57
|
isOverloaded: boolean;
|
|
58
58
|
usedRatio: number;
|
|
@@ -60,17 +60,17 @@ declare type CpuSnapshot = {
|
|
|
60
60
|
idle: number;
|
|
61
61
|
total: number;
|
|
62
62
|
};
|
|
63
|
-
}
|
|
64
|
-
|
|
63
|
+
}
|
|
64
|
+
interface EventLoopSnapshot {
|
|
65
65
|
createdAt: Date;
|
|
66
66
|
isOverloaded: boolean;
|
|
67
67
|
exceededMillis: number;
|
|
68
|
-
}
|
|
69
|
-
|
|
68
|
+
}
|
|
69
|
+
interface ClientSnapshot {
|
|
70
70
|
createdAt: Date;
|
|
71
71
|
isOverloaded: boolean;
|
|
72
72
|
rateLimitErrorCount: number;
|
|
73
|
-
}
|
|
73
|
+
}
|
|
74
74
|
/**
|
|
75
75
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
76
76
|
* as either overloaded or not during the last interval. Keeps a history of the snapshots.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAIzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAO5D,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAIzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAO5D,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,cAAc;IAAG,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE;AACvF,UAAU,WAAW;IAAG,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE;AAC5H,UAAU,iBAAiB;IAAG,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE;AAC9F,UAAU,cAAc;IAAG,SAAS,EAAE,IAAI,CAAC;IAAC,YAAY,EAAE,OAAO,CAAC;IAAC,mBAAmB,EAAE,MAAM,CAAA;CAAE;AAEhG;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA0C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAchD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IAmBvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EAAE,GAAG,EAAE,IAAI;IAUzH;;OAEG;YACW,cAAc;CAG/B"}
|
package/configuration.d.ts
CHANGED
|
@@ -51,7 +51,7 @@ export interface ConfigurationOptions {
|
|
|
51
51
|
* ---|---|---
|
|
52
52
|
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
53
53
|
* `logLevel` | `CRAWLEE_LOG_LEVEL` | -
|
|
54
|
-
* `headless` | `CRAWLEE_HEADLESS` |
|
|
54
|
+
* `headless` | `CRAWLEE_HEADLESS` | `true`
|
|
55
55
|
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
56
56
|
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
57
57
|
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
@@ -70,36 +70,10 @@ export declare class Configuration {
|
|
|
70
70
|
/**
|
|
71
71
|
* Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
|
|
72
72
|
*/
|
|
73
|
-
protected static ENV_MAP:
|
|
74
|
-
CRAWLEE_AVAILABLE_MEMORY_RATIO: string;
|
|
75
|
-
CRAWLEE_PURGE_ON_START: string;
|
|
76
|
-
CRAWLEE_MEMORY_MBYTES: string;
|
|
77
|
-
CRAWLEE_DEFAULT_DATASET_ID: string;
|
|
78
|
-
CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: string;
|
|
79
|
-
CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: string;
|
|
80
|
-
CRAWLEE_INPUT_KEY: string;
|
|
81
|
-
CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: string;
|
|
82
|
-
CRAWLEE_HEADLESS: string;
|
|
83
|
-
CRAWLEE_XVFB: string;
|
|
84
|
-
CRAWLEE_CHROME_EXECUTABLE_PATH: string;
|
|
85
|
-
CRAWLEE_DEFAULT_BROWSER_PATH: string;
|
|
86
|
-
CRAWLEE_DISABLE_BROWSER_SANDBOX: string;
|
|
87
|
-
CRAWLEE_LOG_LEVEL: string;
|
|
88
|
-
};
|
|
73
|
+
protected static ENV_MAP: Dictionary;
|
|
89
74
|
protected static BOOLEAN_VARS: string[];
|
|
90
75
|
protected static INTEGER_VARS: string[];
|
|
91
|
-
protected static DEFAULTS:
|
|
92
|
-
defaultKeyValueStoreId: string;
|
|
93
|
-
defaultDatasetId: string;
|
|
94
|
-
defaultRequestQueueId: string;
|
|
95
|
-
inputKey: string;
|
|
96
|
-
maxUsedCpuRatio: number;
|
|
97
|
-
availableMemoryRatio: number;
|
|
98
|
-
storageClientOptions: {};
|
|
99
|
-
purgeOnStart: boolean;
|
|
100
|
-
persistStateIntervalMillis: number;
|
|
101
|
-
systemInfoIntervalMillis: number;
|
|
102
|
-
};
|
|
76
|
+
protected static DEFAULTS: Dictionary;
|
|
103
77
|
/**
|
|
104
78
|
* Provides access to the current-instance-scoped Configuration without passing it around in parameters.
|
|
105
79
|
* @internal
|
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;CAClD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAelC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAAiE;IAE9F,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAYnC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAe9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAatE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
|
package/configuration.js
CHANGED
|
@@ -35,7 +35,7 @@ const events_1 = require("./events");
|
|
|
35
35
|
* ---|---|---
|
|
36
36
|
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
37
37
|
* `logLevel` | `CRAWLEE_LOG_LEVEL` | -
|
|
38
|
-
* `headless` | `CRAWLEE_HEADLESS` |
|
|
38
|
+
* `headless` | `CRAWLEE_HEADLESS` | `true`
|
|
39
39
|
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
40
40
|
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
41
41
|
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
@@ -261,6 +261,7 @@ Object.defineProperty(Configuration, "DEFAULTS", {
|
|
|
261
261
|
availableMemoryRatio: 0.25,
|
|
262
262
|
storageClientOptions: {},
|
|
263
263
|
purgeOnStart: true,
|
|
264
|
+
headless: true,
|
|
264
265
|
persistStateIntervalMillis: 60000,
|
|
265
266
|
systemInfoIntervalMillis: 60000,
|
|
266
267
|
}
|
package/configuration.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAwB7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAa,aAAa;
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAwB7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,MAAa,aAAa;IAmDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACvH,aAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;SACnC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AArOL,sCAsOC;AArOG;;GAEG;AACH;;;;WAAuC;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;KAChC;GAAC;AAEF;;;;WAAgC,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC;GAAC;AAE9F;;;;WAAgC,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE3G;;;;WAAwC;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import type { BrowserLikeResponse, Dictionary, Cookie as CookieObject } from '@crawlee/types';
|
|
3
|
+
import type { IncomingMessage } from 'node:http';
|
|
4
|
+
import { Cookie } from 'tough-cookie';
|
|
5
|
+
/**
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | {
|
|
9
|
+
headers: Dictionary<string | string[]>;
|
|
10
|
+
}): Cookie[];
|
|
11
|
+
/**
|
|
12
|
+
* Calculate cookie expiration date
|
|
13
|
+
* @param maxAgeSecs
|
|
14
|
+
* @returns Calculated date by session max age seconds.
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export declare function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
|
|
18
|
+
/**
|
|
19
|
+
* Transforms tough-cookie to puppeteer cookie.
|
|
20
|
+
* @param toughCookie Cookie from CookieJar
|
|
21
|
+
* @return Cookie compatible with browser pool
|
|
22
|
+
* @internal
|
|
23
|
+
*/
|
|
24
|
+
export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject;
|
|
25
|
+
/**
|
|
26
|
+
* Transforms browser-pool cookie to tough-cookie.
|
|
27
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
28
|
+
* @internal
|
|
29
|
+
*/
|
|
30
|
+
export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number): Cookie;
|
|
31
|
+
/**
|
|
32
|
+
* @internal
|
|
33
|
+
* @param cookieString The cookie string to attempt parsing
|
|
34
|
+
* @returns Browser pool compatible cookie, or null if cookie cannot be parsed
|
|
35
|
+
*/
|
|
36
|
+
export declare function cookieStringToToughCookie(cookieString: string): CookieObject | null;
|
|
37
|
+
/**
|
|
38
|
+
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
39
|
+
* if we see two cookies with same keys but different casing.
|
|
40
|
+
* @internal
|
|
41
|
+
*/
|
|
42
|
+
export declare function mergeCookies(url: string, sourceCookies: string[]): string;
|
|
43
|
+
//# sourceMappingURL=cookie_utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAIjD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAW7I;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAYhF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAgB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CA4BzE"}
|
package/cookie_utils.js
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mergeCookies = exports.cookieStringToToughCookie = exports.browserPoolCookieToToughCookie = exports.toughCookieToBrowserPoolCookie = exports.getDefaultCookieExpirationDate = exports.getCookiesFromResponse = void 0;
|
|
4
|
+
const tough_cookie_1 = require("tough-cookie");
|
|
5
|
+
const errors_1 = require("./session_pool/errors");
|
|
6
|
+
const log_1 = require("./log");
|
|
7
|
+
/**
|
|
8
|
+
* @internal
|
|
9
|
+
*/
|
|
10
|
+
function getCookiesFromResponse(response) {
|
|
11
|
+
const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
|
|
12
|
+
const cookieHeader = headers['set-cookie'] || '';
|
|
13
|
+
try {
|
|
14
|
+
return Array.isArray(cookieHeader)
|
|
15
|
+
? cookieHeader.map((cookie) => tough_cookie_1.Cookie.parse(cookie))
|
|
16
|
+
: [tough_cookie_1.Cookie.parse(cookieHeader)];
|
|
17
|
+
}
|
|
18
|
+
catch (e) {
|
|
19
|
+
throw new errors_1.CookieParseError(cookieHeader);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
exports.getCookiesFromResponse = getCookiesFromResponse;
|
|
23
|
+
/**
|
|
24
|
+
* Calculate cookie expiration date
|
|
25
|
+
* @param maxAgeSecs
|
|
26
|
+
* @returns Calculated date by session max age seconds.
|
|
27
|
+
* @internal
|
|
28
|
+
*/
|
|
29
|
+
function getDefaultCookieExpirationDate(maxAgeSecs) {
|
|
30
|
+
return new Date(Date.now() + (maxAgeSecs * 1000));
|
|
31
|
+
}
|
|
32
|
+
exports.getDefaultCookieExpirationDate = getDefaultCookieExpirationDate;
|
|
33
|
+
/**
|
|
34
|
+
* Transforms tough-cookie to puppeteer cookie.
|
|
35
|
+
* @param toughCookie Cookie from CookieJar
|
|
36
|
+
* @return Cookie compatible with browser pool
|
|
37
|
+
* @internal
|
|
38
|
+
*/
|
|
39
|
+
function toughCookieToBrowserPoolCookie(toughCookie) {
|
|
40
|
+
return {
|
|
41
|
+
name: toughCookie.key,
|
|
42
|
+
value: toughCookie.value,
|
|
43
|
+
// Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
|
|
44
|
+
// If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
|
|
45
|
+
expires: toughCookie.expires === 'Infinity' ? undefined : new Date(toughCookie.expires).getTime() / 1000,
|
|
46
|
+
domain: toughCookie.domain ?? undefined,
|
|
47
|
+
path: toughCookie.path ?? undefined,
|
|
48
|
+
secure: toughCookie.secure,
|
|
49
|
+
httpOnly: toughCookie.httpOnly,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
exports.toughCookieToBrowserPoolCookie = toughCookieToBrowserPoolCookie;
|
|
53
|
+
/**
|
|
54
|
+
* Transforms browser-pool cookie to tough-cookie.
|
|
55
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
56
|
+
* @internal
|
|
57
|
+
*/
|
|
58
|
+
function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
|
|
59
|
+
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
|
|
60
|
+
const expires = isExpiresValid ? new Date(cookieObject.expires * 1000) : getDefaultCookieExpirationDate(maxAgeSecs);
|
|
61
|
+
const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
|
|
62
|
+
? cookieObject.domain.slice(1)
|
|
63
|
+
: cookieObject.domain;
|
|
64
|
+
return new tough_cookie_1.Cookie({
|
|
65
|
+
key: cookieObject.name,
|
|
66
|
+
value: cookieObject.value,
|
|
67
|
+
expires,
|
|
68
|
+
domain,
|
|
69
|
+
path: cookieObject.path,
|
|
70
|
+
secure: cookieObject.secure,
|
|
71
|
+
httpOnly: cookieObject.httpOnly,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
exports.browserPoolCookieToToughCookie = browserPoolCookieToToughCookie;
|
|
75
|
+
/**
|
|
76
|
+
* @internal
|
|
77
|
+
* @param cookieString The cookie string to attempt parsing
|
|
78
|
+
* @returns Browser pool compatible cookie, or null if cookie cannot be parsed
|
|
79
|
+
*/
|
|
80
|
+
function cookieStringToToughCookie(cookieString) {
|
|
81
|
+
const parsed = tough_cookie_1.Cookie.parse(cookieString);
|
|
82
|
+
if (parsed) {
|
|
83
|
+
return toughCookieToBrowserPoolCookie(parsed);
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
exports.cookieStringToToughCookie = cookieStringToToughCookie;
|
|
88
|
+
/**
|
|
89
|
+
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
90
|
+
* if we see two cookies with same keys but different casing.
|
|
91
|
+
* @internal
|
|
92
|
+
*/
|
|
93
|
+
function mergeCookies(url, sourceCookies) {
|
|
94
|
+
const jar = new tough_cookie_1.CookieJar();
|
|
95
|
+
// ignore empty cookies
|
|
96
|
+
for (const sourceCookieString of sourceCookies) {
|
|
97
|
+
// ignore empty cookies
|
|
98
|
+
if (!sourceCookieString)
|
|
99
|
+
continue;
|
|
100
|
+
const cookies = sourceCookieString.split(/ *; */);
|
|
101
|
+
for (const cookieString of cookies) {
|
|
102
|
+
// ignore extra spaces
|
|
103
|
+
if (!cookieString)
|
|
104
|
+
continue;
|
|
105
|
+
const cookie = tough_cookie_1.Cookie.parse(cookieString);
|
|
106
|
+
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
|
|
107
|
+
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
|
|
108
|
+
});
|
|
109
|
+
if (similarKeyCookie) {
|
|
110
|
+
log_1.log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
|
|
111
|
+
}
|
|
112
|
+
jar.setCookieSync(cookie, url);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return jar.getCookieStringSync(url);
|
|
116
|
+
}
|
|
117
|
+
exports.mergeCookies = mergeCookies;
|
|
118
|
+
//# sourceMappingURL=cookie_utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":";;;AAEA,+CAAiD;AACjD,kDAAyD;AACzD,+BAA4B;AAE5B;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAA4F;IAC/H,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI;QACA,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;KACvC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;KAC5C;AACL,CAAC;AAXD,wDAWC;AAED;;;;;GAKG;AACH,SAAgB,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;AACtD,CAAC;AAFD,wEAEC;AAED;;;;;GAKG;AACH,SAAgB,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EAAE,WAAW,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;QACxG,MAAM,EAAE,WAAW,CAAC,MAAM,IAAI,SAAS;QACvC,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAZD,wEAYC;AAED;;;;GAIG;AACH,SAAgB,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACrH,MAAM,MAAM,GAAG,OAAO,YAAY,CAAC,MAAM,KAAK,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;QACzF,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAE1B,OAAO,IAAI,qBAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;KAClC,CAAC,CAAC;AACP,CAAC;AAhBD,wEAgBC;AAED;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE;QACR,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;KACjD;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AARD,8DAQC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE;QAC5C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE;YAChC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE;gBAClB,SAAG,CAAC,UAAU,CAAC,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC;aAC1H;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC;KACJ;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AA5BD,oCA4BC"}
|
|
@@ -7,14 +7,4 @@ export declare function handleRequestTimeout({ session, errorMessage }: {
|
|
|
7
7
|
session?: Session;
|
|
8
8
|
errorMessage: string;
|
|
9
9
|
}): void;
|
|
10
|
-
/**
|
|
11
|
-
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
12
|
-
* if we see two cookies with same keys but different casing.
|
|
13
|
-
* @internal
|
|
14
|
-
*/
|
|
15
|
-
export declare function mergeCookies(url: string, sourceCookies: string[]): string;
|
|
16
|
-
/**
|
|
17
|
-
* @internal
|
|
18
|
-
*/
|
|
19
|
-
export declare function diffCookies(url: string, cookieString1?: string, cookieString2?: string): string;
|
|
20
10
|
//# sourceMappingURL=crawler_utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G"}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.handleRequestTimeout = void 0;
|
|
4
4
|
const timeout_1 = require("@apify/timeout");
|
|
5
|
-
const tough_cookie_1 = require("tough-cookie");
|
|
6
|
-
const log_1 = require("../log");
|
|
7
5
|
/**
|
|
8
6
|
* Handles timeout request
|
|
9
7
|
* @internal
|
|
@@ -15,48 +13,4 @@ function handleRequestTimeout({ session, errorMessage }) {
|
|
|
15
13
|
throw new timeout_1.TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
|
|
16
14
|
}
|
|
17
15
|
exports.handleRequestTimeout = handleRequestTimeout;
|
|
18
|
-
/**
|
|
19
|
-
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
20
|
-
* if we see two cookies with same keys but different casing.
|
|
21
|
-
* @internal
|
|
22
|
-
*/
|
|
23
|
-
function mergeCookies(url, sourceCookies) {
|
|
24
|
-
const jar = new tough_cookie_1.CookieJar();
|
|
25
|
-
// ignore empty cookies
|
|
26
|
-
for (const sourceCookieString of sourceCookies.filter((c) => c)) {
|
|
27
|
-
const cookies = sourceCookieString.split(/ *; */).filter((c) => c); // ignore extra spaces
|
|
28
|
-
for (const cookieString of cookies) {
|
|
29
|
-
const cookie = tough_cookie_1.Cookie.parse(cookieString);
|
|
30
|
-
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
|
|
31
|
-
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
|
|
32
|
-
});
|
|
33
|
-
if (similarKeyCookie) {
|
|
34
|
-
log_1.log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
|
|
35
|
-
}
|
|
36
|
-
jar.setCookieSync(cookie, url);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
return jar.getCookieStringSync(url);
|
|
40
|
-
}
|
|
41
|
-
exports.mergeCookies = mergeCookies;
|
|
42
|
-
/**
|
|
43
|
-
* @internal
|
|
44
|
-
*/
|
|
45
|
-
function diffCookies(url, cookieString1 = '', cookieString2 = '') {
|
|
46
|
-
if (cookieString1 === cookieString2 || !cookieString2) {
|
|
47
|
-
return '';
|
|
48
|
-
}
|
|
49
|
-
if (!cookieString1) {
|
|
50
|
-
return cookieString2;
|
|
51
|
-
}
|
|
52
|
-
const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
|
|
53
|
-
const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
|
|
54
|
-
const added = cookies2.filter((newCookie) => {
|
|
55
|
-
return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
|
|
56
|
-
});
|
|
57
|
-
const jar = new tough_cookie_1.CookieJar();
|
|
58
|
-
added.forEach((cookie) => jar.setCookieSync(cookie, url));
|
|
59
|
-
return jar.getCookieStringSync(url);
|
|
60
|
-
}
|
|
61
|
-
exports.diffCookies = diffCookies;
|
|
62
16
|
//# sourceMappingURL=crawler_utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,4CAA8C;
|
|
1
|
+
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,4CAA8C;AAG9C;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,sBAAY,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AACjF,CAAC;AALD,oDAKC"}
|
|
@@ -139,8 +139,12 @@ export declare enum EnqueueStrategy {
|
|
|
139
139
|
export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
|
140
140
|
/**
|
|
141
141
|
* @internal
|
|
142
|
+
* This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
|
|
143
|
+
* - If a user provides a base url, we always return it
|
|
144
|
+
* - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original request domain, or a redirected one
|
|
145
|
+
* - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
|
|
142
146
|
*/
|
|
143
|
-
export declare function
|
|
147
|
+
export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string;
|
|
144
148
|
/**
|
|
145
149
|
* @internal
|
|
146
150
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAQ3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAuGhG;AAED;;;;;;GAMG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,UAiBhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.resolveBaseUrlForEnqueueLinksFiltering = exports.enqueueLinks = exports.EnqueueStrategy = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const tldts_1 = require("tldts");
|
|
6
6
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
@@ -124,8 +124,12 @@ async function enqueueLinks(options) {
|
|
|
124
124
|
exports.enqueueLinks = enqueueLinks;
|
|
125
125
|
/**
|
|
126
126
|
* @internal
|
|
127
|
+
* This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
|
|
128
|
+
* - If a user provides a base url, we always return it
|
|
129
|
+
* - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original request domain, or a redirected one
|
|
130
|
+
* - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
|
|
127
131
|
*/
|
|
128
|
-
function
|
|
132
|
+
function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
|
|
129
133
|
// User provided base url takes priority
|
|
130
134
|
if (userProvidedBaseUrl) {
|
|
131
135
|
return userProvidedBaseUrl;
|
|
@@ -134,19 +138,11 @@ function resolveBaseUrl({ enqueueStrategy, finalRequestUrl, originalRequestUrl,
|
|
|
134
138
|
const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
|
|
135
139
|
// We can assume users want to go off the domain in this case
|
|
136
140
|
if (enqueueStrategy === EnqueueStrategy.All) {
|
|
137
|
-
return finalUrlOrigin
|
|
141
|
+
return finalUrlOrigin;
|
|
138
142
|
}
|
|
139
|
-
//
|
|
140
|
-
|
|
141
|
-
const originalHostname = (0, tldts_1.getDomain)(originalUrlOrigin, { mixedInputs: false });
|
|
142
|
-
const finalHostname = (0, tldts_1.getDomain)(finalUrlOrigin, { mixedInputs: false });
|
|
143
|
-
if (originalHostname === finalHostname) {
|
|
144
|
-
return finalUrlOrigin;
|
|
145
|
-
}
|
|
146
|
-
return undefined;
|
|
147
|
-
}
|
|
148
|
-
// Always enqueue urls that are from the same origin
|
|
143
|
+
// Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
|
|
144
|
+
// before actually finding the urls
|
|
149
145
|
return originalUrlOrigin;
|
|
150
146
|
}
|
|
151
|
-
exports.
|
|
147
|
+
exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
|
|
152
148
|
//# sourceMappingURL=enqueue_links.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAMkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBACrD,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,iBAAiB,CAAC,IAAI,CAClB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,iBAAiB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBACxD;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,IAAI,QAAQ,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAvGD,oCAuGC;AAED;;;;;;GAMG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAtBD,wFAsBC"}
|
package/index.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export * from './serialization';
|
|
|
13
13
|
export * from './session_pool';
|
|
14
14
|
export * from './storages';
|
|
15
15
|
export * from './validators';
|
|
16
|
+
export * from './cookie_utils';
|
|
16
17
|
export { PseudoUrl } from '@apify/pseudo_url';
|
|
17
18
|
export { Dictionary, Awaitable, Constructor, StorageClient } from '@crawlee/types';
|
|
18
19
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js
CHANGED
|
@@ -17,6 +17,7 @@ tslib_1.__exportStar(require("./serialization"), exports);
|
|
|
17
17
|
tslib_1.__exportStar(require("./session_pool"), exports);
|
|
18
18
|
tslib_1.__exportStar(require("./storages"), exports);
|
|
19
19
|
tslib_1.__exportStar(require("./validators"), exports);
|
|
20
|
+
tslib_1.__exportStar(require("./cookie_utils"), exports);
|
|
20
21
|
var pseudo_url_1 = require("@apify/pseudo_url");
|
|
21
22
|
Object.defineProperty(exports, "PseudoUrl", { enumerable: true, get: function () { return pseudo_url_1.PseudoUrl; } });
|
|
22
23
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,mDAAyB;AACzB,wDAA8B;AAC9B,0DAAgC;AAChC,sDAA4B;AAC5B,qDAA2B;AAC3B,0DAAgC;AAChC,mDAAyB;AACzB,gDAAsB;AACtB,gEAAsC;AACtC,oDAA0B;AAC1B,mDAAyB;AACzB,0DAAgC;AAChC,yDAA+B;AAC/B,qDAA2B;AAC3B,uDAA6B;AAC7B,gDAA8C;AAArC,uGAAA,SAAS,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,mDAAyB;AACzB,wDAA8B;AAC9B,0DAAgC;AAChC,sDAA4B;AAC5B,qDAA2B;AAC3B,0DAAgC;AAChC,mDAAyB;AACzB,gDAAsB;AACtB,gEAAsC;AACtC,oDAA0B;AAC1B,mDAAyB;AACzB,0DAAgC;AAChC,yDAA+B;AAC/B,qDAA2B;AAC3B,uDAA6B;AAC7B,yDAA+B;AAC/B,gDAA8C;AAArC,uGAAA,SAAS,OAAA"}
|
package/index.mjs
CHANGED
|
@@ -41,26 +41,29 @@ export const Snapshotter = mod.Snapshotter;
|
|
|
41
41
|
export const Statistics = mod.Statistics;
|
|
42
42
|
export const StorageManager = mod.StorageManager;
|
|
43
43
|
export const SystemStatus = mod.SystemStatus;
|
|
44
|
+
export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
|
|
44
45
|
export const checkAndSerialize = mod.checkAndSerialize;
|
|
45
46
|
export const chunkBySize = mod.chunkBySize;
|
|
46
47
|
export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
|
|
47
48
|
export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
|
|
48
49
|
export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
|
|
50
|
+
export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
|
|
49
51
|
export const createDeserialize = mod.createDeserialize;
|
|
50
52
|
export const createRequestOptions = mod.createRequestOptions;
|
|
51
53
|
export const createRequests = mod.createRequests;
|
|
52
54
|
export const deserializeArray = mod.deserializeArray;
|
|
53
|
-
export const diffCookies = mod.diffCookies;
|
|
54
55
|
export const enqueueLinks = mod.enqueueLinks;
|
|
55
56
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
57
|
+
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
56
58
|
export const getRequestId = mod.getRequestId;
|
|
57
59
|
export const handleRequestTimeout = mod.handleRequestTimeout;
|
|
58
60
|
export const log = mod.log;
|
|
59
61
|
export const maybeStringify = mod.maybeStringify;
|
|
60
62
|
export const mergeCookies = mod.mergeCookies;
|
|
61
63
|
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
62
|
-
export const
|
|
64
|
+
export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
|
|
63
65
|
export const serializeArray = mod.serializeArray;
|
|
66
|
+
export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
|
|
64
67
|
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
|
65
68
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
66
69
|
export const validators = mod.validators;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.1-beta.
|
|
3
|
+
"version": "3.0.1-beta.22",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -58,10 +58,9 @@
|
|
|
58
58
|
"@apify/pseudo_url": "^2.0.0",
|
|
59
59
|
"@apify/timeout": "^0.3.0",
|
|
60
60
|
"@apify/utilities": "^2.0.0",
|
|
61
|
-
"@crawlee/
|
|
62
|
-
"@crawlee/
|
|
63
|
-
"@crawlee/
|
|
64
|
-
"@crawlee/utils": "^3.0.1-beta.2",
|
|
61
|
+
"@crawlee/memory-storage": "^3.0.1-beta.22",
|
|
62
|
+
"@crawlee/types": "^3.0.1-beta.22",
|
|
63
|
+
"@crawlee/utils": "^3.0.1-beta.22",
|
|
65
64
|
"@types/tough-cookie": "^4.0.2",
|
|
66
65
|
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
67
66
|
"fs-extra": "^10.1.0",
|
package/router.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAK5C,MAAM,WAAW,aAAa,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,CAAE,SAAQ,MAAM,CAAC,OAAO,CAAC;IACrG,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,MAAM,CAAC,OAAO,SAAS,eAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsE;IAE7F;;;OAGG;IACH,SAAS;IAET;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAK7E;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAK5D;;OAEG;IACH,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"router.d.ts","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAK5C,MAAM,WAAW,aAAa,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,CAAE,SAAQ,MAAM,CAAC,OAAO,CAAC;IACrG,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACnC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,qBAAa,MAAM,CAAC,OAAO,SAAS,eAAe;IAC/C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsE;IAE7F;;;OAGG;IACH,SAAS;IAET;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAK7E;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAK5D;;OAEG;IACH,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC;IAiBtE;;OAEG;IACH,OAAO,CAAC,QAAQ;IAShB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM,CAAC,OAAO,SAAS,eAAe,GAAG,eAAe,KAAK,aAAa,CAAC,OAAO,CAAC;CAkB7F"}
|
package/router.js
CHANGED
|
@@ -83,7 +83,8 @@ class Router {
|
|
|
83
83
|
return this.routes.get(defaultRoute);
|
|
84
84
|
}
|
|
85
85
|
if (!label) {
|
|
86
|
-
|
|
86
|
+
// eslint-disable-next-line max-len
|
|
87
|
+
throw new errors_1.MissingRouteError(`No default route set up. Please specify 'requestHandler' option or provide default route via 'crawler.router.addDefaultRoute()'.`);
|
|
87
88
|
}
|
|
88
89
|
throw new errors_1.MissingRouteError(`Route not found for label '${String(label)}' and no default route set up!`);
|
|
89
90
|
}
|