apify 3.0.0-beta.7 → 3.0.0-beta.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ import { Configuration as CoreConfiguration } from '@crawlee/core';
2
+ import type { ConfigurationOptions as CoreConfigurationOptions } from '@crawlee/core';
3
+ export interface ConfigurationOptions extends CoreConfigurationOptions {
4
+ metamorphAfterSleepMillis?: number;
5
+ actorEventsWsUrl?: string;
6
+ token?: string;
7
+ actorId?: string;
8
+ actorRunId?: string;
9
+ actorTaskId?: string;
10
+ apiBaseUrl?: string;
11
+ containerPort?: number;
12
+ containerUrl?: string;
13
+ proxyHostname?: string;
14
+ proxyPassword?: string;
15
+ proxyPort?: number;
16
+ proxyStatusUrl?: string;
17
+ isAtHome?: boolean;
18
+ userId?: string;
19
+ }
20
+ /**
21
+ * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
22
+ *
23
+ * 1. When using `Actor` class, we can get the instance configuration via `sdk.config`
24
+ * ```js
25
+ * import { Actor } from 'apify';
26
+ *
27
+ * const sdk = new Actor({ token: '123' });
28
+ * console.log(sdk.config.get('token')); // '123'
29
+ * const crawler = new BasicCrawler({ ... }, config);
30
+ * ```
31
+ * 2. To get the global configuration (singleton instance). It will respect the environment variables.
32
+ * ```js
33
+ * import { BasicCrawler, Configuration } from 'crawlee';
34
+ *
35
+ * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
36
+ * const crawler = new BasicCrawler({ ... }, config);
37
+ * ```
38
+ *
39
+ * ## Supported Configuration Options
40
+ *
41
+ * Key | Environment Variable | Default Value
42
+ * ---|---|---
43
+ * `memoryMbytes` | `APIFY_MEMORY_MBYTES` | -
44
+ * `headless` | `APIFY_HEADLESS` | -
45
+ * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
46
+ * `token` | `APIFY_TOKEN` | -
47
+ * `isAtHome` | `APIFY_IS_AT_HOME` | -
48
+ * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
49
+ * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
50
+ * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
51
+ *
52
+ * ## Advanced Configuration Options
53
+ *
54
+ * Key | Environment Variable | Default Value
55
+ * ---|---|---
56
+ * `actorEventsWsUrl` | `APIFY_ACTOR_EVENTS_WS_URL` | -
57
+ * `actorId` | `APIFY_ACTOR_ID` | -
58
+ * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
59
+ * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
60
+ * `apiBaseUrl` | `APIFY_API_BASE_URL` | `'https://api.apify.com'`
61
+ * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
62
+ * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
63
+ * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
64
+ * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
65
+ * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
66
+ * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
67
+ * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
68
+ * `proxyStatusUrl` | `APIFY_PROXY_STATUS_URL` | `'http://proxy.apify.com'`
69
+ * `userId` | `APIFY_USER_ID` | -
70
+ * `xvfb` | `APIFY_XVFB` | -
71
+ * `chromeExecutablePath` | `APIFY_CHROME_EXECUTABLE_PATH` | -
72
+ * `defaultBrowserPath` | `APIFY_DEFAULT_BROWSER_PATH` | -
73
+ */
74
+ export declare class Configuration extends CoreConfiguration {
75
+ /** @inheritDoc */
76
+ static globalConfig?: Configuration;
77
+ protected static ENV_MAP: {
78
+ APIFY_AVAILABLE_MEMORY_RATIO: string;
79
+ APIFY_PURGE_ON_START: string;
80
+ APIFY_MEMORY_MBYTES: string;
81
+ APIFY_DEFAULT_DATASET_ID: string;
82
+ APIFY_DEFAULT_KEY_VALUE_STORE_ID: string;
83
+ APIFY_DEFAULT_REQUEST_QUEUE_ID: string;
84
+ APIFY_INPUT_KEY: string;
85
+ APIFY_PERSIST_STATE_INTERVAL_MILLIS: string;
86
+ APIFY_HEADLESS: string;
87
+ APIFY_XVFB: string;
88
+ APIFY_CHROME_EXECUTABLE_PATH: string;
89
+ APIFY_DEFAULT_BROWSER_PATH: string;
90
+ APIFY_DISABLE_BROWSER_SANDBOX: string;
91
+ APIFY_TOKEN: string;
92
+ APIFY_METAMORPH_AFTER_SLEEP_MILLIS: string;
93
+ APIFY_TEST_PERSIST_INTERVAL_MILLIS: string;
94
+ APIFY_ACTOR_EVENTS_WS_URL: string;
95
+ APIFY_ACTOR_ID: string;
96
+ APIFY_API_BASE_URL: string;
97
+ APIFY_IS_AT_HOME: string;
98
+ APIFY_ACTOR_RUN_ID: string;
99
+ APIFY_ACTOR_TASK_ID: string;
100
+ APIFY_CONTAINER_PORT: string;
101
+ APIFY_CONTAINER_URL: string;
102
+ APIFY_USER_ID: string;
103
+ APIFY_PROXY_HOSTNAME: string;
104
+ APIFY_PROXY_PASSWORD: string;
105
+ APIFY_PROXY_STATUS_URL: string;
106
+ APIFY_PROXY_PORT: string;
107
+ CRAWLEE_AVAILABLE_MEMORY_RATIO: string;
108
+ CRAWLEE_PURGE_ON_START: string;
109
+ CRAWLEE_MEMORY_MBYTES: string;
110
+ CRAWLEE_DEFAULT_DATASET_ID: string;
111
+ CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: string;
112
+ CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: string;
113
+ CRAWLEE_INPUT_KEY: string;
114
+ CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: string;
115
+ CRAWLEE_HEADLESS: string;
116
+ CRAWLEE_XVFB: string;
117
+ CRAWLEE_CHROME_EXECUTABLE_PATH: string;
118
+ CRAWLEE_DEFAULT_BROWSER_PATH: string;
119
+ CRAWLEE_DISABLE_BROWSER_SANDBOX: string;
120
+ CRAWLEE_LOG_LEVEL: string;
121
+ };
122
+ protected static INTEGER_VARS: string[];
123
+ protected static BOOLEAN_VARS: string[];
124
+ protected static DEFAULTS: {
125
+ defaultKeyValueStoreId: string;
126
+ defaultDatasetId: string;
127
+ defaultRequestQueueId: string;
128
+ inputKey: string;
129
+ apiBaseUrl: string;
130
+ proxyStatusUrl: string;
131
+ proxyHostname: string;
132
+ proxyPort: number;
133
+ containerPort: number;
134
+ containerUrl: string;
135
+ metamorphAfterSleepMillis: number;
136
+ persistStateIntervalMillis: number;
137
+ maxUsedCpuRatio: number;
138
+ availableMemoryRatio: number;
139
+ storageClientOptions: {};
140
+ purgeOnStart: boolean;
141
+ systemInfoIntervalMillis: number;
142
+ };
143
+ /**
144
+ * @inheritDoc
145
+ */
146
+ get<T extends keyof ConfigurationOptions, U extends ConfigurationOptions[T]>(key: T, defaultValue?: U): U;
147
+ /**
148
+ * @inheritDoc
149
+ */
150
+ set(key: keyof ConfigurationOptions, value?: any): void;
151
+ /**
152
+ * @inheritDoc
153
+ */
154
+ static getGlobalConfig(): Configuration;
155
+ /**
156
+ * Resets global configuration instance. The default instance holds configuration based on env vars,
157
+ * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
158
+ */
159
+ static resetGlobalState(): void;
160
+ }
161
+ //# sourceMappingURL=configuration.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACnE,OAAO,KAAK,EAAE,oBAAoB,IAAI,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAEtF,MAAM,WAAW,oBAAqB,SAAQ,wBAAwB;IAClE,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;IAChD,kBAAkB;IAClB,OAAgB,YAAY,CAAC,EAAE,aAAa,CAAC;IAG7C,iBAA0B,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAoC/B;IAEF,iBAA0B,YAAY,WAAsF;IAE5H,iBAA0B,YAAY,WAAuC;IAE7E,iBAA0B,QAAQ;;;;;;;;;;;;;;;;;;MAchC;IAEF;;OAEG;IACM,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IAIlH;;OAEG;IACM,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG;IAIzD;;OAEG;WACa,eAAe,IAAI,aAAa;IAShD;;;OAGG;WACa,gBAAgB,IAAI,IAAI;CAG3C"}
@@ -0,0 +1,177 @@
1
+ "use strict";
2
+ var _a, _b;
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.Configuration = void 0;
5
+ const consts_1 = require("@apify/consts");
6
+ const core_1 = require("@crawlee/core");
7
+ /**
8
+ * `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
9
+ *
10
+ * 1. When using `Actor` class, we can get the instance configuration via `sdk.config`
11
+ * ```js
12
+ * import { Actor } from 'apify';
13
+ *
14
+ * const sdk = new Actor({ token: '123' });
15
+ * console.log(sdk.config.get('token')); // '123'
16
+ * const crawler = new BasicCrawler({ ... }, config);
17
+ * ```
18
+ * 2. To get the global configuration (singleton instance). It will respect the environment variables.
19
+ * ```js
20
+ * import { BasicCrawler, Configuration } from 'crawlee';
21
+ *
22
+ * const config = new Configuration({ persistStateIntervalMillis: 30_000 });
23
+ * const crawler = new BasicCrawler({ ... }, config);
24
+ * ```
25
+ *
26
+ * ## Supported Configuration Options
27
+ *
28
+ * Key | Environment Variable | Default Value
29
+ * ---|---|---
30
+ * `memoryMbytes` | `APIFY_MEMORY_MBYTES` | -
31
+ * `headless` | `APIFY_HEADLESS` | -
32
+ * `persistStateIntervalMillis` | `APIFY_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
33
+ * `token` | `APIFY_TOKEN` | -
34
+ * `isAtHome` | `APIFY_IS_AT_HOME` | -
35
+ * `defaultDatasetId` | `APIFY_DEFAULT_DATASET_ID` | `'default'`
36
+ * `defaultKeyValueStoreId` | `APIFY_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
37
+ * `defaultRequestQueueId` | `APIFY_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
38
+ *
39
+ * ## Advanced Configuration Options
40
+ *
41
+ * Key | Environment Variable | Default Value
42
+ * ---|---|---
43
+ * `actorEventsWsUrl` | `APIFY_ACTOR_EVENTS_WS_URL` | -
44
+ * `actorId` | `APIFY_ACTOR_ID` | -
45
+ * `actorRunId` | `APIFY_ACTOR_RUN_ID` | -
46
+ * `actorTaskId` | `APIFY_ACTOR_TASK_ID` | -
47
+ * `apiBaseUrl` | `APIFY_API_BASE_URL` | `'https://api.apify.com'`
48
+ * `containerPort` | `APIFY_CONTAINER_PORT` | `4321`
49
+ * `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
50
+ * `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
51
+ * `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
52
+ * `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
53
+ * `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
54
+ * `proxyPort` | `APIFY_PROXY_PORT` | `8000`
55
+ * `proxyStatusUrl` | `APIFY_PROXY_STATUS_URL` | `'http://proxy.apify.com'`
56
+ * `userId` | `APIFY_USER_ID` | -
57
+ * `xvfb` | `APIFY_XVFB` | -
58
+ * `chromeExecutablePath` | `APIFY_CHROME_EXECUTABLE_PATH` | -
59
+ * `defaultBrowserPath` | `APIFY_DEFAULT_BROWSER_PATH` | -
60
+ */
61
+ class Configuration extends (_b = core_1.Configuration) {
62
+ /**
63
+ * @inheritDoc
64
+ */
65
+ get(key, defaultValue) {
66
+ return super.get(key, defaultValue);
67
+ }
68
+ /**
69
+ * @inheritDoc
70
+ */
71
+ set(key, value) {
72
+ super.set(key, value);
73
+ }
74
+ /**
75
+ * @inheritDoc
76
+ */
77
+ static getGlobalConfig() {
78
+ if (Configuration.storage.getStore()) {
79
+ return Configuration.storage.getStore();
80
+ }
81
+ Configuration.globalConfig ?? (Configuration.globalConfig = new Configuration());
82
+ return Configuration.globalConfig;
83
+ }
84
+ /**
85
+ * Resets global configuration instance. The default instance holds configuration based on env vars,
86
+ * if we want to change them, we need to first reset the global state. Used mainly for testing purposes.
87
+ */
88
+ static resetGlobalState() {
89
+ delete this.globalConfig;
90
+ }
91
+ }
92
+ exports.Configuration = Configuration;
93
+ _a = Configuration;
94
+ // maps environment variables to config keys (e.g. `APIFY_MEMORY_MBYTES` to `memoryMbytes`)
95
+ Object.defineProperty(Configuration, "ENV_MAP", {
96
+ enumerable: true,
97
+ configurable: true,
98
+ writable: true,
99
+ value: {
100
+ // regular crawlee env vars are also supported
101
+ ...Reflect.get(_b, "ENV_MAP", _a),
102
+ // support crawlee env vars prefixed with `APIFY_` too
103
+ APIFY_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
104
+ APIFY_PURGE_ON_START: 'purgeOnStart',
105
+ APIFY_MEMORY_MBYTES: 'memoryMbytes',
106
+ APIFY_DEFAULT_DATASET_ID: 'defaultDatasetId',
107
+ APIFY_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
108
+ APIFY_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
109
+ APIFY_INPUT_KEY: 'inputKey',
110
+ APIFY_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
111
+ APIFY_HEADLESS: 'headless',
112
+ APIFY_XVFB: 'xvfb',
113
+ APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
114
+ APIFY_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
115
+ APIFY_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
116
+ // as well as apify specific ones
117
+ APIFY_TOKEN: 'token',
118
+ APIFY_METAMORPH_AFTER_SLEEP_MILLIS: 'metamorphAfterSleepMillis',
119
+ APIFY_TEST_PERSIST_INTERVAL_MILLIS: 'persistStateIntervalMillis',
120
+ APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl',
121
+ APIFY_ACTOR_ID: 'actorId',
122
+ APIFY_API_BASE_URL: 'apiBaseUrl',
123
+ APIFY_IS_AT_HOME: 'isAtHome',
124
+ APIFY_ACTOR_RUN_ID: 'actorRunId',
125
+ APIFY_ACTOR_TASK_ID: 'actorTaskId',
126
+ APIFY_CONTAINER_PORT: 'containerPort',
127
+ APIFY_CONTAINER_URL: 'containerUrl',
128
+ APIFY_USER_ID: 'userId',
129
+ APIFY_PROXY_HOSTNAME: 'proxyHostname',
130
+ APIFY_PROXY_PASSWORD: 'proxyPassword',
131
+ APIFY_PROXY_STATUS_URL: 'proxyStatusUrl',
132
+ APIFY_PROXY_PORT: 'proxyPort',
133
+ }
134
+ });
135
+ Object.defineProperty(Configuration, "INTEGER_VARS", {
136
+ enumerable: true,
137
+ configurable: true,
138
+ writable: true,
139
+ value: [...Reflect.get(_b, "INTEGER_VARS", _a), 'proxyPort', 'containerPort', 'metamorphAfterSleepMillis']
140
+ });
141
+ Object.defineProperty(Configuration, "BOOLEAN_VARS", {
142
+ enumerable: true,
143
+ configurable: true,
144
+ writable: true,
145
+ value: [...Reflect.get(_b, "BOOLEAN_VARS", _a), 'isAtHome']
146
+ });
147
+ Object.defineProperty(Configuration, "DEFAULTS", {
148
+ enumerable: true,
149
+ configurable: true,
150
+ writable: true,
151
+ value: {
152
+ ...Reflect.get(_b, "DEFAULTS", _a),
153
+ defaultKeyValueStoreId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
154
+ defaultDatasetId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_DATASET_ID],
155
+ defaultRequestQueueId: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
156
+ inputKey: 'INPUT',
157
+ apiBaseUrl: 'https://api.apify.com',
158
+ proxyStatusUrl: 'http://proxy.apify.com',
159
+ proxyHostname: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_HOSTNAME],
160
+ proxyPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_PORT],
161
+ containerPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_PORT],
162
+ containerUrl: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_URL],
163
+ metamorphAfterSleepMillis: 300e3,
164
+ persistStateIntervalMillis: 60e3, // This value is mentioned in jsdoc in `events.js`, if you update it here, update it there too.
165
+ }
166
+ });
167
+ // monkey patch the core class so it respects the new options too
168
+ core_1.Configuration.getGlobalConfig = Configuration.getGlobalConfig;
169
+ // @ts-expect-error protected property
170
+ core_1.Configuration.ENV_MAP = Configuration.ENV_MAP;
171
+ // @ts-expect-error protected property
172
+ core_1.Configuration.INTEGER_VARS = Configuration.INTEGER_VARS;
173
+ // @ts-expect-error protected property
174
+ core_1.Configuration.BOOLEAN_VARS = Configuration.BOOLEAN_VARS;
175
+ // @ts-expect-error protected property
176
+ core_1.Configuration.DEFAULTS = Configuration.DEFAULTS;
177
+ //# sourceMappingURL=configuration.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AAAA,0CAAyD;AACzD,wCAAmE;AAqBnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AACH,MAAa,aAAc,SAAQ,MAAA,oBAAiB,CAAA;IA+DhD;;OAEG;IACM,GAAG,CAA0E,GAAM,EAAE,YAAgB;QAC1G,OAAO,KAAK,CAAC,GAAG,CAAC,GAAqC,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAED;;OAEG;IACM,GAAG,CAAC,GAA+B,EAAE,KAAW;QACrD,KAAK,CAAC,GAAG,CAAC,GAAqC,EAAE,KAAK,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,MAAM,CAAU,eAAe;QAC3B,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAmB,CAAC;SAC5D;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAA6B,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAU,gBAAgB;QAC5B,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;;AA/FL,sCAgGC;;AA5FG,2FAA2F;AAC3F;;;;WAAoC;QAChC,8CAA8C;QAC9C,GAAG,8BAAa;QAEhB,sDAAsD;QACtD,4BAA4B,EAAE,sBAAsB;QACpD,oBAAoB,EAAE,cAAc;QACpC,mBAAmB,EAAE,cAAc;QACnC,wBAAwB,EAAE,kBAAkB;QAC5C,gCAAgC,EAAE,wBAAwB;QAC1D,8BAA8B,EAAE,uBAAuB;QACvD,eAAe,EAAE,UAAU;QAC3B,mCAAmC,EAAE,4BAA4B;QACjE,cAAc,EAAE,UAAU;QAC1B,UAAU,EAAE,MAAM;QAClB,4BAA4B,EAAE,sBAAsB;QACpD,0BAA0B,EAAE,oBAAoB;QAChD,6BAA6B,EAAE,uBAAuB;QAEtD,iCAAiC;QACjC,WAAW,EAAE,OAAO;QACpB,kCAAkC,EAAE,2BAA2B;QAC/D,kCAAkC,EAAE,4BAA4B;QAChE,yBAAyB,EAAE,kBAAkB;QAC7C,cAAc,EAAE,SAAS;QACzB,kBAAkB,EAAE,YAAY;QAChC,gBAAgB,EAAE,UAAU;QAC5B,kBAAkB,EAAE,YAAY;QAChC,mBAAmB,EAAE,aAAa;QAClC,oBAAoB,EAAE,eAAe;QACrC,mBAAmB,EAAE,cAAc;QACnC,aAAa,EAAE,QAAQ;QACvB,oBAAoB,EAAE,eAAe;QACrC,oBAAoB,EAAE,eAAe;QACrC,sBAAsB,EAAE,gBAAgB;QACxC,gBAAgB,EAAE,WAAW;KAChC;EAAC,CAAA;AAEF;;;;WAAyC,CAAC,GAAG,mCAAkB,EAAE,WAAW,EAAE,eAAe,EAAE,2BAA2B,CAAC;EAAC,CAAA;AAE5H;;;;WAAyC,CAAC,GAAG,mCAAkB,EAAE,UAAU,CAAC;EAAC,CAAA;AAE7E;;;;WAAqC;QACjC,GAAG,+BAAc;QACjB,sBAAsB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,0BAA0B,CAAC;QAC3E,gBAAgB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,kBAAkB,CAAC;QAC7D,qBAAqB,EAAE,uBAAc,CAAC,iBAAQ,CAAC,wBAAwB,CAAC;QACxE,QAAQ,EAAE,OAAO;QACjB,UAAU,EAAE,uBAAuB;QACnC,cAAc,EAAE,wBAAwB;QACxC,aAAa,EAAE,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACtD,SAAS,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,UAAU,CAAC;QAC/C,aAAa,EAAE,CAAC,uBAAc,CAAC,iBAAQ,CAAC,cAAc,CAAC;QACvD,YAAY,EAAE,uBAAc,CAAC,iBAAQ,CAAC,aAAa,CAAC;QACpD,yBAAyB,EAAE,KAAK;QAChC,0BAA0B,EAAE,IAAI,EAAE,+FAA+F;KACpI;EAAC,CAAA;AAqCN,iEAAiE;AACjE,oBAAiB,CAAC,eAAe,GAAG,aAAa,CAAC,eAAe,CAAC;AAClE,sCAAsC;AACtC,oBAAiB,CAAC,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC;AAClD,sCAAsC;AACtC,oBAAiB,CAAC,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;AAC5D,sCAAsC;AACtC,oBAAiB,CAAC,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;AAC5D,sCAAsC;AACtC,oBAAiB,CAAC,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC"}
package/index.d.ts CHANGED
@@ -1,4 +1,7 @@
1
1
  export * from './actor';
2
+ export * from './configuration';
2
3
  export * from './proxy_configuration';
3
4
  export * from './platform_event_manager';
5
+ export * from './key_value_store';
6
+ export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue, QueueOperationInfo, RequestQueueOperationOptions, RequestQueueOptions, QueueOperationInfoOptions, KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, } from '@crawlee/core';
4
7
  //# sourceMappingURL=index.d.ts.map
package/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,mBAAmB,CAAC;AAClC,OAAO,EACH,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EACnI,YAAY,EAAE,kBAAkB,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,yBAAyB,EAC9G,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,4BAA4B,GACjF,MAAM,eAAe,CAAC"}
package/index.js CHANGED
@@ -1,7 +1,13 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RequestQueue = exports.Dataset = void 0;
3
4
  const tslib_1 = require("tslib");
4
5
  tslib_1.__exportStar(require("./actor"), exports);
6
+ tslib_1.__exportStar(require("./configuration"), exports);
5
7
  tslib_1.__exportStar(require("./proxy_configuration"), exports);
6
8
  tslib_1.__exportStar(require("./platform_event_manager"), exports);
9
+ tslib_1.__exportStar(require("./key_value_store"), exports);
10
+ var core_1 = require("@crawlee/core");
11
+ Object.defineProperty(exports, "Dataset", { enumerable: true, get: function () { return core_1.Dataset; } });
12
+ Object.defineProperty(exports, "RequestQueue", { enumerable: true, get: function () { return core_1.RequestQueue; } });
7
13
  //# sourceMappingURL=index.js.map
package/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,kDAAwB;AACxB,gEAAsC;AACtC,mEAAyC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,kDAAwB;AACxB,0DAAgC;AAChC,gEAAsC;AACtC,mEAAyC;AACzC,4DAAkC;AAClC,sCAIuB;AAHnB,+FAAA,OAAO,OAAA;AACP,oGAAA,YAAY,OAAA"}
package/index.mjs CHANGED
@@ -2,6 +2,10 @@ import mod from "./index.js";
2
2
 
3
3
  export default mod;
4
4
  export const Actor = mod.Actor;
5
+ export const Configuration = mod.Configuration;
6
+ export const Dataset = mod.Dataset;
5
7
  export const EXIT_CODES = mod.EXIT_CODES;
8
+ export const KeyValueStore = mod.KeyValueStore;
6
9
  export const PlatformEventManager = mod.PlatformEventManager;
7
10
  export const ProxyConfiguration = mod.ProxyConfiguration;
11
+ export const RequestQueue = mod.RequestQueue;
@@ -0,0 +1,17 @@
1
+ import type { StorageManagerOptions } from '@crawlee/core';
2
+ import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
3
+ /**
4
+ * @inheritDoc
5
+ */
6
+ export declare class KeyValueStore extends CoreKeyValueStore {
7
+ /**
8
+ * Returns a URL for the given key that may be used to publicly
9
+ * access the value in the remote key-value store.
10
+ */
11
+ getPublicUrl(key: string): string;
12
+ /**
13
+ * @inheritDoc
14
+ */
15
+ static open(storeIdOrName?: string | null, options?: StorageManagerOptions): Promise<KeyValueStore>;
16
+ }
17
+ //# sourceMappingURL=key_value_store.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../src/key_value_store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEnE;;GAEG;AACH,qBAAa,aAAc,SAAQ,iBAAiB;IAChD;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIjC;;OAEG;WACmB,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;CAGzH"}
@@ -0,0 +1,26 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.KeyValueStore = void 0;
4
+ const core_1 = require("@crawlee/core");
5
+ /**
6
+ * @inheritDoc
7
+ */
8
+ class KeyValueStore extends core_1.KeyValueStore {
9
+ /**
10
+ * Returns a URL for the given key that may be used to publicly
11
+ * access the value in the remote key-value store.
12
+ */
13
+ getPublicUrl(key) {
14
+ return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`;
15
+ }
16
+ /**
17
+ * @inheritDoc
18
+ */
19
+ static async open(storeIdOrName, options = {}) {
20
+ return super.open(storeIdOrName, options);
21
+ }
22
+ }
23
+ exports.KeyValueStore = KeyValueStore;
24
+ // @ts-expect-error extension of the core class to make this only a type-issue
25
+ core_1.KeyValueStore.prototype.getPublicUrl = KeyValueStore.prototype.getPublicUrl;
26
+ //# sourceMappingURL=key_value_store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../src/key_value_store.ts"],"names":[],"mappings":";;;AACA,wCAAmE;AAEnE;;GAEG;AACH,MAAa,aAAc,SAAQ,oBAAiB;IAChD;;;OAGG;IACH,YAAY,CAAC,GAAW;QACpB,OAAO,6CAA6C,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,MAAM,CAAU,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QACzF,OAAO,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAA6B,CAAC;IAC1E,CAAC;CACJ;AAfD,sCAeC;AAED,8EAA8E;AAC9E,oBAAiB,CAAC,SAAS,CAAC,YAAY,GAAG,aAAa,CAAC,SAAS,CAAC,YAAY,CAAC"}
package/package.json CHANGED
@@ -1,16 +1,20 @@
1
1
  {
2
2
  "name": "apify",
3
- "version": "3.0.0-beta.7",
3
+ "version": "3.0.0-beta.72",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=16.0.0"
7
7
  },
8
- "types": "index.d.ts",
8
+ "main": "./index.js",
9
+ "module": "./index.mjs",
10
+ "types": "./index.d.ts",
9
11
  "exports": {
10
12
  ".": {
11
13
  "import": "./index.mjs",
12
- "require": "./index.js"
13
- }
14
+ "require": "./index.js",
15
+ "types": "./index.d.ts"
16
+ },
17
+ "./package.json": "./package.json"
14
18
  },
15
19
  "keywords": [
16
20
  "apify",
@@ -33,12 +37,12 @@
33
37
  "license": "Apache-2.0",
34
38
  "repository": {
35
39
  "type": "git",
36
- "url": "git+https://github.com/apify/apify-js"
40
+ "url": "git+https://github.com/apify/apify-ts"
37
41
  },
38
42
  "bugs": {
39
- "url": "https://github.com/apify/apify-js/issues"
43
+ "url": "https://github.com/apify/apify-ts/issues"
40
44
  },
41
- "homepage": "https://sdk.apify.com/",
45
+ "homepage": "https://apify.github.io/apify-ts/",
42
46
  "scripts": {
43
47
  "build": "npm run clean && npm run compile && npm run copy && npm run fixApifyExport",
44
48
  "clean": "rimraf ./dist",
@@ -50,15 +54,16 @@
50
54
  "access": "public"
51
55
  },
52
56
  "dependencies": {
53
- "@apify/consts": "^1.10.0",
54
- "@apify/log": "^1.2.3",
55
- "@apify/utilities": "^1.2.12",
56
- "@crawlee/core": "^3.0.0-beta.7",
57
- "@crawlee/types": "^3.0.0-beta.7",
58
- "@crawlee/utils": "^3.0.0-beta.7",
59
- "semver": "^7.3.7",
60
- "apify-client": "^2.5.0",
57
+ "@apify/consts": "^2.0.0",
58
+ "@apify/log": "^2.0.0",
59
+ "@apify/timeout": "^0.3.0",
60
+ "@apify/utilities": "^2.0.0",
61
+ "@crawlee/core": "^3.0.0-beta.72",
62
+ "@crawlee/types": "^3.0.0-beta.72",
63
+ "@crawlee/utils": "^3.0.0-beta.72",
64
+ "apify-client": "^2.6.0-beta.1",
61
65
  "ow": "^0.28.1",
62
- "ws": "^8.7.0"
66
+ "semver": "^7.3.7",
67
+ "ws": "^7.5.8"
63
68
  }
64
69
  }
@@ -1,4 +1,5 @@
1
1
  import { EventManager } from '@crawlee/core';
2
+ import { Configuration } from './configuration';
2
3
  /**
3
4
  * Gets an instance of a Node.js'
4
5
  * [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)
@@ -37,8 +38,10 @@ import { EventManager } from '@crawlee/core';
37
38
  * you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
38
39
  */
39
40
  export declare class PlatformEventManager extends EventManager {
41
+ readonly config: Configuration;
40
42
  /** Websocket connection to actor events. */
41
43
  private eventsWs?;
44
+ constructor(config?: Configuration);
42
45
  /**
43
46
  * Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
44
47
  * This is an internal function that is automatically called by `Actor.main()`.
@@ -1 +1 @@
1
- {"version":3,"file":"platform_event_manager.d.ts","sourceRoot":"","sources":["../src/platform_event_manager.ts"],"names":[],"mappings":"AAEA,OAAO,EAAa,YAAY,EAAE,MAAM,eAAe,CAAC;AAGxD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,oBAAqB,SAAQ,YAAY;IAClD,4CAA4C;IAC5C,OAAO,CAAC,QAAQ,CAAC,CAAY;IAE7B;;;OAGG;IACY,IAAI;IAiBnB,OAAO,CAAC,yBAAyB;IAgCjC;;;;OAIG;IACY,KAAK;CAQvB"}
1
+ {"version":3,"file":"platform_event_manager.d.ts","sourceRoot":"","sources":["../src/platform_event_manager.ts"],"names":[],"mappings":"AAEA,OAAO,EAAa,YAAY,EAAE,MAAM,eAAe,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,oBAAqB,SAAQ,YAAY;aAIpB,MAAM;IAHpC,4CAA4C;IAC5C,OAAO,CAAC,QAAQ,CAAC,CAAY;gBAEC,MAAM,gBAAkC;IAItE;;;OAGG;IACY,IAAI;IAiBnB,OAAO,CAAC,yBAAyB;IAgCjC;;;;OAIG;IACY,KAAK;CAQvB"}
@@ -6,6 +6,7 @@ const consts_1 = require("@apify/consts");
6
6
  const ws_1 = tslib_1.__importDefault(require("ws"));
7
7
  const core_1 = require("@crawlee/core");
8
8
  const utilities_1 = require("@apify/utilities");
9
+ const configuration_1 = require("./configuration");
9
10
  /**
10
11
  * Gets an instance of a Node.js'
11
12
  * [EventEmitter](https://nodejs.org/api/events.html#events_class_eventemitter)
@@ -44,8 +45,14 @@ const utilities_1 = require("@apify/utilities");
44
45
  * you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
45
46
  */
46
47
  class PlatformEventManager extends core_1.EventManager {
47
- constructor() {
48
- super(...arguments);
48
+ constructor(config = configuration_1.Configuration.getGlobalConfig()) {
49
+ super();
50
+ Object.defineProperty(this, "config", {
51
+ enumerable: true,
52
+ configurable: true,
53
+ writable: true,
54
+ value: config
55
+ });
49
56
  /** Websocket connection to actor events. */
50
57
  Object.defineProperty(this, "eventsWs", {
51
58
  enumerable: true,
@@ -63,7 +70,7 @@ class PlatformEventManager extends core_1.EventManager {
63
70
  return;
64
71
  }
65
72
  await super.init();
66
- const eventsWsUrl = process.env[consts_1.ENV_VARS.ACTOR_EVENTS_WS_URL];
73
+ const eventsWsUrl = this.config.get('actorEventsWsUrl');
67
74
  // Locally there is no web socket to connect, so just print a log message.
68
75
  if (!eventsWsUrl) {
69
76
  this.log.debug(`Environment variable ${consts_1.ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, no events from Apify platform will be emitted.`);
@@ -95,7 +102,7 @@ class PlatformEventManager extends core_1.EventManager {
95
102
  this.log.exception(err, 'web socket connection failed');
96
103
  });
97
104
  this.eventsWs.on('close', () => {
98
- this.log.warning('web socket has been closed');
105
+ this.log.debug('web socket has been closed');
99
106
  this.eventsWs = undefined;
100
107
  });
101
108
  }
@@ -1 +1 @@
1
- {"version":3,"file":"platform_event_manager.js","sourceRoot":"","sources":["../src/platform_event_manager.ts"],"names":[],"mappings":";;;;AAAA,0CAA4D;AAC5D,oDAA2B;AAC3B,wCAAwD;AACxD,gDAAuD;AAEvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAa,oBAAqB,SAAQ,mBAAY;IAAtD;;QACI,4CAA4C;QAC5C;;;;;WAA6B;IAoEjC,CAAC;IAlEG;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,mBAAmB,CAAC,CAAC;QAE9D,0EAA0E;QAC1E,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,iBAAQ,CAAC,mBAAmB,6DAA6D,CAAC,CAAC;YAClI,OAAO;SACV;QAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IAEO,yBAAyB,CAAC,WAAmB;QACjD,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAS,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACpC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAErB,IAAI;gBACA,MAAM,EACF,IAAI,EACJ,IAAI,GACP,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAE7B,IAAI,IAAI,KAAK,0BAAiB,CAAC,SAAS,EAAE;oBACtC,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC,CAAC,4CAA4C;oBAC/F,IAAI,CAAC,MAAM,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;iBACpE;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,0BAA0B,CAAC,CAAC;aAChE;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC9B,iFAAiF;YACjF,IAAI,GAAG,CAAC,OAAO,KAAK,4DAA4D;gBAAE,OAAO;YAEzF,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,8BAA8B,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC3B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;YAC/C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;QAC9B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC3B,CAAC;CACJ;AAtED,oDAsEC"}
1
+ {"version":3,"file":"platform_event_manager.js","sourceRoot":"","sources":["../src/platform_event_manager.ts"],"names":[],"mappings":";;;;AAAA,0CAA4D;AAC5D,oDAA2B;AAC3B,wCAAwD;AACxD,gDAAuD;AACvD,mDAAgD;AAEhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAa,oBAAqB,SAAQ,mBAAY;IAIlD,YAA8B,SAAS,6BAAa,CAAC,eAAe,EAAE;QAClE,KAAK,EAAE,CAAC;;;;;mBADkB;;QAH9B,4CAA4C;QAC5C;;;;;WAA6B;IAI7B,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAExD,0EAA0E;QAC1E,IAAI,CAAC,WAAW,EAAE;YACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,iBAAQ,CAAC,mBAAmB,6DAA6D,CAAC,CAAC;YAClI,OAAO;SACV;QAED,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IAEO,yBAAyB,CAAC,WAAmB;QACjD,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAS,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;YACpC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAErB,IAAI;gBACA,MAAM,EACF,IAAI,EACJ,IAAI,GACP,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAE7B,IAAI,IAAI,KAAK,0BAAiB,CAAC,SAAS,EAAE;oBACtC,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC,CAAC,4CAA4C;oBAC/F,IAAI,CAAC,MAAM,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;iBACpE;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAY,EAAE,0BAA0B,CAAC,CAAC;aAChE;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YAC9B,iFAAiF;YACjF,IAAI,GAAG,CAAC,OAAO,KAAK,4DAA4D;gBAAE,OAAO;YAEzF,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,8BAA8B,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC3B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC7C,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;QAC9B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC3B,CAAC;CACJ;AA1ED,oDA0EC"}
@@ -1,4 +1,6 @@
1
- import { Configuration, ProxyConfiguration as CoreProxyConfiguration, ProxyConfigurationOptions as CoreProxyConfigurationOptions, ProxyInfo as CoreProxyInfo } from '@crawlee/core';
1
+ import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions, ProxyInfo as CoreProxyInfo } from '@crawlee/core';
2
+ import { ProxyConfiguration as CoreProxyConfiguration } from '@crawlee/core';
3
+ import { Configuration } from './configuration';
2
4
  export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions {
3
5
  /**
4
6
  * User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
@@ -54,11 +56,11 @@ export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions
54
56
  * const crawler = new CheerioCrawler({
55
57
  * // ...
56
58
  * proxyConfiguration,
57
- * handlePageFunction: ({ proxyInfo }) => {
58
- * // Getting used proxy URL
59
+ * requestHandler({ proxyInfo }) {
60
+ * // Getting used proxy URL
59
61
  * const proxyUrl = proxyInfo.url;
60
62
  *
61
- * // Getting ID of used Session
63
+ * // Getting ID of used Session
62
64
  * const sessionIdentifier = proxyInfo.sessionId;
63
65
  * }
64
66
  * })
@@ -114,7 +116,7 @@ export interface ProxyInfo extends CoreProxyInfo {
114
116
  * const crawler = new CheerioCrawler({
115
117
  * // ...
116
118
  * proxyConfiguration,
117
- * handlePageFunction: ({ proxyInfo }) => {
119
+ * requestHandler({ proxyInfo }) {
118
120
  * const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
119
121
  * }
120
122
  * })
@@ -123,6 +125,7 @@ export interface ProxyInfo extends CoreProxyInfo {
123
125
  * @category Scaling
124
126
  */
125
127
  export declare class ProxyConfiguration extends CoreProxyConfiguration {
128
+ readonly config: Configuration;
126
129
  private groups;
127
130
  private countryCode?;
128
131
  private password?;
@@ -145,7 +148,7 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
145
148
  /**
146
149
  * This function creates a new {@link ProxyInfo} info object.
147
150
  * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
148
- * the currently used proxy via the handlePageFunction parameter: proxyInfo.
151
+ * the currently used proxy via the requestHandler parameter `proxyInfo`.
149
152
  * Use it if you want to work with a rich representation of a proxy URL.
150
153
  * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
151
154
  * @param [sessionId]