apify 4.0.0-beta.18 → 4.0.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +28 -4
- package/dist/actor.js +45 -45
- package/dist/apify_dataset_backend.d.ts +18 -0
- package/dist/apify_dataset_backend.js +33 -0
- package/dist/apify_key_value_store_backend.d.ts +23 -0
- package/dist/apify_key_value_store_backend.js +54 -0
- package/dist/apify_request_queue_backend.d.ts +78 -0
- package/dist/apify_request_queue_backend.js +114 -0
- package/dist/apify_request_queue_shared_backend.d.ts +44 -0
- package/dist/apify_request_queue_shared_backend.js +211 -0
- package/dist/apify_request_queue_single_backend.d.ts +52 -0
- package/dist/apify_request_queue_single_backend.js +256 -0
- package/dist/apify_storage_backend.d.ts +109 -0
- package/dist/apify_storage_backend.js +249 -0
- package/dist/configuration.d.ts +6 -4
- package/dist/configuration.js +4 -4
- package/dist/index.d.ts +4 -2
- package/dist/index.js +1 -1
- package/dist/input-schemas.d.ts +1 -1
- package/dist/key_value_store.js +8 -9
- package/dist/platform_event_manager.d.ts +2 -2
- package/dist/platform_event_manager.js +5 -5
- package/dist/proxy_configuration.d.ts +24 -7
- package/dist/proxy_configuration.js +66 -35
- package/dist/storage.d.ts +3 -3
- package/dist/storage.js +4 -4
- package/dist/utils.d.ts +5 -0
- package/dist/utils.js +11 -0
- package/package.json +7 -8
- package/dist/apify_storage_client.d.ts +0 -66
- package/dist/apify_storage_client.js +0 -200
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/* eslint-disable max-classes-per-file */
|
|
2
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { DatasetClient as ApifyDatasetClient } from 'apify-client';
|
|
5
|
+
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
6
|
+
import { ApifyDatasetBackend } from './apify_dataset_backend.js';
|
|
7
|
+
import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
|
|
8
|
+
import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
|
|
9
|
+
import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
|
|
10
|
+
import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
|
|
11
|
+
/** The reserved alias crawlee uses for the default (unnamed) storage. */
|
|
12
|
+
const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
13
|
+
/** The maximum clientKey length accepted by the request queue API. */
|
|
14
|
+
const MAX_CLIENT_KEY_LENGTH = 32;
|
|
15
|
+
const DEFAULT_ID_CONFIG_KEY = {
|
|
16
|
+
Dataset: 'defaultDatasetId',
|
|
17
|
+
KeyValueStore: 'defaultKeyValueStoreId',
|
|
18
|
+
RequestQueue: 'defaultRequestQueueId',
|
|
19
|
+
};
|
|
20
|
+
const ACTOR_STORAGES_TYPE_KEY = {
|
|
21
|
+
Dataset: 'datasets',
|
|
22
|
+
KeyValueStore: 'keyValueStores',
|
|
23
|
+
RequestQueue: 'requestQueues',
|
|
24
|
+
};
|
|
25
|
+
/** Marks a dataset backend whose underlying client charges for pushed items (pay-per-event). @internal */
|
|
26
|
+
export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception');
|
|
27
|
+
export const pushDataChargingContext = new AsyncLocalStorage();
|
|
28
|
+
/**
|
|
29
|
+
* Default `DatasetClient` that charges for pushed items (pay-per-event). Used
|
|
30
|
+
* only for the run's default dataset when a `apify-default-dataset-item` price
|
|
31
|
+
* is configured; for everything else the plain `apify-client` dataset client is
|
|
32
|
+
* used.
|
|
33
|
+
*/
|
|
34
|
+
class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
35
|
+
getChargingManager;
|
|
36
|
+
constructor(options, getChargingManager) {
|
|
37
|
+
super(options);
|
|
38
|
+
this.getChargingManager = getChargingManager;
|
|
39
|
+
}
|
|
40
|
+
normalizeItems(items) {
|
|
41
|
+
if (typeof items === 'string') {
|
|
42
|
+
const parsed = JSON.parse(items);
|
|
43
|
+
return Array.isArray(parsed) ? parsed : [parsed];
|
|
44
|
+
}
|
|
45
|
+
if (Array.isArray(items)) {
|
|
46
|
+
return items.flatMap((item) => typeof item === 'string' ? JSON.parse(item) : item);
|
|
47
|
+
}
|
|
48
|
+
return [items];
|
|
49
|
+
}
|
|
50
|
+
async pushItems(items) {
|
|
51
|
+
const context = pushDataChargingContext.getStore();
|
|
52
|
+
// A single JSON string may encode multiple items (e.g. '[{...},{...}]'),
|
|
53
|
+
// which the charging logic would miscount — parse strings into arrays so
|
|
54
|
+
// each logical item is counted individually.
|
|
55
|
+
const normalizedItems = this.normalizeItems(items);
|
|
56
|
+
const result = await pushDataAndCharge({
|
|
57
|
+
chargingManager: this.getChargingManager(),
|
|
58
|
+
items: normalizedItems,
|
|
59
|
+
eventName: context?.eventName,
|
|
60
|
+
isDefaultDataset: true,
|
|
61
|
+
// stringify for faster validation in the Apify client
|
|
62
|
+
pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)),
|
|
63
|
+
});
|
|
64
|
+
if (!context)
|
|
65
|
+
return;
|
|
66
|
+
// One `Actor.pushData()` may map to several `pushItems()` calls — aggregate.
|
|
67
|
+
context.chargeResult =
|
|
68
|
+
context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
|
|
73
|
+
* `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
|
|
74
|
+
* `StorageBackend` interface (async factory methods accepting an `id`,
|
|
75
|
+
* a `name`, or an `alias`).
|
|
76
|
+
*
|
|
77
|
+
* For the run's default dataset it transparently swaps in a charging-aware
|
|
78
|
+
* dataset client (pay-per-event on `Actor.pushData()`), provided a charging
|
|
79
|
+
* manager is supplied and a default-dataset-item price is configured.
|
|
80
|
+
*
|
|
81
|
+
* `Actor` wires this up automatically; construct it directly only to use Apify
|
|
82
|
+
* platform storage with crawlee's storage classes outside of `Actor` — e.g. to
|
|
83
|
+
* read another run's output with an explicit token:
|
|
84
|
+
*
|
|
85
|
+
* ```ts
|
|
86
|
+
* import { ApifyClient, ApifyStorageBackend, Dataset } from 'apify';
|
|
87
|
+
*
|
|
88
|
+
* const client = new ApifyClient({ token });
|
|
89
|
+
* const dataset = await Dataset.open(datasetId, { storageBackend: new ApifyStorageBackend(client) });
|
|
90
|
+
* const { items } = await dataset.getData();
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
export class ApifyStorageBackend {
|
|
94
|
+
client;
|
|
95
|
+
config;
|
|
96
|
+
requestQueueAccess;
|
|
97
|
+
getChargingManager;
|
|
98
|
+
/** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
|
|
99
|
+
aliasIdCache = new Map();
|
|
100
|
+
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
101
|
+
fallbackClientKey;
|
|
102
|
+
constructor(client, options = {}) {
|
|
103
|
+
this.client = client;
|
|
104
|
+
this.config = options.configuration;
|
|
105
|
+
this.requestQueueAccess = options.requestQueueAccess ?? 'single';
|
|
106
|
+
this.getChargingManager = options.getChargingManager;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
|
|
110
|
+
* opened through two differently-authenticated backends is cached separately. The request
|
|
111
|
+
* queue access mode is deliberately not part of the key — opening the same queue in `single`
|
|
112
|
+
* and `shared` mode at once is not supported, and whichever backend opens it first wins.
|
|
113
|
+
*/
|
|
114
|
+
getStorageBackendCacheKey() {
|
|
115
|
+
const hash = createHash('sha256')
|
|
116
|
+
.update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`)
|
|
117
|
+
.digest('hex')
|
|
118
|
+
.slice(0, 8);
|
|
119
|
+
return `ApifyStorageBackend:${hash}`;
|
|
120
|
+
}
|
|
121
|
+
async storageExists(id, type) {
|
|
122
|
+
// Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one
|
|
123
|
+
// exists on the platform) and fall back to a name otherwise; without this, crawlee would
|
|
124
|
+
// treat every string as a name and silently create a new storage named like the passed id.
|
|
125
|
+
// Apify's `GET /v2/{kind}/{idOrName}` matches by either id or name;
|
|
126
|
+
// confirm it was an *id* match so crawlee can fall through to `{ name }`.
|
|
127
|
+
const info = await this.resourceClient(id, type).get();
|
|
128
|
+
return info?.id === id;
|
|
129
|
+
}
|
|
130
|
+
async createDatasetBackend(options) {
|
|
131
|
+
const id = await this.resolveId(options, 'Dataset');
|
|
132
|
+
const chargingClient = this.chargingDatasetClient(id);
|
|
133
|
+
const backend = new ApifyDatasetBackend(chargingClient ?? this.client.dataset(id));
|
|
134
|
+
if (chargingClient) {
|
|
135
|
+
// `Actor.pushData()` looks for this marker on the dataset's backend to know the
|
|
136
|
+
// pay-per-event charging happens inside the intercepted `pushItems()` calls.
|
|
137
|
+
Object.assign(backend, { [USES_PUSH_DATA_INTERCEPTION]: true });
|
|
138
|
+
}
|
|
139
|
+
return backend;
|
|
140
|
+
}
|
|
141
|
+
async createKeyValueStoreBackend(options) {
|
|
142
|
+
const id = await this.resolveId(options, 'KeyValueStore');
|
|
143
|
+
return new ApifyKeyValueStoreBackend(this.client.keyValueStore(id));
|
|
144
|
+
}
|
|
145
|
+
async createRequestQueueBackend(options) {
|
|
146
|
+
const id = await this.resolveId(options, 'RequestQueue');
|
|
147
|
+
const client = this.client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
|
|
148
|
+
return this.requestQueueAccess === 'shared'
|
|
149
|
+
? new ApifyRequestQueueSharedBackend(client)
|
|
150
|
+
: new ApifyRequestQueueSingleBackend(client);
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* A stable per-run client key makes the API's `hadMultipleClients` flag meaningful and lets a
|
|
154
|
+
* migrated or resurrected run re-acquire the request locks of its previous incarnation.
|
|
155
|
+
*/
|
|
156
|
+
requestQueueClientKey() {
|
|
157
|
+
const key = this.config?.actorRunId ?? (this.fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
|
|
158
|
+
return key.slice(0, MAX_CLIENT_KEY_LENGTH);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Returns a charging-aware dataset client when `id` is the run's default
|
|
162
|
+
* dataset and a default-dataset-item price is configured; otherwise
|
|
163
|
+
* `undefined` (caller uses the plain client).
|
|
164
|
+
*/
|
|
165
|
+
chargingDatasetClient(id) {
|
|
166
|
+
const { getChargingManager } = this;
|
|
167
|
+
if (!getChargingManager)
|
|
168
|
+
return undefined;
|
|
169
|
+
if (id !== this.config?.defaultDatasetId)
|
|
170
|
+
return undefined;
|
|
171
|
+
const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
|
|
172
|
+
if (!hasDefaultDatasetItemEvent)
|
|
173
|
+
return undefined;
|
|
174
|
+
return new PpeAwareDatasetClient({
|
|
175
|
+
id,
|
|
176
|
+
baseUrl: this.client.baseUrl,
|
|
177
|
+
publicBaseUrl: this.client.publicBaseUrl,
|
|
178
|
+
apifyClient: this.client,
|
|
179
|
+
httpClient: this.client.httpClient,
|
|
180
|
+
}, getChargingManager);
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
184
|
+
*
|
|
185
|
+
* Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
|
|
186
|
+
* default storage, and other aliases to the storages declared in the Actor's schema (via the
|
|
187
|
+
* `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the
|
|
188
|
+
* platform, an unnamed storage is created per alias instead (remembered for this process only).
|
|
189
|
+
*/
|
|
190
|
+
async resolveId(options, type) {
|
|
191
|
+
if (options?.id)
|
|
192
|
+
return options.id;
|
|
193
|
+
if (options?.name) {
|
|
194
|
+
return (await this.collectionClient(type).getOrCreate(options.name)).id;
|
|
195
|
+
}
|
|
196
|
+
const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS;
|
|
197
|
+
if (alias === DEFAULT_STORAGE_ALIAS) {
|
|
198
|
+
const defaultId = this.config?.[DEFAULT_ID_CONFIG_KEY[type]];
|
|
199
|
+
if (defaultId)
|
|
200
|
+
return defaultId;
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
const declaredId = this.aliasFromActorStorages(alias, type);
|
|
204
|
+
if (declaredId)
|
|
205
|
+
return declaredId;
|
|
206
|
+
if (this.config?.isAtHome) {
|
|
207
|
+
throw new Error(`Storage alias "${alias}" cannot be resolved because it is not declared in the Actor's schema storages. ` +
|
|
208
|
+
`Declare it in the Actor schema, or open the storage by name instead.`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
// No platform-provided id for this alias (e.g. cloud storage used locally via an API
|
|
212
|
+
// token) — create an unnamed storage for it, one per alias per process.
|
|
213
|
+
const cacheKey = `${type}:${alias}`;
|
|
214
|
+
const cachedId = this.aliasIdCache.get(cacheKey);
|
|
215
|
+
if (cachedId)
|
|
216
|
+
return cachedId;
|
|
217
|
+
const created = await this.collectionClient(type).getOrCreate();
|
|
218
|
+
this.aliasIdCache.set(cacheKey, created.id);
|
|
219
|
+
return created.id;
|
|
220
|
+
}
|
|
221
|
+
/** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
|
|
222
|
+
aliasFromActorStorages(alias, type) {
|
|
223
|
+
const storagesJson = this.config?.actorStoragesJson;
|
|
224
|
+
if (!storagesJson)
|
|
225
|
+
return undefined;
|
|
226
|
+
let storages;
|
|
227
|
+
try {
|
|
228
|
+
storages = JSON.parse(storagesJson);
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
throw new Error(`Failed to parse ACTOR_STORAGES_JSON environment variable: ${storagesJson}`);
|
|
232
|
+
}
|
|
233
|
+
return storages[ACTOR_STORAGES_TYPE_KEY[type]]?.[alias];
|
|
234
|
+
}
|
|
235
|
+
resourceClient(id, type) {
|
|
236
|
+
if (type === 'Dataset')
|
|
237
|
+
return this.client.dataset(id);
|
|
238
|
+
if (type === 'KeyValueStore')
|
|
239
|
+
return this.client.keyValueStore(id);
|
|
240
|
+
return this.client.requestQueue(id);
|
|
241
|
+
}
|
|
242
|
+
collectionClient(type) {
|
|
243
|
+
if (type === 'Dataset')
|
|
244
|
+
return this.client.datasets();
|
|
245
|
+
if (type === 'KeyValueStore')
|
|
246
|
+
return this.client.keyValueStores();
|
|
247
|
+
return this.client.requestQueues();
|
|
248
|
+
}
|
|
249
|
+
}
|
package/dist/configuration.d.ts
CHANGED
|
@@ -47,9 +47,11 @@ export declare const apifyConfigFields: {
|
|
|
47
47
|
actorStoragesJson: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
48
48
|
storageClientOptions: ConfigField<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
49
49
|
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
50
|
+
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
50
51
|
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
|
|
51
52
|
logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof import("@apify/log").LogLevel>>>>;
|
|
52
53
|
persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
54
|
+
storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
53
55
|
containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
|
|
54
56
|
};
|
|
55
57
|
export type ApifyConfigurationInput = FieldsInput<typeof apifyConfigFields>;
|
|
@@ -61,13 +63,13 @@ export interface Configuration extends ApifyResolvedConfigValues {
|
|
|
61
63
|
/**
|
|
62
64
|
* `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
|
|
63
65
|
*
|
|
64
|
-
* 1. When using `Actor` class, we can get the instance configuration via `sdk.
|
|
66
|
+
* 1. When using `Actor` class, we can get the instance configuration via `sdk.configuration`
|
|
65
67
|
*
|
|
66
68
|
* ```javascript
|
|
67
69
|
* import { Actor } from 'apify';
|
|
68
70
|
*
|
|
69
71
|
* const sdk = new Actor({ token: '123' });
|
|
70
|
-
* console.log(sdk.
|
|
72
|
+
* console.log(sdk.configuration.token); // '123'
|
|
71
73
|
* ```
|
|
72
74
|
*
|
|
73
75
|
* 2. To get the global configuration (singleton instance). It will respect the environment variables.
|
|
@@ -75,7 +77,7 @@ export interface Configuration extends ApifyResolvedConfigValues {
|
|
|
75
77
|
* ```javascript
|
|
76
78
|
* import { Configuration } from 'apify';
|
|
77
79
|
*
|
|
78
|
-
* const config = Configuration.
|
|
80
|
+
* const config = Configuration.getGlobalConfiguration();
|
|
79
81
|
* console.log(config.headless);
|
|
80
82
|
* console.log(config.persistStateIntervalMillis);
|
|
81
83
|
* ```
|
|
@@ -139,5 +141,5 @@ export declare class Configuration extends CoreConfiguration {
|
|
|
139
141
|
* what crawlee internals resolve against; this singleton is only the
|
|
140
142
|
* fallback for code reaching for a configuration without an explicit one.
|
|
141
143
|
*/
|
|
142
|
-
static
|
|
144
|
+
static getGlobalConfiguration(): Configuration;
|
|
143
145
|
}
|
package/dist/configuration.js
CHANGED
|
@@ -89,13 +89,13 @@ export const apifyConfigFields = {
|
|
|
89
89
|
/**
|
|
90
90
|
* `Configuration` is a value object holding the SDK configuration. We can use it in two ways:
|
|
91
91
|
*
|
|
92
|
-
* 1. When using `Actor` class, we can get the instance configuration via `sdk.
|
|
92
|
+
* 1. When using `Actor` class, we can get the instance configuration via `sdk.configuration`
|
|
93
93
|
*
|
|
94
94
|
* ```javascript
|
|
95
95
|
* import { Actor } from 'apify';
|
|
96
96
|
*
|
|
97
97
|
* const sdk = new Actor({ token: '123' });
|
|
98
|
-
* console.log(sdk.
|
|
98
|
+
* console.log(sdk.configuration.token); // '123'
|
|
99
99
|
* ```
|
|
100
100
|
*
|
|
101
101
|
* 2. To get the global configuration (singleton instance). It will respect the environment variables.
|
|
@@ -103,7 +103,7 @@ export const apifyConfigFields = {
|
|
|
103
103
|
* ```javascript
|
|
104
104
|
* import { Configuration } from 'apify';
|
|
105
105
|
*
|
|
106
|
-
* const config = Configuration.
|
|
106
|
+
* const config = Configuration.getGlobalConfiguration();
|
|
107
107
|
* console.log(config.headless);
|
|
108
108
|
* console.log(config.persistStateIntervalMillis);
|
|
109
109
|
* ```
|
|
@@ -173,7 +173,7 @@ export class Configuration extends CoreConfiguration {
|
|
|
173
173
|
* what crawlee internals resolve against; this singleton is only the
|
|
174
174
|
* fallback for code reaching for a configuration without an explicit one.
|
|
175
175
|
*/
|
|
176
|
-
static
|
|
176
|
+
static getGlobalConfiguration() {
|
|
177
177
|
Configuration.globalConfig ??= new Configuration();
|
|
178
178
|
return Configuration.globalConfig;
|
|
179
179
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './actor.js';
|
|
2
|
-
export {
|
|
2
|
+
export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_storage_backend.js';
|
|
3
|
+
export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
3
4
|
export { ArgumentValidationError } from './utils.js';
|
|
4
5
|
export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage.js';
|
|
5
6
|
export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
|
|
@@ -7,5 +8,6 @@ export * from './configuration.js';
|
|
|
7
8
|
export * from './proxy_configuration.js';
|
|
8
9
|
export * from './platform_event_manager.js';
|
|
9
10
|
export * from './key_value_store.js';
|
|
10
|
-
export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue,
|
|
11
|
+
export { Dataset, DatasetDataOptions, DatasetIteratorOptions, DatasetConsumer, DatasetMapper, DatasetReducer, DatasetOptions, DatasetContent, RequestQueue, RequestQueueOperationOptions, RequestQueueOptions, KeyConsumer, KeyValueStoreOptions, RecordOptions, KeyValueStoreIteratorOptions, log, Log, LoggerOptions, LogLevel, Logger, LoggerJson, LoggerText, } from '@crawlee/core';
|
|
12
|
+
export type { QueueOperationInfo } from '@crawlee/types';
|
|
11
13
|
export { ApifyClient, ApifyClientOptions } from 'apify-client';
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export * from './actor.js';
|
|
2
|
-
export {
|
|
2
|
+
export { ApifyStorageBackend } from './apify_storage_backend.js';
|
|
3
3
|
export { ArgumentValidationError } from './utils.js';
|
|
4
4
|
export { ChargingManager } from './charging.js';
|
|
5
5
|
export * from './configuration.js';
|
package/dist/input-schemas.d.ts
CHANGED
package/dist/key_value_store.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
|
|
2
|
-
import { KeyValueStoreClient as RemoteKeyValueStoreClient } from 'apify-client';
|
|
1
|
+
import { KeyValueStore as CoreKeyValueStore, serviceLocator } from '@crawlee/core';
|
|
3
2
|
import { createHmacSignature } from '@apify/utilities';
|
|
3
|
+
import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
|
|
4
4
|
/**
|
|
5
5
|
* @inheritDoc
|
|
6
6
|
*/
|
|
@@ -15,18 +15,17 @@ export class KeyValueStore extends CoreKeyValueStore {
|
|
|
15
15
|
* implementation (which produces a `file://` URL or returns `undefined`).
|
|
16
16
|
*/
|
|
17
17
|
async getPublicUrl(key) {
|
|
18
|
-
const config =
|
|
19
|
-
// Detect a remote (Apify) store by its
|
|
18
|
+
const config = serviceLocator.getConfiguration();
|
|
19
|
+
// Detect a remote (Apify) store by its backend type rather than by
|
|
20
20
|
// `isAtHome`, so that a `forceCloud` store opened locally still gets a
|
|
21
|
-
// signed Apify URL (matching the platform behaviour). `
|
|
21
|
+
// signed Apify URL (matching the platform behaviour). `backend` is
|
|
22
22
|
// `private` on `CoreKeyValueStore`, so bypass the visibility check.
|
|
23
|
-
const {
|
|
24
|
-
|
|
25
|
-
if (isLocalStore) {
|
|
23
|
+
const { backend } = this;
|
|
24
|
+
if (!(backend instanceof ApifyKeyValueStoreBackend)) {
|
|
26
25
|
return super.getPublicUrl(key);
|
|
27
26
|
}
|
|
28
27
|
const publicUrl = new URL(`${config.apiPublicBaseUrl}/v2/key-value-stores/${this.id}/records/${key}`);
|
|
29
|
-
const metadata = (await
|
|
28
|
+
const metadata = (await backend.getMetadata());
|
|
30
29
|
if (metadata?.urlSigningSecretKey) {
|
|
31
30
|
publicUrl.searchParams.append('signature', createHmacSignature(metadata.urlSigningSecretKey, key));
|
|
32
31
|
}
|
|
@@ -39,10 +39,10 @@ import { Configuration } from './configuration.js';
|
|
|
39
39
|
* you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
|
|
40
40
|
*/
|
|
41
41
|
export declare class PlatformEventManager extends EventManager {
|
|
42
|
-
readonly
|
|
42
|
+
readonly configuration: Configuration;
|
|
43
43
|
/** Websocket connection to Actor events. */
|
|
44
44
|
private eventsWs?;
|
|
45
|
-
constructor(
|
|
45
|
+
constructor(configuration?: Configuration);
|
|
46
46
|
/**
|
|
47
47
|
* Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
|
|
48
48
|
* This is an internal function that is automatically called by `Actor.main()`.
|
|
@@ -42,14 +42,14 @@ import { Configuration } from './configuration.js';
|
|
|
42
42
|
* you can achieve the same effect using `setInterval()` and listening for the `migrating` event.
|
|
43
43
|
*/
|
|
44
44
|
export class PlatformEventManager extends EventManager {
|
|
45
|
-
|
|
45
|
+
configuration;
|
|
46
46
|
/** Websocket connection to Actor events. */
|
|
47
47
|
eventsWs;
|
|
48
|
-
constructor(
|
|
48
|
+
constructor(configuration = Configuration.getGlobalConfiguration()) {
|
|
49
49
|
super({
|
|
50
|
-
persistStateIntervalMillis:
|
|
50
|
+
persistStateIntervalMillis: configuration.persistStateIntervalMillis,
|
|
51
51
|
});
|
|
52
|
-
this.
|
|
52
|
+
this.configuration = configuration;
|
|
53
53
|
}
|
|
54
54
|
/**
|
|
55
55
|
* Initializes `Actor.events` event emitter by creating a connection to a websocket that provides them.
|
|
@@ -60,7 +60,7 @@ export class PlatformEventManager extends EventManager {
|
|
|
60
60
|
return;
|
|
61
61
|
}
|
|
62
62
|
await super.init();
|
|
63
|
-
const eventsWsUrl = this.
|
|
63
|
+
const eventsWsUrl = this.configuration.actorEventsWsUrl;
|
|
64
64
|
// Locally there is no web socket to connect, so just print a log message.
|
|
65
65
|
if (!eventsWsUrl) {
|
|
66
66
|
this.log.debug(`Environment variable ${ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL} is not set, no events from Apify platform will be emitted.`);
|
|
@@ -2,6 +2,12 @@ import type { ProxyConfigurationOptions as CoreProxyConfigurationOptions } from
|
|
|
2
2
|
import { ProxyConfiguration as CoreProxyConfiguration } from '@crawlee/core';
|
|
3
3
|
import type { ProxyInfo as CoreProxyInfo } from '@crawlee/types';
|
|
4
4
|
import { Configuration } from './configuration.js';
|
|
5
|
+
/** Response of the Apify Proxy status endpoint (`proxy.apify.com/?format=json`). */
|
|
6
|
+
interface ProxyStatus {
|
|
7
|
+
connected: boolean;
|
|
8
|
+
connectionError: string;
|
|
9
|
+
isManInTheMiddle: boolean;
|
|
10
|
+
}
|
|
5
11
|
type NewUrlOptions = Parameters<CoreProxyConfiguration['newProxyInfo']>[0];
|
|
6
12
|
export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions {
|
|
7
13
|
/**
|
|
@@ -145,7 +151,7 @@ export interface ProxyInfo extends CoreProxyInfo {
|
|
|
145
151
|
* @category Scaling
|
|
146
152
|
*/
|
|
147
153
|
export declare class ProxyConfiguration extends CoreProxyConfiguration {
|
|
148
|
-
readonly
|
|
154
|
+
readonly configuration: Configuration;
|
|
149
155
|
private groups;
|
|
150
156
|
private countryCode?;
|
|
151
157
|
private subdivisionCode?;
|
|
@@ -153,10 +159,11 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
153
159
|
private hostname;
|
|
154
160
|
private port?;
|
|
155
161
|
private usesApifyProxy?;
|
|
162
|
+
protected readonly log: import("@apify/log").Log;
|
|
156
163
|
/**
|
|
157
164
|
* @internal
|
|
158
165
|
*/
|
|
159
|
-
constructor(options?: ProxyConfigurationOptions,
|
|
166
|
+
constructor(options?: ProxyConfigurationOptions, configuration?: Configuration);
|
|
160
167
|
/**
|
|
161
168
|
* Loads proxy password if token is provided and checks access to Apify Proxy and provided proxy groups
|
|
162
169
|
* if Apify Proxy configuration is used.
|
|
@@ -198,15 +205,25 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
198
205
|
/**
|
|
199
206
|
* Apify Proxy can be down for a second or a minute, but this should not crash processes.
|
|
200
207
|
*/
|
|
201
|
-
protected _fetchStatus(): Promise<
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
protected _fetchStatus(): Promise<ProxyStatus | undefined>;
|
|
209
|
+
/**
|
|
210
|
+
* Fetches the Apify Proxy status endpoint once, *through* the proxy, so the
|
|
211
|
+
* response reports on this exact connection (auth + man-in-the-middle).
|
|
212
|
+
*
|
|
213
|
+
* Uses a native `node:http` forward-proxy request — an absolute request URL
|
|
214
|
+
* plus a `Proxy-Authorization` header — so no proxy-agent dependency is
|
|
215
|
+
* needed. The status endpoint (`http://proxy.apify.com`) is plain HTTP.
|
|
216
|
+
*/
|
|
217
|
+
protected _requestStatus(statusUrl: string, proxyUrl: string): Promise<ProxyStatus>;
|
|
206
218
|
/**
|
|
207
219
|
* Throws cannot combine custom proxies with Apify Proxy
|
|
208
220
|
* @internal
|
|
209
221
|
*/
|
|
210
222
|
protected _throwCannotCombineCustomWithApify(): void;
|
|
223
|
+
/**
|
|
224
|
+
* Throws cannot combine custom proxies with custom generating function
|
|
225
|
+
* @internal
|
|
226
|
+
*/
|
|
227
|
+
protected _throwCannotCombineCustomMethods(): void;
|
|
211
228
|
}
|
|
212
229
|
export {};
|