apify 4.0.0-beta.38 → 4.0.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/actor.d.ts +68 -57
- package/dist/actor.js +142 -70
- package/dist/apify_dataset_backend.js +1 -1
- package/dist/apify_file_system_storage_backend.d.ts +34 -0
- package/dist/apify_file_system_storage_backend.js +41 -0
- package/dist/apify_key_value_store_backend.js +3 -1
- package/dist/apify_request_queue_backend.js +1 -1
- package/dist/charging.js +2 -2
- package/dist/configuration.d.ts +7 -7
- package/dist/configuration.js +7 -1
- package/dist/enqueue_links_filters.d.ts +2 -2
- package/dist/enqueue_links_filters.js +1 -1
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +22 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/proxy_configuration.d.ts +3 -4
- package/dist/proxy_configuration.js +5 -5
- package/dist/utils.d.ts +11 -0
- package/dist/utils.js +23 -0
- package/package.json +7 -6
package/README.md
CHANGED
package/dist/actor.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { EventManager, EventTypeName, RecordOptions, UseStateOptions } from '@crawlee/core';
|
|
2
2
|
import { Dataset, KeyValueStore, RequestQueue } from '@crawlee/core';
|
|
3
3
|
import type { Awaitable, Dictionary, StorageBackend } from '@crawlee/types';
|
|
4
|
-
import type { ActorCallOptions, ActorStartOptions, ApifyClientOptions, RunAbortOptions, TaskCallOptions, Webhook, WebhookEventType } from 'apify-client';
|
|
4
|
+
import type { ActorCallOptions, ActorInput, ActorStartOptions, ApifyClientOptions, RunAbortOptions, TaskCallOptions, Webhook, WebhookEventType } from 'apify-client';
|
|
5
5
|
import { ActorRun as ClientActorRun, ApifyClient } from 'apify-client';
|
|
6
6
|
import { type ACTOR_PERMISSION_LEVEL } from '@apify/consts';
|
|
7
7
|
import type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
@@ -222,11 +222,11 @@ export interface Timeout {
|
|
|
222
222
|
*/
|
|
223
223
|
timeout?: number | 'inherit';
|
|
224
224
|
}
|
|
225
|
-
export interface CallOptions extends Omit<ActorCallOptions, '
|
|
225
|
+
export interface CallOptions extends Omit<ActorCallOptions, 'runTimeoutSecs'>, Token, Timeout {
|
|
226
226
|
}
|
|
227
|
-
export interface StartOptions extends Omit<ActorStartOptions, 'waitForFinish' | '
|
|
227
|
+
export interface StartOptions extends Omit<ActorStartOptions, 'waitForFinish' | 'runTimeoutSecs'>, Token, Timeout {
|
|
228
228
|
}
|
|
229
|
-
export interface CallTaskOptions extends Omit<TaskCallOptions, '
|
|
229
|
+
export interface CallTaskOptions extends Omit<TaskCallOptions, 'runTimeoutSecs'>, Token, Timeout {
|
|
230
230
|
}
|
|
231
231
|
export interface AbortOptions extends RunAbortOptions, Token {
|
|
232
232
|
/** Exit with given status message */
|
|
@@ -293,7 +293,7 @@ export interface MetamorphOptions {
|
|
|
293
293
|
* Content type for the `input`. If not specified,
|
|
294
294
|
* `input` is expected to be an object that will be stringified to JSON and content type set to
|
|
295
295
|
* `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a
|
|
296
|
-
* `
|
|
296
|
+
* `Buffer`.
|
|
297
297
|
*/
|
|
298
298
|
contentType?: string;
|
|
299
299
|
/**
|
|
@@ -446,13 +446,12 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
446
446
|
* @param actorId
|
|
447
447
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
448
448
|
* @param [input]
|
|
449
|
-
* Input for the Actor.
|
|
450
|
-
*
|
|
451
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
449
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
450
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
452
451
|
* @param [options]
|
|
453
452
|
* @ignore
|
|
454
453
|
*/
|
|
455
|
-
call(actorId: string, input?:
|
|
454
|
+
call(actorId: string, input?: ActorInput, options?: CallOptions): Promise<ClientActorRun>;
|
|
456
455
|
/**
|
|
457
456
|
* Runs an Actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
458
457
|
* unlike `Actor.call`, this method just starts the run without waiting for finish.
|
|
@@ -471,13 +470,12 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
471
470
|
* @param actorId
|
|
472
471
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
473
472
|
* @param [input]
|
|
474
|
-
* Input for the Actor.
|
|
475
|
-
*
|
|
476
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
473
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
474
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
477
475
|
* @param [options]
|
|
478
476
|
* @ignore
|
|
479
477
|
*/
|
|
480
|
-
start(actorId: string, input?:
|
|
478
|
+
start(actorId: string, input?: ActorInput, options?: StartOptions): Promise<ClientActorRun>;
|
|
481
479
|
/**
|
|
482
480
|
* Aborts given Actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
483
481
|
*
|
|
@@ -529,13 +527,12 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
529
527
|
* @param targetActorId
|
|
530
528
|
* Either `username/actor-name` or Actor ID of an Actor to which we want to metamorph.
|
|
531
529
|
* @param [input]
|
|
532
|
-
* Input for the Actor.
|
|
533
|
-
*
|
|
534
|
-
* Otherwise, the `options.contentType` parameter must be provided.
|
|
530
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
531
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
535
532
|
* @param [options]
|
|
536
533
|
* @ignore
|
|
537
534
|
*/
|
|
538
|
-
metamorph(targetActorId: string, input?:
|
|
535
|
+
metamorph(targetActorId: string, input?: ActorInput, options?: MetamorphOptions): Promise<void>;
|
|
539
536
|
/**
|
|
540
537
|
* Internally reboots this Actor. The system stops the current container and starts
|
|
541
538
|
* a new container with the same run ID.
|
|
@@ -676,17 +673,17 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
676
673
|
/**
|
|
677
674
|
* Gets the Actor input value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
678
675
|
*
|
|
679
|
-
*
|
|
680
|
-
*
|
|
681
|
-
*
|
|
682
|
-
* const input = await Actor.getInput();
|
|
683
|
-
* ```
|
|
676
|
+
* The input is the record stored under the configured input key (`ACTOR_INPUT_KEY`, default `INPUT`).
|
|
677
|
+
* A record stored as `application/octet-stream` — which is what a local input file without an
|
|
678
|
+
* extension is read as — is parsed with JSON5 when it parses, and returned as a `Buffer` otherwise.
|
|
684
679
|
*
|
|
685
|
-
* is
|
|
686
|
-
*
|
|
687
|
-
*
|
|
688
|
-
*
|
|
689
|
-
*
|
|
680
|
+
* When running locally and the store holds no such record, the input is read from a `<inputKey>` or
|
|
681
|
+
* `<inputKey>.json` file in the current working directory instead.
|
|
682
|
+
*
|
|
683
|
+
* Throws an {@link ActorInputError} when no input is found (`code: 'NOT_FOUND'`), when both input
|
|
684
|
+
* files exist in the working directory (`'MULTIPLE_FILES'`), when a JSON input does not parse
|
|
685
|
+
* (`'PARSE_FAILED'`) or when its secret fields cannot be decrypted (`'DECRYPTION_FAILED'`). If your
|
|
686
|
+
* Actor can run without input, catch the `'NOT_FOUND'` case and rethrow the rest.
|
|
690
687
|
*
|
|
691
688
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
692
689
|
* If you need to use the input multiple times in your Actor,
|
|
@@ -698,16 +695,20 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
698
695
|
* @returns
|
|
699
696
|
* Returns a promise that resolves to an object, string
|
|
700
697
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
701
|
-
* on the MIME content type of the record
|
|
702
|
-
*
|
|
698
|
+
* on the MIME content type of the record.
|
|
699
|
+
* @throws {ActorInputError} When the input cannot be produced; see `code`.
|
|
703
700
|
* @ignore
|
|
704
701
|
*/
|
|
705
|
-
getInput<T = Dictionary | string | Buffer>(): Promise<T
|
|
702
|
+
getInput<T = Dictionary | string | Buffer>(): Promise<T>;
|
|
706
703
|
/**
|
|
707
|
-
*
|
|
708
|
-
*
|
|
704
|
+
* Reads the input from a `<inputKey>` or `<inputKey>.json` file in the working directory, for local runs
|
|
705
|
+
* whose default key-value store holds no input record. Mirrors how `ApifyFileSystemStorageBackend` adopts
|
|
706
|
+
* such files inside the store directory: the `.json` file is JSON, the bare one is bytes that
|
|
707
|
+
* {@link parseInputValue} tries as JSON5. Both files present is an error rather than a guess.
|
|
708
|
+
*
|
|
709
|
+
* @returns `undefined` when running on the platform or when neither file exists.
|
|
709
710
|
*/
|
|
710
|
-
|
|
711
|
+
private readInputFromWorkingDirectory;
|
|
711
712
|
/**
|
|
712
713
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
713
714
|
*
|
|
@@ -978,12 +979,11 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
978
979
|
* @param actorId
|
|
979
980
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
980
981
|
* @param [input]
|
|
981
|
-
* Input for the Actor.
|
|
982
|
-
*
|
|
983
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
982
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
983
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
984
984
|
* @param [options]
|
|
985
985
|
*/
|
|
986
|
-
static call(actorId: string, input?:
|
|
986
|
+
static call(actorId: string, input?: ActorInput, options?: CallOptions): Promise<ClientActorRun>;
|
|
987
987
|
/**
|
|
988
988
|
* Runs an Actor task on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
989
989
|
*
|
|
@@ -1028,9 +1028,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1028
1028
|
* @param actorId
|
|
1029
1029
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
1030
1030
|
* @param [input]
|
|
1031
|
-
* Input for the Actor.
|
|
1032
|
-
*
|
|
1033
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
1031
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
1032
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
1034
1033
|
* @param [options]
|
|
1035
1034
|
*/
|
|
1036
1035
|
static start(actorId: string, input?: Dictionary, options?: StartOptions): Promise<ClientActorRun>;
|
|
@@ -1057,12 +1056,11 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1057
1056
|
* @param targetActorId
|
|
1058
1057
|
* Either `username/actor-name` or Actor ID of an Actor to which we want to metamorph.
|
|
1059
1058
|
* @param [input]
|
|
1060
|
-
* Input for the Actor.
|
|
1061
|
-
*
|
|
1062
|
-
* Otherwise, the `options.contentType` parameter must be provided.
|
|
1059
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
1060
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
1063
1061
|
* @param [options]
|
|
1064
1062
|
*/
|
|
1065
|
-
static metamorph(targetActorId: string, input?:
|
|
1063
|
+
static metamorph(targetActorId: string, input?: ActorInput, options?: MetamorphOptions): Promise<void>;
|
|
1066
1064
|
/**
|
|
1067
1065
|
* Internally reboots this Actor run. The system stops the current container and starts
|
|
1068
1066
|
* a new container with the same run id.
|
|
@@ -1224,16 +1222,28 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1224
1222
|
/**
|
|
1225
1223
|
* Gets the Actor input value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
1226
1224
|
*
|
|
1227
|
-
*
|
|
1228
|
-
* For example, calling the following code:
|
|
1225
|
+
* The input is the record stored under the configured input key (`ACTOR_INPUT_KEY`, default `INPUT`):
|
|
1229
1226
|
* ```js
|
|
1230
1227
|
* const input = await Actor.getInput();
|
|
1231
1228
|
* ```
|
|
1232
1229
|
*
|
|
1233
|
-
* is
|
|
1230
|
+
* A record stored as `application/octet-stream` — which is what a local input file without an
|
|
1231
|
+
* extension is read as — is parsed with JSON5 when it parses, and returned as a `Buffer` otherwise.
|
|
1232
|
+
*
|
|
1233
|
+
* When running locally and the store holds no such record, the input is read from a `<inputKey>` or
|
|
1234
|
+
* `<inputKey>.json` file in the current working directory instead.
|
|
1235
|
+
*
|
|
1236
|
+
* Throws an {@link ActorInputError} when no input is found (`code: 'NOT_FOUND'`), when both input
|
|
1237
|
+
* files exist in the working directory (`'MULTIPLE_FILES'`), when a JSON input does not parse
|
|
1238
|
+
* (`'PARSE_FAILED'`) or when its secret fields cannot be decrypted (`'DECRYPTION_FAILED'`). If your
|
|
1239
|
+
* Actor can run without input, catch the `'NOT_FOUND'` case and rethrow the rest:
|
|
1234
1240
|
* ```js
|
|
1235
|
-
*
|
|
1236
|
-
*
|
|
1241
|
+
* let input = {};
|
|
1242
|
+
* try {
|
|
1243
|
+
* input = await Actor.getInput();
|
|
1244
|
+
* } catch (error) {
|
|
1245
|
+
* if (!(error instanceof ActorInputError) || error.code !== 'NOT_FOUND') throw error;
|
|
1246
|
+
* }
|
|
1237
1247
|
* ```
|
|
1238
1248
|
*
|
|
1239
1249
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
@@ -1245,15 +1255,10 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1245
1255
|
* @returns
|
|
1246
1256
|
* Returns a promise that resolves to an object, string
|
|
1247
1257
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
1248
|
-
* on the MIME content type of the record
|
|
1249
|
-
*
|
|
1258
|
+
* on the MIME content type of the record.
|
|
1259
|
+
* @throws {ActorInputError} When the input cannot be produced; see `code`.
|
|
1250
1260
|
*/
|
|
1251
|
-
static getInput<T = Dictionary | string | Buffer>(): Promise<T
|
|
1252
|
-
/**
|
|
1253
|
-
* Gets the Actor input value just like the {@link Actor.getInput} method,
|
|
1254
|
-
* but throws if it is not found.
|
|
1255
|
-
*/
|
|
1256
|
-
static getInputOrThrow<T = Dictionary | string | Buffer>(): Promise<T>;
|
|
1261
|
+
static getInput<T = Dictionary | string | Buffer>(): Promise<T>;
|
|
1257
1262
|
/**
|
|
1258
1263
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
1259
1264
|
*
|
|
@@ -1381,6 +1386,12 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1381
1386
|
* @internal
|
|
1382
1387
|
*/
|
|
1383
1388
|
static setDefaultInstance(instance?: Actor): void;
|
|
1389
|
+
/**
|
|
1390
|
+
* The same choice crawlee's `ServiceLocator` makes for its implicit default backend, with the
|
|
1391
|
+
* input-aware {@link ApifyFileSystemStorageBackend} in place of crawlee's — crawlee has no
|
|
1392
|
+
* notion of a run input.
|
|
1393
|
+
*/
|
|
1394
|
+
private createLocalStorageBackend;
|
|
1384
1395
|
/**
|
|
1385
1396
|
* crawlee's service locator is set-once, so a backend registered before `init()` wins and the
|
|
1386
1397
|
* Actor cannot install its own over it. Replaces the resulting `ServiceConflictError` with the
|
package/dist/actor.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createPrivateKey } from 'node:crypto';
|
|
2
|
-
import {
|
|
2
|
+
import { readFile, stat } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { Dataset, EventType, KeyValueStore, MemoryStorageBackend, purgeDefaultStorages, rejectOperationInTransaction, RequestQueue, serviceLocator, } from '@crawlee/core';
|
|
3
5
|
import { sleep } from '@crawlee/utils';
|
|
4
6
|
import { ApifyClient } from 'apify-client';
|
|
5
7
|
import { z } from 'zod';
|
|
@@ -8,15 +10,29 @@ import { decryptInputSecrets } from '@apify/input_secrets';
|
|
|
8
10
|
import log from '@apify/log';
|
|
9
11
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
10
12
|
import { parseArgument } from '@apify/validations';
|
|
13
|
+
import { ApifyFileSystemStorageBackend } from './apify_file_system_storage_backend.js';
|
|
11
14
|
import { ApifyStorageBackend } from './apify_storage_backend.js';
|
|
12
15
|
import { ChargingManager, DEFAULT_DATASET_ITEM_EVENT } from './charging.js';
|
|
13
16
|
import { ChargingDatasetBackend, ChargingStorageBackend } from './charging_storage_backend.js';
|
|
14
17
|
import { Configuration } from './configuration.js';
|
|
18
|
+
import { ActorInputError } from './errors.js';
|
|
15
19
|
import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
|
|
16
20
|
import { PlatformEventManager } from './platform_event_manager.js';
|
|
17
21
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
18
22
|
import { SmartApifyStorageBackend } from './smart_apify_storage_backend.js';
|
|
19
|
-
import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
|
|
23
|
+
import { BINARY_CONTENT_TYPE, checkCrawleeVersion, getSystemInfo, isNonEmptyObject, JSON_CONTENT_TYPE, parseInputValue, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
|
|
24
|
+
/**
|
|
25
|
+
* {@link parseInputValue} with the parser error wrapped as an {@link ActorInputError}, so a malformed
|
|
26
|
+
* JSON input reports where it came from.
|
|
27
|
+
*/
|
|
28
|
+
function parseInput(value, contentType, source) {
|
|
29
|
+
try {
|
|
30
|
+
return parseInputValue(value, contentType);
|
|
31
|
+
}
|
|
32
|
+
catch (cause) {
|
|
33
|
+
throw new ActorInputError('PARSE_FAILED', `The input in ${source} is not valid JSON.`, { cause });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
20
36
|
/**
|
|
21
37
|
* Exit codes for the Actor process.
|
|
22
38
|
* The error codes must be in the range 1-128, to avoid collision with signal exits
|
|
@@ -378,17 +394,16 @@ export class Actor {
|
|
|
378
394
|
* @param actorId
|
|
379
395
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
380
396
|
* @param [input]
|
|
381
|
-
* Input for the Actor.
|
|
382
|
-
*
|
|
383
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
397
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
398
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
384
399
|
* @param [options]
|
|
385
400
|
* @ignore
|
|
386
401
|
*/
|
|
387
402
|
async call(actorId, input, options = {}) {
|
|
388
|
-
const
|
|
389
|
-
const { token, ...rest } = options;
|
|
403
|
+
const runTimeoutSecs = options.timeout === 'inherit' ? this.getRemainingTimeSecs() : options.timeout;
|
|
404
|
+
const { token, timeout: _timeout, ...rest } = options;
|
|
390
405
|
const client = token ? this.newClient({ token }) : this.apifyClient;
|
|
391
|
-
return client.actor(actorId).call(input, { ...rest,
|
|
406
|
+
return client.actor(actorId).call(input, { ...rest, runTimeoutSecs });
|
|
392
407
|
}
|
|
393
408
|
/**
|
|
394
409
|
* Runs an Actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
|
|
@@ -408,17 +423,16 @@ export class Actor {
|
|
|
408
423
|
* @param actorId
|
|
409
424
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
410
425
|
* @param [input]
|
|
411
|
-
* Input for the Actor.
|
|
412
|
-
*
|
|
413
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
426
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
427
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
414
428
|
* @param [options]
|
|
415
429
|
* @ignore
|
|
416
430
|
*/
|
|
417
431
|
async start(actorId, input, options = {}) {
|
|
418
|
-
const
|
|
419
|
-
const { token, ...rest } = options;
|
|
432
|
+
const runTimeoutSecs = options.timeout === 'inherit' ? this.getRemainingTimeSecs() : options.timeout;
|
|
433
|
+
const { token, timeout: _timeout, ...rest } = options;
|
|
420
434
|
const client = token ? this.newClient({ token }) : this.apifyClient;
|
|
421
|
-
return client.actor(actorId).start(input, { ...rest,
|
|
435
|
+
return client.actor(actorId).start(input, { ...rest, runTimeoutSecs });
|
|
422
436
|
}
|
|
423
437
|
/**
|
|
424
438
|
* Aborts given Actor run on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable).
|
|
@@ -472,10 +486,10 @@ export class Actor {
|
|
|
472
486
|
* @ignore
|
|
473
487
|
*/
|
|
474
488
|
async callTask(taskId, input, options = {}) {
|
|
475
|
-
const
|
|
476
|
-
const { token, ...rest } = options;
|
|
489
|
+
const runTimeoutSecs = options.timeout === 'inherit' ? this.getRemainingTimeSecs() : options.timeout;
|
|
490
|
+
const { token, timeout: _timeout, ...rest } = options;
|
|
477
491
|
const client = token ? this.newClient({ token }) : this.apifyClient;
|
|
478
|
-
return client.task(taskId).call(input, { ...rest,
|
|
492
|
+
return client.task(taskId).call(input, { ...rest, runTimeoutSecs });
|
|
479
493
|
}
|
|
480
494
|
/**
|
|
481
495
|
* Transforms this Actor run to an Actor run of a given Actor. The system stops the current container and starts
|
|
@@ -485,9 +499,8 @@ export class Actor {
|
|
|
485
499
|
* @param targetActorId
|
|
486
500
|
* Either `username/actor-name` or Actor ID of an Actor to which we want to metamorph.
|
|
487
501
|
* @param [input]
|
|
488
|
-
* Input for the Actor.
|
|
489
|
-
*
|
|
490
|
-
* Otherwise, the `options.contentType` parameter must be provided.
|
|
502
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
503
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
491
504
|
* @param [options]
|
|
492
505
|
* @ignore
|
|
493
506
|
*/
|
|
@@ -800,17 +813,17 @@ export class Actor {
|
|
|
800
813
|
/**
|
|
801
814
|
* Gets the Actor input value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
802
815
|
*
|
|
803
|
-
*
|
|
804
|
-
*
|
|
805
|
-
*
|
|
806
|
-
* const input = await Actor.getInput();
|
|
807
|
-
* ```
|
|
816
|
+
* The input is the record stored under the configured input key (`ACTOR_INPUT_KEY`, default `INPUT`).
|
|
817
|
+
* A record stored as `application/octet-stream` — which is what a local input file without an
|
|
818
|
+
* extension is read as — is parsed with JSON5 when it parses, and returned as a `Buffer` otherwise.
|
|
808
819
|
*
|
|
809
|
-
* is
|
|
810
|
-
*
|
|
811
|
-
*
|
|
812
|
-
*
|
|
813
|
-
*
|
|
820
|
+
* When running locally and the store holds no such record, the input is read from a `<inputKey>` or
|
|
821
|
+
* `<inputKey>.json` file in the current working directory instead.
|
|
822
|
+
*
|
|
823
|
+
* Throws an {@link ActorInputError} when no input is found (`code: 'NOT_FOUND'`), when both input
|
|
824
|
+
* files exist in the working directory (`'MULTIPLE_FILES'`), when a JSON input does not parse
|
|
825
|
+
* (`'PARSE_FAILED'`) or when its secret fields cannot be decrypted (`'DECRYPTION_FAILED'`). If your
|
|
826
|
+
* Actor can run without input, catch the `'NOT_FOUND'` case and rethrow the rest.
|
|
814
827
|
*
|
|
815
828
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
816
829
|
* If you need to use the input multiple times in your Actor,
|
|
@@ -822,21 +835,39 @@ export class Actor {
|
|
|
822
835
|
* @returns
|
|
823
836
|
* Returns a promise that resolves to an object, string
|
|
824
837
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
825
|
-
* on the MIME content type of the record
|
|
826
|
-
*
|
|
838
|
+
* on the MIME content type of the record.
|
|
839
|
+
* @throws {ActorInputError} When the input cannot be produced; see `code`.
|
|
827
840
|
* @ignore
|
|
828
841
|
*/
|
|
829
842
|
async getInput() {
|
|
830
843
|
this.ensureActorInit('getInput');
|
|
831
|
-
const { inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.configuration;
|
|
832
|
-
const
|
|
844
|
+
const { inputKey, inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.configuration;
|
|
845
|
+
const store = await this.openKeyValueStore();
|
|
846
|
+
const record = await store.getRecord(inputKey);
|
|
847
|
+
const rawInput = record
|
|
848
|
+
? parseInput(record.value, record.contentType, `the "${inputKey}" record of the default key-value store`)
|
|
849
|
+
: await this.readInputFromWorkingDirectory(inputKey);
|
|
850
|
+
if (rawInput == null) {
|
|
851
|
+
const locations = [`the "${inputKey}" record of the default key-value store`];
|
|
852
|
+
if (!this.configuration.isAtHome) {
|
|
853
|
+
locations.push(`a "${inputKey}" or "${inputKey}.json" file in the working directory`);
|
|
854
|
+
}
|
|
855
|
+
throw new ActorInputError('NOT_FOUND', `Input does not exist. Expected ${locations.join(' or ')}.`);
|
|
856
|
+
}
|
|
833
857
|
let input = rawInput;
|
|
834
858
|
if (isNonEmptyObject(rawInput) && inputSecretsPrivateKeyFile && inputSecretsPrivateKeyPassphrase) {
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
859
|
+
try {
|
|
860
|
+
const privateKey = createPrivateKey({
|
|
861
|
+
key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'),
|
|
862
|
+
passphrase: inputSecretsPrivateKeyPassphrase,
|
|
863
|
+
});
|
|
864
|
+
input = decryptInputSecrets({ input: rawInput, privateKey });
|
|
865
|
+
}
|
|
866
|
+
catch (cause) {
|
|
867
|
+
throw new ActorInputError('DECRYPTION_FAILED', 'Failed to decrypt the secret fields of the input.', {
|
|
868
|
+
cause,
|
|
869
|
+
});
|
|
870
|
+
}
|
|
840
871
|
}
|
|
841
872
|
if (isNonEmptyObject(input) && !Buffer.isBuffer(input)) {
|
|
842
873
|
input = await this.inferDefaultsFromInputSchema(input);
|
|
@@ -844,15 +875,37 @@ export class Actor {
|
|
|
844
875
|
return input;
|
|
845
876
|
}
|
|
846
877
|
/**
|
|
847
|
-
*
|
|
848
|
-
*
|
|
878
|
+
* Reads the input from a `<inputKey>` or `<inputKey>.json` file in the working directory, for local runs
|
|
879
|
+
* whose default key-value store holds no input record. Mirrors how `ApifyFileSystemStorageBackend` adopts
|
|
880
|
+
* such files inside the store directory: the `.json` file is JSON, the bare one is bytes that
|
|
881
|
+
* {@link parseInputValue} tries as JSON5. Both files present is an error rather than a guess.
|
|
882
|
+
*
|
|
883
|
+
* @returns `undefined` when running on the platform or when neither file exists.
|
|
849
884
|
*/
|
|
850
|
-
async
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
throw new Error('Input does not exist');
|
|
885
|
+
async readInputFromWorkingDirectory(inputKey) {
|
|
886
|
+
if (this.configuration.isAtHome) {
|
|
887
|
+
return undefined;
|
|
854
888
|
}
|
|
855
|
-
|
|
889
|
+
const candidates = [
|
|
890
|
+
{ filename: inputKey, contentType: BINARY_CONTENT_TYPE },
|
|
891
|
+
{ filename: `${inputKey}.json`, contentType: JSON_CONTENT_TYPE },
|
|
892
|
+
];
|
|
893
|
+
const found = [];
|
|
894
|
+
for (const candidate of candidates) {
|
|
895
|
+
const path = join(process.cwd(), candidate.filename);
|
|
896
|
+
const stats = await stat(path).catch(() => null);
|
|
897
|
+
if (stats?.isFile()) {
|
|
898
|
+
found.push({ ...candidate, path });
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
if (found.length > 1) {
|
|
902
|
+
throw new ActorInputError('MULTIPLE_FILES', `Found multiple input files in the working directory: ${found.map((file) => `"${file.filename}"`).join(', ')}. Keep only one of them.`);
|
|
903
|
+
}
|
|
904
|
+
if (found.length === 0) {
|
|
905
|
+
return undefined;
|
|
906
|
+
}
|
|
907
|
+
const [file] = found;
|
|
908
|
+
return parseInput(await readFile(file.path), file.contentType, `the "${file.filename}" file in the working directory`);
|
|
856
909
|
}
|
|
857
910
|
/**
|
|
858
911
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
@@ -1224,9 +1277,8 @@ export class Actor {
|
|
|
1224
1277
|
* @param actorId
|
|
1225
1278
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
1226
1279
|
* @param [input]
|
|
1227
|
-
* Input for the Actor.
|
|
1228
|
-
*
|
|
1229
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
1280
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
1281
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
1230
1282
|
* @param [options]
|
|
1231
1283
|
*/
|
|
1232
1284
|
static async call(actorId, input, options = {}) {
|
|
@@ -1278,9 +1330,8 @@ export class Actor {
|
|
|
1278
1330
|
* @param actorId
|
|
1279
1331
|
* Allowed formats are `username/actor-name`, `userId/actor-name` or Actor ID.
|
|
1280
1332
|
* @param [input]
|
|
1281
|
-
* Input for the Actor.
|
|
1282
|
-
*
|
|
1283
|
-
* Otherwise the `options.contentType` parameter must be provided.
|
|
1333
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
1334
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
1284
1335
|
* @param [options]
|
|
1285
1336
|
*/
|
|
1286
1337
|
static async start(actorId, input, options = {}) {
|
|
@@ -1311,9 +1362,8 @@ export class Actor {
|
|
|
1311
1362
|
* @param targetActorId
|
|
1312
1363
|
* Either `username/actor-name` or Actor ID of an Actor to which we want to metamorph.
|
|
1313
1364
|
* @param [input]
|
|
1314
|
-
* Input for the Actor.
|
|
1315
|
-
*
|
|
1316
|
-
* Otherwise, the `options.contentType` parameter must be provided.
|
|
1365
|
+
* Input for the Actor. A plain object is stringified to JSON with content type
|
|
1366
|
+
* `application/json; charset=utf-8`. A `Buffer` is sent as-is and needs `options.contentType`.
|
|
1317
1367
|
* @param [options]
|
|
1318
1368
|
*/
|
|
1319
1369
|
static async metamorph(targetActorId, input, options = {}) {
|
|
@@ -1469,16 +1519,28 @@ export class Actor {
|
|
|
1469
1519
|
/**
|
|
1470
1520
|
* Gets the Actor input value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
1471
1521
|
*
|
|
1472
|
-
*
|
|
1473
|
-
* For example, calling the following code:
|
|
1522
|
+
* The input is the record stored under the configured input key (`ACTOR_INPUT_KEY`, default `INPUT`):
|
|
1474
1523
|
* ```js
|
|
1475
1524
|
* const input = await Actor.getInput();
|
|
1476
1525
|
* ```
|
|
1477
1526
|
*
|
|
1478
|
-
* is
|
|
1527
|
+
* A record stored as `application/octet-stream` — which is what a local input file without an
|
|
1528
|
+
* extension is read as — is parsed with JSON5 when it parses, and returned as a `Buffer` otherwise.
|
|
1529
|
+
*
|
|
1530
|
+
* When running locally and the store holds no such record, the input is read from a `<inputKey>` or
|
|
1531
|
+
* `<inputKey>.json` file in the current working directory instead.
|
|
1532
|
+
*
|
|
1533
|
+
* Throws an {@link ActorInputError} when no input is found (`code: 'NOT_FOUND'`), when both input
|
|
1534
|
+
* files exist in the working directory (`'MULTIPLE_FILES'`), when a JSON input does not parse
|
|
1535
|
+
* (`'PARSE_FAILED'`) or when its secret fields cannot be decrypted (`'DECRYPTION_FAILED'`). If your
|
|
1536
|
+
* Actor can run without input, catch the `'NOT_FOUND'` case and rethrow the rest:
|
|
1479
1537
|
* ```js
|
|
1480
|
-
*
|
|
1481
|
-
*
|
|
1538
|
+
* let input = {};
|
|
1539
|
+
* try {
|
|
1540
|
+
* input = await Actor.getInput();
|
|
1541
|
+
* } catch (error) {
|
|
1542
|
+
* if (!(error instanceof ActorInputError) || error.code !== 'NOT_FOUND') throw error;
|
|
1543
|
+
* }
|
|
1482
1544
|
* ```
|
|
1483
1545
|
*
|
|
1484
1546
|
* Note that the `getInput()` function does not cache the value read from the key-value store.
|
|
@@ -1490,19 +1552,12 @@ export class Actor {
|
|
|
1490
1552
|
* @returns
|
|
1491
1553
|
* Returns a promise that resolves to an object, string
|
|
1492
1554
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
1493
|
-
* on the MIME content type of the record
|
|
1494
|
-
*
|
|
1555
|
+
* on the MIME content type of the record.
|
|
1556
|
+
* @throws {ActorInputError} When the input cannot be produced; see `code`.
|
|
1495
1557
|
*/
|
|
1496
1558
|
static async getInput() {
|
|
1497
1559
|
return Actor.getDefaultInstance().getInput();
|
|
1498
1560
|
}
|
|
1499
|
-
/**
|
|
1500
|
-
* Gets the Actor input value just like the {@link Actor.getInput} method,
|
|
1501
|
-
* but throws if it is not found.
|
|
1502
|
-
*/
|
|
1503
|
-
static async getInputOrThrow() {
|
|
1504
|
-
return Actor.getDefaultInstance().getInputOrThrow();
|
|
1505
|
-
}
|
|
1506
1561
|
/**
|
|
1507
1562
|
* Opens a key-value store and returns a promise resolving to an instance of the {@link KeyValueStore} class.
|
|
1508
1563
|
*
|
|
@@ -1674,10 +1729,27 @@ export class Actor {
|
|
|
1674
1729
|
configuration: this.configuration,
|
|
1675
1730
|
requestQueueAccess: this.#requestQueueAccess,
|
|
1676
1731
|
}), charging),
|
|
1677
|
-
localStorageBackend: new ChargingStorageBackend(
|
|
1732
|
+
localStorageBackend: new ChargingStorageBackend(this.createLocalStorageBackend(), charging),
|
|
1678
1733
|
configuration: this.configuration,
|
|
1679
1734
|
}));
|
|
1680
1735
|
}
|
|
1736
|
+
/**
|
|
1737
|
+
* The same choice crawlee's `ServiceLocator` makes for its implicit default backend, with the
|
|
1738
|
+
* input-aware {@link ApifyFileSystemStorageBackend} in place of crawlee's — crawlee has no
|
|
1739
|
+
* notion of a run input.
|
|
1740
|
+
*/
|
|
1741
|
+
createLocalStorageBackend() {
|
|
1742
|
+
const { persistStorage, storageDir, inputKey } = this.configuration;
|
|
1743
|
+
const logger = serviceLocator.getLogger();
|
|
1744
|
+
if (!persistStorage) {
|
|
1745
|
+
return new MemoryStorageBackend({ logger: logger.child({ prefix: 'MemoryStorageBackend' }) });
|
|
1746
|
+
}
|
|
1747
|
+
return new ApifyFileSystemStorageBackend({
|
|
1748
|
+
localDataDirectory: storageDir,
|
|
1749
|
+
inputKey,
|
|
1750
|
+
logger: logger.child({ prefix: 'ApifyFileSystemStorageBackend' }),
|
|
1751
|
+
});
|
|
1752
|
+
}
|
|
1681
1753
|
/** Whether items reaching the run's default dataset are counted for pay-per-event charging. */
|
|
1682
1754
|
#chargesDefaultDatasetItems() {
|
|
1683
1755
|
const installed = serviceLocator.getStorageBackend();
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { AdoptionCandidate, FileSystemStorageOptions, KeyValueStoreHookOptions, PurgeableKeyValueStoreBackend } from '@crawlee/fs-storage';
|
|
2
|
+
import { FileSystemStorageBackend } from '@crawlee/fs-storage';
|
|
3
|
+
export interface ApifyFileSystemStorageOptions extends FileSystemStorageOptions {
|
|
4
|
+
/**
|
|
5
|
+
* The key the Actor input is read from — `Configuration.inputKey` (`ACTOR_INPUT_KEY`).
|
|
6
|
+
*
|
|
7
|
+
* `INPUT` is treated as an input key regardless of this setting, so a project's `INPUT.json` stays
|
|
8
|
+
* intact when the Apify CLI redirects the run to a temporary key.
|
|
9
|
+
*
|
|
10
|
+
* @default 'INPUT'
|
|
11
|
+
*/
|
|
12
|
+
inputKey?: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Crawlee's file-system storage backend, aware of the Actor input.
|
|
16
|
+
*
|
|
17
|
+
* Locally the input arrives in the default key-value store as a bare `INPUT` or `INPUT.json` file with
|
|
18
|
+
* no metadata sidecar — written by the Apify CLI, a project template, or by hand. This backend adopts
|
|
19
|
+
* such a file as the record under the input key when the store is opened (both files present fails the
|
|
20
|
+
* open), and keeps the input keys when the default store is purged on start. Everything else is
|
|
21
|
+
* inherited from {@link FileSystemStorageBackend}.
|
|
22
|
+
*
|
|
23
|
+
* `Actor` installs it as the local backend; pass an instance to `Actor.init({ storage })` only if you
|
|
24
|
+
* need to construct it yourself.
|
|
25
|
+
*/
|
|
26
|
+
export declare class ApifyFileSystemStorageBackend extends FileSystemStorageBackend {
|
|
27
|
+
/** `INPUT` plus the configured input key, deduplicated. */
|
|
28
|
+
private readonly inputKeys;
|
|
29
|
+
constructor(options: ApifyFileSystemStorageOptions);
|
|
30
|
+
/** Claims a bare `<key>` or `<key>.json` in the default store for each input key, ahead of the generic sweep. */
|
|
31
|
+
protected keyValueStoreAdoptionCandidates(options: KeyValueStoreHookOptions): AdoptionCandidate[];
|
|
32
|
+
/** Spares the input keys in the default store; every other run-scoped store is emptied. */
|
|
33
|
+
protected purgeKeyValueStore(store: PurgeableKeyValueStoreBackend, options: KeyValueStoreHookOptions): Promise<void>;
|
|
34
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { FileSystemStorageBackend } from '@crawlee/fs-storage';
|
|
2
|
+
import { KEY_VALUE_STORE_KEYS } from '@apify/consts';
|
|
3
|
+
import { BINARY_CONTENT_TYPE, JSON_CONTENT_TYPE } from './utils.js';
|
|
4
|
+
/**
|
|
5
|
+
* Crawlee's file-system storage backend, aware of the Actor input.
|
|
6
|
+
*
|
|
7
|
+
* Locally the input arrives in the default key-value store as a bare `INPUT` or `INPUT.json` file with
|
|
8
|
+
* no metadata sidecar — written by the Apify CLI, a project template, or by hand. This backend adopts
|
|
9
|
+
* such a file as the record under the input key when the store is opened (both files present fails the
|
|
10
|
+
* open), and keeps the input keys when the default store is purged on start. Everything else is
|
|
11
|
+
* inherited from {@link FileSystemStorageBackend}.
|
|
12
|
+
*
|
|
13
|
+
* `Actor` installs it as the local backend; pass an instance to `Actor.init({ storage })` only if you
|
|
14
|
+
* need to construct it yourself.
|
|
15
|
+
*/
|
|
16
|
+
export class ApifyFileSystemStorageBackend extends FileSystemStorageBackend {
|
|
17
|
+
/** `INPUT` plus the configured input key, deduplicated. */
|
|
18
|
+
inputKeys;
|
|
19
|
+
constructor(options) {
|
|
20
|
+
const { inputKey = KEY_VALUE_STORE_KEYS.INPUT, ...fileSystemOptions } = options;
|
|
21
|
+
super(fileSystemOptions);
|
|
22
|
+
this.inputKeys = [...new Set([KEY_VALUE_STORE_KEYS.INPUT, inputKey])];
|
|
23
|
+
}
|
|
24
|
+
/** Claims a bare `<key>` or `<key>.json` in the default store for each input key, ahead of the generic sweep. */
|
|
25
|
+
keyValueStoreAdoptionCandidates(options) {
|
|
26
|
+
const inputCandidates = options.isDefaultStore
|
|
27
|
+
? this.inputKeys.map((key) => ({
|
|
28
|
+
key,
|
|
29
|
+
files: [
|
|
30
|
+
{ filename: key, contentType: BINARY_CONTENT_TYPE },
|
|
31
|
+
{ filename: `${key}.json`, contentType: JSON_CONTENT_TYPE },
|
|
32
|
+
],
|
|
33
|
+
}))
|
|
34
|
+
: [];
|
|
35
|
+
return [...inputCandidates, ...super.keyValueStoreAdoptionCandidates(options)];
|
|
36
|
+
}
|
|
37
|
+
/** Spares the input keys in the default store; every other run-scoped store is emptied. */
|
|
38
|
+
async purgeKeyValueStore(store, options) {
|
|
39
|
+
await (options.isDefaultStore ? store.purgeExcept([...this.inputKeys]) : store.purge());
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -16,7 +16,7 @@ export class ApifyKeyValueStoreBackend {
|
|
|
16
16
|
if (!metadata) {
|
|
17
17
|
throw new Error('Key-value store not found or has been deleted.');
|
|
18
18
|
}
|
|
19
|
-
return metadata;
|
|
19
|
+
return { ...metadata, name: metadata.name ?? undefined };
|
|
20
20
|
}
|
|
21
21
|
async drop() {
|
|
22
22
|
await this.#client.delete();
|
|
@@ -42,6 +42,8 @@ export class ApifyKeyValueStoreBackend {
|
|
|
42
42
|
// requires the field, so it is left undefined via the cast.
|
|
43
43
|
return {
|
|
44
44
|
...result,
|
|
45
|
+
exclusiveStartKey: result.exclusiveStartKey ?? undefined,
|
|
46
|
+
nextExclusiveStartKey: result.nextExclusiveStartKey ?? undefined,
|
|
45
47
|
items: result.items.map(({ key, size }) => ({ key, size })),
|
|
46
48
|
};
|
|
47
49
|
}
|
package/dist/charging.js
CHANGED
|
@@ -102,9 +102,9 @@ export class ChargingManager {
|
|
|
102
102
|
}
|
|
103
103
|
// Load per-event pricing information
|
|
104
104
|
if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') {
|
|
105
|
-
for (const [eventName, eventPricing] of Object.entries(pricingInfo.pricingPerEvent.actorChargeEvents)) {
|
|
105
|
+
for (const [eventName, eventPricing] of Object.entries(pricingInfo.pricingPerEvent.actorChargeEvents ?? {})) {
|
|
106
106
|
this.#pricingInfo[eventName] = {
|
|
107
|
-
price: eventPricing.eventPriceUsd,
|
|
107
|
+
price: eventPricing.eventPriceUsd ?? 0,
|
|
108
108
|
title: eventPricing.eventTitle,
|
|
109
109
|
};
|
|
110
110
|
}
|
package/dist/configuration.d.ts
CHANGED
|
@@ -2,14 +2,13 @@ import type { ConfigField, FieldsInput, FieldsOutput } from '@crawlee/core';
|
|
|
2
2
|
import { Configuration as CoreConfiguration } from '@crawlee/core';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
export declare const apifyConfigFields: {
|
|
5
|
-
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
6
|
-
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
|
|
7
|
-
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
|
|
8
|
-
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof import("@crawlee/core").LogLevel
|
|
9
|
-
persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
5
|
+
maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
6
|
+
internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
7
|
+
systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
8
|
+
logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof import("@crawlee/core").LogLevel>, unknown>>>;
|
|
9
|
+
persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
10
10
|
storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
11
|
-
containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
|
|
12
|
-
inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
11
|
+
containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
13
12
|
memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
14
13
|
availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
15
14
|
disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean, unknown>>>;
|
|
@@ -22,6 +21,7 @@ export declare const apifyConfigFields: {
|
|
|
22
21
|
defaultDatasetId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
23
22
|
defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
24
23
|
defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
24
|
+
inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
|
|
25
25
|
metamorphAfterSleepMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber, unknown>>>;
|
|
26
26
|
actorEventsWsUrl: ConfigField<z.ZodOptional<z.ZodString>>;
|
|
27
27
|
token: ConfigField<z.ZodOptional<z.ZodString>>;
|
package/dist/configuration.js
CHANGED
|
@@ -20,7 +20,6 @@ export const apifyConfigFields = {
|
|
|
20
20
|
// take precedence; crawlee's own CRAWLEE_* var is reused as the fallback,
|
|
21
21
|
// never re-typed). A schema is passed only where the SDK needs a different
|
|
22
22
|
// default than crawlee's.
|
|
23
|
-
inputKey: withApifyEnv(crawleeConfigFields.inputKey, [ACTOR_ENV_VARS.INPUT_KEY, APIFY_ENV_VARS.INPUT_KEY]),
|
|
24
23
|
memoryMbytes: withApifyEnv(crawleeConfigFields.memoryMbytes, [
|
|
25
24
|
ACTOR_ENV_VARS.MEMORY_MBYTES,
|
|
26
25
|
APIFY_ENV_VARS.MEMORY_MBYTES,
|
|
@@ -51,6 +50,13 @@ export const apifyConfigFields = {
|
|
|
51
50
|
APIFY_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID,
|
|
52
51
|
]),
|
|
53
52
|
// Apify-specific fields
|
|
53
|
+
// Crawlee has no notion of a run input; `CRAWLEE_INPUT_KEY` stays as the
|
|
54
|
+
// lowest-priority alias because the Apify CLI sets all three variables.
|
|
55
|
+
inputKey: field(z.string().default('INPUT'), [
|
|
56
|
+
ACTOR_ENV_VARS.INPUT_KEY,
|
|
57
|
+
APIFY_ENV_VARS.INPUT_KEY,
|
|
58
|
+
'CRAWLEE_INPUT_KEY',
|
|
59
|
+
]),
|
|
54
60
|
metamorphAfterSleepMillis: field(coerceNumber.default(300_000), APIFY_ENV_VARS.METAMORPH_AFTER_SLEEP_MILLIS),
|
|
55
61
|
actorEventsWsUrl: field(z.string().optional(), [
|
|
56
62
|
ACTOR_ENV_VARS.EVENTS_WEBSOCKET_URL,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { RequestOptions
|
|
1
|
+
import type { RequestOptions } from '@crawlee/core';
|
|
2
2
|
/** Request options that the `globs` and `pseudoUrls` input editors can attach to a single URL pattern. */
|
|
3
3
|
export interface UrlPatternRequestOptions extends Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'> {
|
|
4
4
|
}
|
|
@@ -26,4 +26,4 @@ export interface UrlPatternFilters {
|
|
|
26
26
|
* await enqueueLinks({ transformRequestFunction: createTransformRequestFunction({ globs: input.globs }) });
|
|
27
27
|
* ```
|
|
28
28
|
*/
|
|
29
|
-
export declare function createTransformRequestFunction({ globs, pseudoUrls }: UrlPatternFilters):
|
|
29
|
+
export declare function createTransformRequestFunction({ globs, pseudoUrls, }: UrlPatternFilters): (request: RequestOptions) => RequestOptions | false | 'unchanged';
|
|
@@ -10,7 +10,7 @@ import { purlToRegExp } from '@apify/pseudo_url';
|
|
|
10
10
|
* await enqueueLinks({ transformRequestFunction: createTransformRequestFunction({ globs: input.globs }) });
|
|
11
11
|
* ```
|
|
12
12
|
*/
|
|
13
|
-
export function createTransformRequestFunction({ globs = [], pseudoUrls = [] }) {
|
|
13
|
+
export function createTransformRequestFunction({ globs = [], pseudoUrls = [], }) {
|
|
14
14
|
const patterns = [
|
|
15
15
|
...globs.flatMap((item) => {
|
|
16
16
|
if (item == null) {
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** Why {@link Actor.getInput} could not produce the input. */
|
|
2
|
+
export type ActorInputErrorCode =
|
|
3
|
+
/** No input record in the default key-value store, and locally no input file in the working directory either. */
|
|
4
|
+
'NOT_FOUND'
|
|
5
|
+
/** Both `<inputKey>` and `<inputKey>.json` exist in the working directory, so neither is picked. */
|
|
6
|
+
| 'MULTIPLE_FILES'
|
|
7
|
+
/** The secret fields of the input could not be decrypted; `cause` holds the underlying error. */
|
|
8
|
+
| 'DECRYPTION_FAILED'
|
|
9
|
+
/** A JSON input record or file does not parse; `cause` holds the parser error. */
|
|
10
|
+
| 'PARSE_FAILED';
|
|
11
|
+
/**
|
|
12
|
+
* Thrown by {@link Actor.getInput} when the Actor input cannot be produced. `code` says why, and
|
|
13
|
+
* `cause` carries the underlying error where there is one. Anything else thrown by `getInput()`, such as
|
|
14
|
+
* an Apify API error, is not an input problem and is passed through as is.
|
|
15
|
+
*
|
|
16
|
+
* ```js
|
|
17
|
+
* try {
|
|
18
|
+
* input = await Actor.getInput();
|
|
19
|
+
* } catch (error) {
|
|
20
|
+
* if (!(error instanceof ActorInputError) || error.code !== 'NOT_FOUND') throw error;
|
|
21
|
+
* input = {};
|
|
22
|
+
* }
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export declare class ActorInputError extends Error {
|
|
26
|
+
readonly name = "ActorInputError";
|
|
27
|
+
readonly code: ActorInputErrorCode;
|
|
28
|
+
constructor(code: ActorInputErrorCode, message: string, options?: {
|
|
29
|
+
cause?: unknown;
|
|
30
|
+
});
|
|
31
|
+
}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thrown by {@link Actor.getInput} when the Actor input cannot be produced. `code` says why, and
|
|
3
|
+
* `cause` carries the underlying error where there is one. Anything else thrown by `getInput()`, such as
|
|
4
|
+
* an Apify API error, is not an input problem and is passed through as is.
|
|
5
|
+
*
|
|
6
|
+
* ```js
|
|
7
|
+
* try {
|
|
8
|
+
* input = await Actor.getInput();
|
|
9
|
+
* } catch (error) {
|
|
10
|
+
* if (!(error instanceof ActorInputError) || error.code !== 'NOT_FOUND') throw error;
|
|
11
|
+
* input = {};
|
|
12
|
+
* }
|
|
13
|
+
* ```
|
|
14
|
+
*/
|
|
15
|
+
export class ActorInputError extends Error {
|
|
16
|
+
name = 'ActorInputError';
|
|
17
|
+
code;
|
|
18
|
+
constructor(code, message, options) {
|
|
19
|
+
super(message, options);
|
|
20
|
+
this.code = code;
|
|
21
|
+
}
|
|
22
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export * from './actor.js';
|
|
2
2
|
export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_storage_backend.js';
|
|
3
|
+
export { ApifyFileSystemStorageBackend, type ApifyFileSystemStorageOptions, } from './apify_file_system_storage_backend.js';
|
|
3
4
|
export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
4
5
|
export { ArgumentValidationError } from './utils.js';
|
|
6
|
+
export { ActorInputError, type ActorInputErrorCode } from './errors.js';
|
|
5
7
|
export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
|
|
6
8
|
export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier } from './storage.js';
|
|
7
9
|
export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export * from './actor.js';
|
|
2
2
|
export { ApifyStorageBackend } from './apify_storage_backend.js';
|
|
3
|
+
export { ApifyFileSystemStorageBackend, } from './apify_file_system_storage_backend.js';
|
|
3
4
|
export { ArgumentValidationError } from './utils.js';
|
|
5
|
+
export { ActorInputError } from './errors.js';
|
|
4
6
|
export { createTransformRequestFunction, } from './enqueue_links_filters.js';
|
|
5
7
|
export { ChargingManager } from './charging.js';
|
|
6
8
|
export * from './configuration.js';
|
|
@@ -8,7 +8,6 @@ interface ProxyStatus {
|
|
|
8
8
|
connectionError: string;
|
|
9
9
|
isManInTheMiddle: boolean;
|
|
10
10
|
}
|
|
11
|
-
type NewUrlOptions = Parameters<CoreProxyConfiguration['newProxyInfo']>[0];
|
|
12
11
|
export interface ProxyConfigurationOptions extends CoreProxyConfigurationOptions {
|
|
13
12
|
/**
|
|
14
13
|
* User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
|
|
@@ -153,7 +152,7 @@ export interface ProxyInfo extends CoreProxyInfo {
|
|
|
153
152
|
export declare class ProxyConfiguration extends CoreProxyConfiguration {
|
|
154
153
|
#private;
|
|
155
154
|
readonly configuration: Configuration;
|
|
156
|
-
protected readonly log: import("@
|
|
155
|
+
protected readonly log: import("@crawlee/core").Log;
|
|
157
156
|
/**
|
|
158
157
|
* @internal
|
|
159
158
|
*/
|
|
@@ -174,13 +173,13 @@ export declare class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
174
173
|
* independent URL; for Apify Proxy a random session id is embedded so consecutive
|
|
175
174
|
* calls resolve to different IPs.
|
|
176
175
|
*/
|
|
177
|
-
newProxyInfo(
|
|
176
|
+
newProxyInfo(): Promise<ProxyInfo | undefined>;
|
|
178
177
|
/**
|
|
179
178
|
* Returns a new proxy URL. For Apify Proxy, each call generates a URL with a fresh
|
|
180
179
|
* random session id, so consecutive calls return independent URLs. For custom
|
|
181
180
|
* `proxyUrls`, the URLs are rotated round-robin.
|
|
182
181
|
*/
|
|
183
|
-
newUrl(
|
|
182
|
+
newUrl(): Promise<string | undefined>;
|
|
184
183
|
/**
|
|
185
184
|
* Returns proxy username.
|
|
186
185
|
*/
|
|
@@ -147,8 +147,8 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
147
147
|
* independent URL; for Apify Proxy a random session id is embedded so consecutive
|
|
148
148
|
* calls resolve to different IPs.
|
|
149
149
|
*/
|
|
150
|
-
async newProxyInfo(
|
|
151
|
-
const url = await this.newUrl(
|
|
150
|
+
async newProxyInfo() {
|
|
151
|
+
const url = await this.newUrl();
|
|
152
152
|
if (!url)
|
|
153
153
|
return undefined;
|
|
154
154
|
const parsed = new URL(url);
|
|
@@ -173,9 +173,9 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
173
173
|
* random session id, so consecutive calls return independent URLs. For custom
|
|
174
174
|
* `proxyUrls`, the URLs are rotated round-robin.
|
|
175
175
|
*/
|
|
176
|
-
async newUrl(
|
|
176
|
+
async newUrl() {
|
|
177
177
|
if (!this.#usesApifyProxy) {
|
|
178
|
-
return super.newUrl(
|
|
178
|
+
return super.newUrl();
|
|
179
179
|
}
|
|
180
180
|
return this.composeDefaultUrl(cryptoRandomObjectId(SESSION_ID_LENGTH));
|
|
181
181
|
}
|
|
@@ -213,7 +213,7 @@ export class ProxyConfiguration extends CoreProxyConfiguration {
|
|
|
213
213
|
return;
|
|
214
214
|
try {
|
|
215
215
|
const user = await Actor.apifyClient.user().get();
|
|
216
|
-
this.#password = user
|
|
216
|
+
this.#password = user?.proxy?.password;
|
|
217
217
|
}
|
|
218
218
|
catch (error) {
|
|
219
219
|
if (Actor.isAtHome()) {
|
package/dist/utils.d.ts
CHANGED
|
@@ -5,6 +5,17 @@ export { ArgumentValidationError } from '@apify/validations';
|
|
|
5
5
|
* @internal
|
|
6
6
|
*/
|
|
7
7
|
export declare function isNonEmptyObject(value: unknown): value is Record<string, unknown>;
|
|
8
|
+
/** Content type of a JSON record, as `KeyValueStore` and the file-system adoption write it. */
|
|
9
|
+
export declare const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
|
|
10
|
+
/** Content type of raw bytes, which is what a local input file without an extension is adopted as. */
|
|
11
|
+
export declare const BINARY_CONTENT_TYPE = "application/octet-stream";
|
|
12
|
+
/**
|
|
13
|
+
* Parses a raw input record the way `KeyValueStore.getValue` would, with one exception:
|
|
14
|
+
* `application/octet-stream` — the type a local input file without an extension is adopted as — is
|
|
15
|
+
* tried as JSON5 first, so such a file still yields an object. Bytes that do not parse come back as they are.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export declare function parseInputValue(value: Buffer | ArrayBuffer, contentType: string | null): unknown;
|
|
8
19
|
/**
|
|
9
20
|
* Converts a `SNAKE_CASE` string to `camelCase` (previously provided by `@crawlee/utils`).
|
|
10
21
|
* @internal
|
package/dist/utils.js
CHANGED
|
@@ -2,6 +2,7 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import { type } from 'node:os';
|
|
4
4
|
import { normalize } from 'node:path';
|
|
5
|
+
import { parseValue } from '@crawlee/core';
|
|
5
6
|
// @ts-ignore if we enable resolveJsonModule, we end up with `src` folder in `dist`
|
|
6
7
|
import crawleePkgJson from '@crawlee/core/package.json' with { type: 'json' };
|
|
7
8
|
// @ts-ignore if we enable resolveJsonModule, we end up with `src` folder in `dist`
|
|
@@ -21,6 +22,28 @@ const require = createRequire(import.meta.url);
|
|
|
21
22
|
export function isNonEmptyObject(value) {
|
|
22
23
|
return typeof value === 'object' && value !== null && !Array.isArray(value) && Object.keys(value).length > 0;
|
|
23
24
|
}
|
|
25
|
+
/** Content type of a JSON record, as `KeyValueStore` and the file-system adoption write it. */
|
|
26
|
+
export const JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
|
|
27
|
+
/** Content type of raw bytes, which is what a local input file without an extension is adopted as. */
|
|
28
|
+
export const BINARY_CONTENT_TYPE = 'application/octet-stream';
|
|
29
|
+
/**
|
|
30
|
+
* Parses a raw input record the way `KeyValueStore.getValue` would, with one exception:
|
|
31
|
+
* `application/octet-stream` — the type a local input file without an extension is adopted as — is
|
|
32
|
+
* tried as JSON5 first, so such a file still yields an object. Bytes that do not parse come back as they are.
|
|
33
|
+
* @internal
|
|
34
|
+
*/
|
|
35
|
+
export function parseInputValue(value, contentType) {
|
|
36
|
+
const mediaType = contentType?.split(';')[0].trim().toLowerCase();
|
|
37
|
+
if (mediaType === BINARY_CONTENT_TYPE) {
|
|
38
|
+
try {
|
|
39
|
+
return parseValue(value, JSON_CONTENT_TYPE);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return value;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return parseValue(value, contentType);
|
|
46
|
+
}
|
|
24
47
|
/**
|
|
25
48
|
* Converts a `SNAKE_CASE` string to `camelCase` (previously provided by `@crawlee/utils`).
|
|
26
49
|
* @internal
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.40",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -61,10 +61,11 @@
|
|
|
61
61
|
"@apify/timeout": "^1.0.1",
|
|
62
62
|
"@apify/utilities": "^3.0.1",
|
|
63
63
|
"@apify/validations": "^1.1.0",
|
|
64
|
-
"@crawlee/core": "
|
|
65
|
-
"@crawlee/
|
|
66
|
-
"@crawlee/
|
|
67
|
-
"
|
|
64
|
+
"@crawlee/core": ">=4.0.0-beta.198 <4.0.0-rc.0 || >4.0.0-rc.0 <5.0.0",
|
|
65
|
+
"@crawlee/fs-storage": ">=4.0.0-beta.198 <4.0.0-rc.0 || >4.0.0-rc.0 <5.0.0",
|
|
66
|
+
"@crawlee/types": ">=4.0.0-beta.198 <4.0.0-rc.0 || >4.0.0-rc.0 <5.0.0",
|
|
67
|
+
"@crawlee/utils": ">=4.0.0-beta.198 <4.0.0-rc.0 || >4.0.0-rc.0 <5.0.0",
|
|
68
|
+
"apify-client": "^3.0.0-beta.26",
|
|
68
69
|
"minimatch": "^10.2.6",
|
|
69
70
|
"semver": "^7.8.5",
|
|
70
71
|
"tslib": "^2.8.1",
|
|
@@ -81,7 +82,7 @@
|
|
|
81
82
|
"@types/tough-cookie": "^4.0.5",
|
|
82
83
|
"@types/ws": "^8.18.1",
|
|
83
84
|
"commitlint": "^21.2.2",
|
|
84
|
-
"crawlee": "
|
|
85
|
+
"crawlee": ">=4.0.0-beta.198 <4.0.0-rc.0 || >4.0.0-rc.0 <5.0.0",
|
|
85
86
|
"husky": "^9.1.7",
|
|
86
87
|
"lint-staged": "^17.3.0",
|
|
87
88
|
"oxfmt": "0.64.0",
|