apify 4.0.0-beta.24 → 4.0.0-beta.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/dist/actor.js +4 -4
  2. package/dist/configuration.d.ts +21 -21
  3. package/dist/enqueue_links_filters.d.ts +29 -0
  4. package/dist/enqueue_links_filters.js +52 -0
  5. package/dist/index.d.ts +1 -0
  6. package/dist/index.js +1 -0
  7. package/dist/platform_event_manager.js +2 -2
  8. package/package.json +9 -7
package/dist/actor.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createPrivateKey } from 'node:crypto';
2
- import { Dataset, KeyValueStore, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
2
+ import { Dataset, EventType, KeyValueStore, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
3
3
  import { sleep } from '@crawlee/utils';
4
4
  import { ApifyClient } from 'apify-client';
5
5
  import { z } from 'zod';
@@ -281,7 +281,7 @@ export class Actor {
281
281
  await events.close();
282
282
  log.debug(`Events closed`);
283
283
  // Emit the exit event
284
- events.emit("exit" /* EventType.EXIT */, options);
284
+ events.emit(EventType.EXIT, options);
285
285
  // Wait for all event listeners to be processed
286
286
  log.debug(`Waiting for all event listeners to complete their execution (with ${options.timeoutSecs} seconds timeout)`);
287
287
  if (options.exit) {
@@ -507,12 +507,12 @@ export class Actor {
507
507
  // `persistState` for individual RequestLists, RequestQueue... instances to be persisted
508
508
  ...serviceLocator
509
509
  .getEventManager()
510
- .listeners("persistState" /* EventType.PERSIST_STATE */)
510
+ .listeners(EventType.PERSIST_STATE)
511
511
  .map(async (x) => x({})),
512
512
  // `migrating` to pause Apify crawlers
513
513
  ...serviceLocator
514
514
  .getEventManager()
515
- .listeners("migrating" /* EventType.MIGRATING */)
515
+ .listeners(EventType.MIGRATING)
516
516
  .map(async (x) => x({})),
517
517
  ]);
518
518
  const runId = this.configuration.actorRunId;
package/dist/configuration.d.ts CHANGED
@@ -6,16 +6,16 @@ export declare const apifyConfigFields: {
6
6
  defaultKeyValueStoreId: ConfigField<z.ZodDefault<z.ZodString>>;
7
7
  defaultRequestQueueId: ConfigField<z.ZodDefault<z.ZodString>>;
8
8
  inputKey: ConfigField<z.ZodDefault<z.ZodString>>;
9
- memoryMbytes: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
10
- availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
11
- disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
12
- persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
13
- headless: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
14
- xvfb: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
9
+ memoryMbytes: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
10
+ availableMemoryRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
11
+ disableBrowserSandbox: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
12
+ persistStateIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
13
+ headless: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
14
+ xvfb: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
15
15
  chromeExecutablePath: ConfigField<z.ZodOptional<z.ZodString>>;
16
16
  defaultBrowserPath: ConfigField<z.ZodOptional<z.ZodString>>;
17
- purgeOnStart: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
18
- metamorphAfterSleepMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
17
+ purgeOnStart: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
18
+ metamorphAfterSleepMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
19
19
  actorEventsWsUrl: ConfigField<z.ZodOptional<z.ZodString>>;
20
20
  token: ConfigField<z.ZodOptional<z.ZodString>>;
21
21
  actorId: ConfigField<z.ZodOptional<z.ZodString>>;
@@ -23,36 +23,36 @@ export declare const apifyConfigFields: {
23
23
  actorTaskId: ConfigField<z.ZodOptional<z.ZodString>>;
24
24
  apiBaseUrl: ConfigField<z.ZodDefault<z.ZodString>>;
25
25
  apiPublicBaseUrl: ConfigField<z.ZodDefault<z.ZodString>>;
26
- containerPort: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
26
+ containerPort: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
27
27
  containerUrl: ConfigField<z.ZodDefault<z.ZodString>>;
28
28
  proxyHostname: ConfigField<z.ZodDefault<z.ZodString>>;
29
29
  proxyPassword: ConfigField<z.ZodOptional<z.ZodString>>;
30
- proxyPort: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
30
+ proxyPort: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
31
31
  proxyStatusUrl: ConfigField<z.ZodDefault<z.ZodString>>;
32
32
  /** @deprecated use `containerPort` instead */
33
- standbyPort: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
33
+ standbyPort: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
34
34
  standbyUrl: ConfigField<z.ZodOptional<z.ZodString>>;
35
- isAtHome: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
35
+ isAtHome: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
36
36
  userId: ConfigField<z.ZodOptional<z.ZodString>>;
37
37
  userIsPaying: ConfigField<z.ZodOptional<z.ZodString>>;
38
38
  actorPermissionLevel: ConfigField<z.ZodOptional<z.ZodString>>;
39
39
  inputSecretsPrivateKeyPassphrase: ConfigField<z.ZodOptional<z.ZodString>>;
40
40
  inputSecretsPrivateKeyFile: ConfigField<z.ZodOptional<z.ZodString>>;
41
- maxTotalChargeUsd: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>, z.ZodTransform<number, number>>>>;
41
+ maxTotalChargeUsd: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodPreprocess<z.ZodNumber>, z.ZodTransform<number, number>>>>;
42
42
  metaOrigin: ConfigField<z.ZodOptional<z.ZodString>>;
43
- testPayPerEvent: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
44
- useChargingLogDataset: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
43
+ testPayPerEvent: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
44
+ useChargingLogDataset: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
45
45
  actorPricingInfo: ConfigField<z.ZodOptional<z.ZodString>>;
46
46
  chargedEventCounts: ConfigField<z.ZodOptional<z.ZodString>>;
47
47
  actorStoragesJson: ConfigField<z.ZodOptional<z.ZodString>>;
48
48
  storageClientOptions: ConfigField<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
49
- maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
50
- internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
51
- systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodNumber>>>;
52
- logLevel: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<{} | null | undefined, unknown>, z.ZodEnum<typeof import("@apify/log").LogLevel>>>>;
53
- persistStorage: ConfigField<z.ZodDefault<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
49
+ maxUsedCpuRatio: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
50
+ internalTimeoutMillis: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodNumber>>>;
51
+ systemInfoIntervalMillis: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodNumber>>>;
52
+ logLevel: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodEnum<typeof import("@apify/log").LogLevel>>>>;
53
+ persistStorage: ConfigField<z.ZodDefault<z.ZodPreprocess<z.ZodBoolean>>>;
54
54
  storageDir: ConfigField<z.ZodDefault<z.ZodString>>;
55
- containerized: ConfigField<z.ZodOptional<z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodBoolean>>>;
55
+ containerized: ConfigField<z.ZodOptional<z.ZodPreprocess<z.ZodBoolean>>>;
56
56
  };
57
57
  export type ApifyConfigurationInput = FieldsInput<typeof apifyConfigFields>;
58
58
  export type ApifyResolvedConfigValues = FieldsOutput<typeof apifyConfigFields>;
package/dist/enqueue_links_filters.d.ts ADDED
@@ -0,0 +1,29 @@
1
+ import type { RequestOptions, RequestTransform } from '@crawlee/core';
2
+ /** Request options that the `globs` and `pseudoUrls` input editors can attach to a single URL pattern. */
3
+ export interface UrlPatternRequestOptions extends Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'> {
4
+ }
5
+ /** A single item of an input schema array field using the `globs` editor. */
6
+ export interface GlobInput extends UrlPatternRequestOptions {
7
+ glob: string;
8
+ }
9
+ /** A single item of an input schema array field using the `pseudoUrls` editor. */
10
+ export interface PseudoUrlInput extends UrlPatternRequestOptions {
11
+ purl: string;
12
+ }
13
+ export interface UrlPatternFilters {
14
+ /** Value of an input schema field using the `globs` editor. */
15
+ globs?: readonly (string | GlobInput)[];
16
+ /** Value of an input schema field using the `pseudoUrls` editor. */
17
+ pseudoUrls?: readonly (string | PseudoUrlInput)[];
18
+ }
19
+ /**
20
+ * Turns the `globs` and `pseudoUrls` input editor values into a `transformRequestFunction` that skips requests
21
+ * matching no pattern (all requests pass when no pattern is given) and applies the options of the matched pattern.
22
+ * When a URL matches multiple patterns, the first one wins, with globs checked before pseudo-URLs. The matched
23
+ * pattern's options shallowly override the request's fields, including `userData` (no deep merge, like in v3).
24
+ *
25
+ * ```ts
26
+ * await enqueueLinks({ transformRequestFunction: createTransformRequestFunction({ globs: input.globs }) });
27
+ * ```
28
+ */
29
+ export declare function createTransformRequestFunction({ globs, pseudoUrls }: UrlPatternFilters): RequestTransform;
package/dist/enqueue_links_filters.js ADDED
@@ -0,0 +1,52 @@
1
+ import { Minimatch } from 'minimatch';
2
+ import { purlToRegExp } from '@apify/pseudo_url';
3
+ /**
4
+ * Turns the `globs` and `pseudoUrls` input editor values into a `transformRequestFunction` that skips requests
5
+ * matching no pattern (all requests pass when no pattern is given) and applies the options of the matched pattern.
6
+ * When a URL matches multiple patterns, the first one wins, with globs checked before pseudo-URLs. The matched
7
+ * pattern's options shallowly override the request's fields, including `userData` (no deep merge, like in v3).
8
+ *
9
+ * ```ts
10
+ * await enqueueLinks({ transformRequestFunction: createTransformRequestFunction({ globs: input.globs }) });
11
+ * ```
12
+ */
13
+ export function createTransformRequestFunction({ globs = [], pseudoUrls = [] }) {
14
+ const patterns = [
15
+ ...globs.flatMap((item) => {
16
+ if (item == null) {
17
+ return [];
18
+ }
19
+ const { glob, ...options } = typeof item === 'string' ? { glob: item } : item;
20
+ const trimmedGlob = typeof glob === 'string' ? glob.trim() : '';
21
+ if (trimmedGlob.length === 0) {
22
+ return [];
23
+ }
24
+ const minimatch = new Minimatch(trimmedGlob, { nocase: true });
25
+ return [{ matches: (url) => minimatch.match(url), options }];
26
+ }),
27
+ ...pseudoUrls.flatMap((item) => {
28
+ if (item == null) {
29
+ return [];
30
+ }
31
+ const { purl, ...options } = typeof item === 'string' ? { purl: item } : item;
32
+ if (typeof purl !== 'string' || purl.trim().length === 0) {
33
+ return [];
34
+ }
35
+ let regexp;
36
+ try {
37
+ regexp = purlToRegExp(purl);
38
+ }
39
+ catch (cause) {
40
+ throw new Error(`Invalid pseudoUrl pattern '${purl}': ${cause.message}`, { cause });
41
+ }
42
+ return [{ matches: (url) => regexp.test(url), options }];
43
+ }),
44
+ ];
45
+ if (patterns.length === 0) {
46
+ return () => 'unchanged';
47
+ }
48
+ return (request) => {
49
+ const matched = patterns.find(({ matches }) => matches(request.url));
50
+ return matched ? { ...request, ...matched.options } : false;
51
+ };
52
+ }
package/dist/index.d.ts CHANGED
@@ -2,6 +2,7 @@ export * from './actor.js';
2
2
  export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_storage_backend.js';
3
3
  export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
4
4
  export { ArgumentValidationError } from './utils.js';
5
+ export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
5
6
  export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage.js';
6
7
  export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
7
8
  export * from './configuration.js';
package/dist/index.js CHANGED
@@ -1,6 +1,7 @@
1
1
  export * from './actor.js';
2
2
  export { ApifyStorageBackend } from './apify_storage_backend.js';
3
3
  export { ArgumentValidationError } from './utils.js';
4
+ export { createTransformRequestFunction, } from './enqueue_links_filters.js';
4
5
  export { ChargingManager } from './charging.js';
5
6
  export * from './configuration.js';
6
7
  export * from './proxy_configuration.js';
package/dist/platform_event_manager.js CHANGED
@@ -1,4 +1,4 @@
1
- import { EventManager } from '@crawlee/core';
1
+ import { EventManager, EventType } from '@crawlee/core';
2
2
  import { WebSocket } from 'ws';
3
3
  import { ACTOR_ENV_VARS, ACTOR_EVENT_NAMES } from '@apify/consts';
4
4
  import { betterClearInterval } from '@apify/utilities';
@@ -78,7 +78,7 @@ export class PlatformEventManager extends EventManager {
78
78
  this.events.emit(name, data);
79
79
  if (name === ACTOR_EVENT_NAMES.MIGRATING) {
80
80
  betterClearInterval(this.intervals.persistState); // Don't send any other persist state event.
81
- this.events.emit("persistState" /* EventType.PERSIST_STATE */, {
81
+ this.events.emit(EventType.PERSIST_STATE, {
82
82
  isMigrating: true,
83
83
  });
84
84
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "apify",
3
- "version": "4.0.0-beta.24",
3
+ "version": "4.0.0-beta.26",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -53,16 +53,18 @@
53
53
  ]
54
54
  },
55
55
  "dependencies": {
56
- "@apify/consts": "^2.51.0",
56
+ "@apify/consts": "^2.57.0",
57
57
  "@apify/datastructures": "^2.0.3",
58
58
  "@apify/input_secrets": "^1.2.0",
59
- "@apify/log": "^2.4.3",
59
+ "@apify/log": "^2.5.48",
60
+ "@apify/pseudo_url": "^2.0.89",
60
61
  "@apify/timeout": "^0.4.0",
61
62
  "@apify/utilities": "^2.13.0",
62
- "@crawlee/core": "^4.0.0-beta.115",
63
- "@crawlee/types": "^4.0.0-beta.115",
64
- "@crawlee/utils": "^4.0.0-beta.115",
63
+ "@crawlee/core": "^4.0.0-beta.128",
64
+ "@crawlee/types": "^4.0.0-beta.128",
65
+ "@crawlee/utils": "^4.0.0-beta.128",
65
66
  "apify-client": "^2.23.4",
67
+ "minimatch": "^10.0.1",
66
68
  "semver": "^7.5.4",
67
69
  "tslib": "^2.6.2",
68
70
  "ws": "^8.18.0",
@@ -79,7 +81,7 @@
79
81
  "@types/tough-cookie": "^4.0.5",
80
82
  "@types/ws": "^8.5.12",
81
83
  "commitlint": "^21.0.0",
82
- "crawlee": "^4.0.0-beta.115",
84
+ "crawlee": "^4.0.0-beta.128",
83
85
  "globby": "^16.0.0",
84
86
  "husky": "^9.1.7",
85
87
  "lint-staged": "^17.0.0",