@crawlee/types 4.0.0-beta.27 → 4.0.0-beta.270

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/session.js CHANGED
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=session.js.map
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Options for setting a crawler run's status message via {@link BasicCrawler.setStatusMessage}.
3
+ *
4
+ * Setting a status message is not a storage concern — the crawler broadcasts it through the event
5
+ * system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their
6
+ * status-reporting backend.
7
+ */
8
+ export interface SetStatusMessageOptions {
9
+ /** Whether this is the final status message of the run. */
10
+ isStatusMessageTerminal?: boolean;
11
+ /** The log level to log the message with. Defaults to `'DEBUG'`. */
12
+ level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
13
+ }
@@ -0,0 +1 @@
1
+ export {};
package/storages.d.ts CHANGED
@@ -11,16 +11,13 @@ export interface QueueOperationInfo {
11
11
  /** The ID of the added request */
12
12
  requestId: string;
13
13
  }
14
- export interface DatasetCollectionClientOptions {
15
- storageDir: string;
16
- }
17
- export interface DatasetCollectionData {
18
- id: string;
19
- name?: string;
20
- createdAt: Date;
21
- modifiedAt: Date;
22
- accessedAt: Date;
23
- }
14
+ /**
15
+ * A single page of items returned by {@link DatasetBackend.getData}.
16
+ *
17
+ * Datasets paginate by offset, so a page is self-describing via `total` / `offset` / `limit`: a
18
+ * frontend assembling all pages knows it has reached the end once `offset + items.length >= total`.
19
+ * The cursor-based counterpart for key-value stores is {@link KeyValueStoreListKeysResult}.
20
+ */
24
21
  export interface PaginatedList<Data> {
25
22
  /** Total count of entries in the dataset. */
26
23
  total: number;
@@ -35,20 +32,7 @@ export interface PaginatedList<Data> {
35
32
  /** Dataset entries based on chosen format parameter. */
36
33
  items: Data[];
37
34
  }
38
- export interface Dataset extends DatasetCollectionData {
39
- itemCount: number;
40
- }
41
- /**
42
- * Dataset collection client.
43
- */
44
- export interface DatasetCollectionClient {
45
- list(): Promise<PaginatedList<Dataset>>;
46
- getOrCreate(name?: string): Promise<DatasetCollectionData>;
47
- }
48
- export interface DatasetClientUpdateOptions {
49
- name?: string;
50
- }
51
- export interface DatasetClientListOptions {
35
+ export interface DatasetBackendListOptions {
52
36
  desc?: boolean;
53
37
  limit?: number;
54
38
  offset?: number;
@@ -60,179 +44,179 @@ export interface DatasetInfo {
60
44
  modifiedAt: Date;
61
45
  accessedAt: Date;
62
46
  itemCount: number;
63
- actId?: string;
64
- actRunId?: string;
65
- }
66
- export interface DatasetStats {
67
- readCount?: number;
68
- writeCount?: number;
69
- deleteCount?: number;
70
- storageBytes?: number;
71
47
  }
72
- export interface DatasetClient<Data extends Dictionary = Dictionary> {
73
- get(): Promise<DatasetInfo | undefined>;
74
- update(newFields: DatasetClientUpdateOptions): Promise<Partial<DatasetInfo>>;
75
- delete(): Promise<void>;
76
- downloadItems(...args: unknown[]): Promise<Buffer>;
77
- listItems(options?: DatasetClientListOptions): Promise<PaginatedList<Data>>;
78
- pushItems(items: Data | Data[] | string | string[]): Promise<void>;
79
- }
80
- export interface KeyValueStoreStats {
81
- readCount?: number;
82
- writeCount?: number;
83
- deleteCount?: number;
84
- listCount?: number;
85
- storageBytes?: number;
48
+ export interface DatasetBackend<Data extends Dictionary = Dictionary> {
49
+ /**
50
+ * Returns metadata about the dataset (id, name, timestamps, item count, etc.).
51
+ *
52
+ * Implementations should throw if the underlying storage no longer exists
53
+ * (e.g. it was deleted externally). This method should never return stale data
54
+ * for a storage that has been removed.
55
+ */
56
+ getMetadata(): Promise<DatasetInfo>;
57
+ /** Remove the dataset and all its data. */
58
+ drop(): Promise<void>;
59
+ /** Remove all items from the dataset but keep the dataset itself. */
60
+ purge(): Promise<void>;
61
+ /** Add items to the dataset. */
62
+ pushData(items: Data[]): Promise<void>;
63
+ /** Fetch a page of items from the dataset. */
64
+ getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Data>>;
86
65
  }
87
66
  export interface KeyValueStoreInfo {
88
67
  id: string;
89
68
  name?: string;
90
- userId?: string;
91
69
  createdAt: Date;
92
70
  modifiedAt: Date;
93
71
  accessedAt: Date;
94
- actId?: string;
95
- actRunId?: string;
96
- stats?: KeyValueStoreStats;
97
72
  }
98
73
  /**
99
- * Key-value store collection client.
74
+ * The value a serialized record carries on the way *into* a storage backend (`setValue`).
75
+ *
76
+ * The `KeyValueStore` frontend has already serialized by this point, so it is a pre-serialized
77
+ * string, raw bytes (`Buffer` / `ArrayBuffer` / typed array), or a stream the client drains.
78
+ */
79
+ export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
80
+ /**
81
+ * A record as returned by a storage backend (`getValue`).
82
+ *
83
+ * Storage backends are byte transports: they persist and return bytes verbatim and never serialize
84
+ * or parse. Interpretation is the `KeyValueStore` frontend's job (it parses according to the content
85
+ * type via `parseValue`). The value is therefore always raw bytes — a `Buffer` (Node) or an
86
+ * `ArrayBuffer` (browser backends).
100
87
  */
101
- export interface KeyValueStoreCollectionClient {
102
- list(): Promise<PaginatedList<KeyValueStoreInfo>>;
103
- getOrCreate(name?: string): Promise<KeyValueStoreInfo>;
104
- }
105
88
  export interface KeyValueStoreRecord {
106
89
  key: string;
107
- value: any;
90
+ value: Buffer | ArrayBuffer;
108
91
  contentType?: string;
109
92
  }
110
- export interface KeyValueStoreRecordOptions {
111
- timeoutSecs?: number;
112
- doNotRetryTimeouts?: boolean;
113
- }
114
- export interface KeyValueStoreClientUpdateOptions {
115
- name?: string;
93
+ /**
94
+ * A record passed to a storage backend for writing (`setValue`). Like {@link KeyValueStoreRecord} but
95
+ * with the lenient, pre-serialized {@link KeyValueStoreRecordInputValue} for the value.
96
+ */
97
+ export interface KeyValueStoreInputRecord {
98
+ key: string;
99
+ value: KeyValueStoreRecordInputValue;
100
+ contentType?: string;
116
101
  }
117
- export interface KeyValueStoreClientListOptions {
118
- limit?: number;
119
- exclusiveStartKey?: string;
120
- collection?: string;
102
+ export interface KeyValueStoreListKeysOptions {
103
+ /** If set, only keys that start with this prefix are returned. */
121
104
  prefix?: string;
105
+ /** All keys up to this one are skipped from the result. */
106
+ exclusiveStartKey?: string;
107
+ /** Maximum number of keys to return. */
108
+ limit?: number;
122
109
  }
123
110
  export interface KeyValueStoreItemData {
124
111
  key: string;
125
112
  size: number;
113
+ /** The MIME content type the record was stored with, if known. */
114
+ contentType: string;
126
115
  }
127
- export interface KeyValueStoreClientListData {
116
+ /**
117
+ * A single page of keys returned by {@link KeyValueStoreBackend.listKeys}.
118
+ *
119
+ * This mirrors {@link PaginatedList} (the shape returned by {@link DatasetBackend.getData}) so that
120
+ * both listing operations on the storage backend layer return a self-describing page. The difference
121
+ * is the pagination model: datasets are offset-based (`total` / `offset`), whereas key-value stores
122
+ * are cursor-based. A frontend assembling all pages should therefore not guess "is this the last
123
+ * page?" from `items.length < limit` — it should rely on {@link isTruncated} and resume from
124
+ * {@link nextExclusiveStartKey}.
125
+ */
126
+ export interface KeyValueStoreListKeysResult {
127
+ /** Keys returned on this page. */
128
+ items: KeyValueStoreItemData[];
129
+ /** Number of keys returned on this page (`items.length`). */
128
130
  count: number;
131
+ /** Maximum number of keys requested for this page. */
129
132
  limit: number;
133
+ /** The `exclusiveStartKey` that produced this page, if any. */
130
134
  exclusiveStartKey?: string;
135
+ /** `true` if there are more keys beyond this page. When `true`, {@link nextExclusiveStartKey} is set. */
131
136
  isTruncated: boolean;
137
+ /** Cursor to pass as the next call's `exclusiveStartKey`, or `undefined` when {@link isTruncated} is `false`. */
132
138
  nextExclusiveStartKey?: string;
133
- items: KeyValueStoreItemData[];
134
- }
135
- export interface KeyValueStoreClientGetRecordOptions {
136
- buffer?: boolean;
137
- stream?: boolean;
138
139
  }
139
140
  /**
140
141
  * Key-value Store client.
141
142
  */
142
- export interface KeyValueStoreClient {
143
- get(): Promise<KeyValueStoreInfo | undefined>;
144
- update(newFields: KeyValueStoreClientUpdateOptions): Promise<Partial<KeyValueStoreInfo>>;
145
- delete(): Promise<void>;
146
- listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;
143
+ export interface KeyValueStoreBackend {
144
+ /**
145
+ * Returns metadata about the key-value store (id, name, timestamps, etc.).
146
+ *
147
+ * Implementations should throw if the underlying storage no longer exists
148
+ * (e.g. it was deleted externally). This method should never return stale data
149
+ * for a storage that has been removed.
150
+ */
151
+ getMetadata(): Promise<KeyValueStoreInfo>;
152
+ /** Remove the key-value store and all its data. */
153
+ drop(): Promise<void>;
154
+ /** Remove all records from the store but keep the store itself. */
155
+ purge(): Promise<void>;
156
+ /**
157
+ * Get a record by key. Returns the raw bytes plus content type, or `undefined` if not found.
158
+ *
159
+ * Clients are byte transports and must not parse the body — the `KeyValueStore` frontend
160
+ * interprets it according to the content type.
161
+ */
162
+ getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
163
+ /** Set a record value. */
164
+ setValue(record: KeyValueStoreInputRecord): Promise<void>;
165
+ /** Delete a record by key. */
166
+ deleteValue(key: string): Promise<void>;
167
+ /**
168
+ * List a single page of keys in the store. Returns at most `limit` keys starting after
169
+ * `exclusiveStartKey`, wrapped in a self-describing page.
170
+ *
171
+ * Like {@link DatasetBackend.getData}, this returns one page rather than the whole collection;
172
+ * assembling all pages (e.g. for `KeyValueStore.keys()`) is the frontend's job. The result carries
173
+ * a cursor (`isTruncated` / `nextExclusiveStartKey`) so the frontend can paginate deterministically
174
+ * instead of inferring the end from `items.length < limit`.
175
+ */
176
+ listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
177
+ /**
178
+ * Get the public URL a record with this key has, or `undefined` if the backend exposes no public
179
+ * URLs at all.
180
+ *
181
+ * The URL is derived from the key; implementations MUST NOT check that the record exists. Callers
182
+ * legitimately ask for the URL of a record that is not on the backend yet — most notably one
183
+ * buffered in an uncommitted storage transaction — and existence is `recordExists`'s job.
184
+ */
185
+ getPublicUrl(key: string): Promise<string | undefined>;
186
+ /** Check whether a record with the given key exists. */
147
187
  recordExists(key: string): Promise<boolean>;
148
- getRecordPublicUrl(key: string): Promise<string | undefined>;
149
- getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
150
- setRecord(record: KeyValueStoreRecord, options?: KeyValueStoreRecordOptions): Promise<void>;
151
- deleteRecord(key: string): Promise<void>;
152
- }
153
- export interface RequestQueueStats {
154
- readCount?: number;
155
- writeCount?: number;
156
- deleteCount?: number;
157
- headItemReadCount?: number;
158
- storageBytes?: number;
159
188
  }
160
189
  export interface RequestQueueInfo {
161
190
  id: string;
162
191
  name?: string;
163
- userId?: string;
164
192
  createdAt: Date;
165
193
  modifiedAt: Date;
166
194
  accessedAt: Date;
167
- expireAt?: string;
168
195
  totalRequestCount: number;
169
196
  handledRequestCount: number;
170
197
  pendingRequestCount: number;
171
- actId?: string;
172
- actRunId?: string;
173
- hadMultipleClients?: boolean;
174
- stats?: RequestQueueStats;
175
198
  }
176
199
  /**
177
- * Request queue collection client.
200
+ * Options for request-queue operations that add or return requests to the queue
201
+ * ({@link RequestQueueBackend.addBatchOfRequests}, {@link RequestQueueBackend.reclaimRequest}).
178
202
  */
179
- export interface RequestQueueCollectionClient {
180
- list(): Promise<PaginatedList<RequestQueueInfo>>;
181
- getOrCreate(name: string): Promise<RequestQueueInfo>;
182
- }
183
- export interface RequestQueueHeadItem {
184
- id: string;
185
- retryCount: number;
186
- uniqueKey: string;
187
- url: string;
188
- method: AllowedHttpMethods;
189
- }
190
- export interface QueueHead {
191
- limit: number;
192
- queueModifiedAt: Date;
193
- hadMultipleClients?: boolean;
194
- items: RequestQueueHeadItem[];
195
- }
196
- export interface ListOptions {
197
- /**
198
- * @default 100
199
- */
200
- limit?: number;
201
- }
202
- export interface ListAndLockOptions extends ListOptions {
203
- lockSecs: number;
204
- }
205
- export interface ListAndLockHeadResult extends QueueHead {
206
- lockSecs: number;
207
- queueHasLockedRequests?: boolean;
208
- }
209
- export interface ProlongRequestLockOptions {
210
- lockSecs: number;
211
- forefront?: boolean;
212
- }
213
- export interface ProlongRequestLockResult {
214
- lockExpiresAt: Date;
215
- }
216
- export interface DeleteRequestLockOptions {
217
- forefront?: boolean;
218
- }
219
- export interface RequestOptions {
203
+ export interface RequestQueueOperationOptions {
204
+ /** Place the affected request(s) at the beginning of the queue so they are processed sooner. */
220
205
  forefront?: boolean;
221
- [k: string]: unknown;
222
206
  }
223
207
  export interface RequestSchema {
224
208
  id?: string;
225
209
  url: string;
226
210
  uniqueKey: string;
227
211
  method?: AllowedHttpMethods;
228
- payload?: string;
229
- noRetry?: boolean;
212
+ payload?: string | null;
213
+ noRetry?: boolean | null;
230
214
  retryCount?: number;
231
- errorMessages?: string[];
232
- headers?: Dictionary<string>;
215
+ errorMessages?: string[] | null;
216
+ headers?: Dictionary<string> | null;
233
217
  userData?: Dictionary;
234
- handledAt?: string;
235
- loadedUrl?: string;
218
+ handledAt?: string | null;
219
+ loadedUrl?: string | null;
236
220
  }
237
221
  export interface UpdateRequestSchema extends RequestSchema {
238
222
  id: string;
@@ -252,45 +236,204 @@ export interface BatchAddRequestsResult {
252
236
  processedRequests: ProcessedRequest[];
253
237
  unprocessedRequests: UnprocessedRequest[];
254
238
  }
255
- export interface RequestQueueClient {
256
- get(): Promise<RequestQueueInfo | undefined>;
257
- update(newFields: {
258
- name?: string;
259
- }): Promise<Partial<RequestQueueInfo> | undefined>;
260
- delete(): Promise<void>;
261
- listHead(options?: ListOptions): Promise<QueueHead>;
262
- addRequest(request: RequestSchema, options?: RequestOptions): Promise<QueueOperationInfo>;
263
- batchAddRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
264
- getRequest(id: string): Promise<RequestOptions | undefined>;
265
- updateRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo>;
266
- deleteRequest(id: string): Promise<unknown>;
267
- listAndLockHead(options: ListAndLockOptions): Promise<ListAndLockHeadResult>;
268
- prolongRequestLock(id: string, options: ProlongRequestLockOptions): Promise<ProlongRequestLockResult>;
269
- deleteRequestLock(id: string, options?: DeleteRequestLockOptions): Promise<void>;
270
- }
271
- export interface RequestQueueOptions {
272
- clientKey?: string;
273
- timeoutSecs?: number;
274
- }
275
- export interface SetStatusMessageOptions {
276
- isStatusMessageTerminal?: boolean;
277
- level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
239
+ /**
240
+ * Operations on a single request queue.
241
+ *
242
+ * A backend implementation owns all request bookkeeping (pending, in-progress, handled). Any
243
+ * coordination required between multiple distributed clients accessing the same queue (e.g. request
244
+ * locking on the Apify platform) is an internal concern of the implementation and is not exposed on
245
+ * this interface.
246
+ */
247
+ export interface RequestQueueBackend {
248
+ /**
249
+ * Returns metadata about the request queue (id, name, timestamps, request counts, etc.).
250
+ *
251
+ * Implementations should throw if the underlying storage no longer exists
252
+ * (e.g. it was deleted externally). This method should never return stale data
253
+ * for a storage that has been removed.
254
+ */
255
+ getMetadata(): Promise<RequestQueueInfo>;
256
+ /** Remove the request queue and all its data. */
257
+ drop(): Promise<void>;
258
+ /** Remove all requests from the queue but keep the queue itself. */
259
+ purge(): Promise<void>;
260
+ /**
261
+ * Add a batch of requests to the queue.
262
+ *
263
+ * Each request is deduplicated by its `uniqueKey`. Duplicates are reported in the result
264
+ * but not re-added. With `forefront`, requests are placed at the beginning of the queue so
265
+ * they are processed sooner.
266
+ */
267
+ addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
268
+ /**
269
+ * Retrieve a request from the queue by its `uniqueKey`, or `undefined` if it does not exist.
270
+ */
271
+ getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
272
+ /**
273
+ * Return the next request in the queue to be processed, or `undefined` if there are currently no
274
+ * pending requests.
275
+ *
276
+ * The returned request is marked as in-progress; it will not be returned again until it is
277
+ * either reclaimed via {@link reclaimRequest} or marked as handled via {@link markRequestAsHandled}.
278
+ *
279
+ * An `undefined` return value does not mean processing is finished — only that there are no pending
280
+ * requests right now. Use {@link isEmpty} (together with the frontend's knowledge of pending
281
+ * add operations) to determine whether the queue is truly finished.
282
+ */
283
+ fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
284
+ /**
285
+ * Mark a request previously returned by {@link fetchNextRequest} as handled.
286
+ *
287
+ * Handled requests are never returned again by {@link fetchNextRequest}. Returns information
288
+ * about the operation, or `undefined` if the request was not in progress.
289
+ *
290
+ * An `undefined` result is a no-op, not an error: the request is simply not something this client is
291
+ * currently processing, so nothing is changed and the request is never added to the queue as a side
292
+ * effect. (Marking an already-handled request is idempotent and still returns operation info with
293
+ * `wasAlreadyHandled: true` rather than `undefined`.)
294
+ */
295
+ markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
296
+ /**
297
+ * Reclaim a failed request back to the queue so it can be processed again by a later call to
298
+ * {@link fetchNextRequest}. With `forefront`, the request is returned to the beginning of the
299
+ * queue. Returns information about the operation, or `undefined` if the request was not in progress.
300
+ *
301
+ * The request is expected to already be present in the queue (it should have been obtained via
302
+ * {@link fetchNextRequest}); reclaiming releases its lock rather than inserting it. An `undefined` result
303
+ * is a no-op, not an error: the request is simply not something this client is currently processing,
304
+ * so nothing is changed and the request is never added to the queue as a side effect. Use
305
+ * {@link addBatchOfRequests} to insert a new request.
306
+ */
307
+ reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
308
+ /**
309
+ * Resolves to `true` if the next call to {@link fetchNextRequest} would return `undefined` — i.e. there
310
+ * are no pending requests to fetch right now.
311
+ *
312
+ * Requests that are currently in progress (fetched but not yet handled or reclaimed, including
313
+ * requests locked by other clients sharing the same queue) are **not** counted. An empty queue
314
+ * therefore does not mean crawling is finished — those in-progress requests may still be reclaimed,
315
+ * and background tasks may still add more requests. Use {@link isFinished} to detect completion.
316
+ */
317
+ isEmpty(): Promise<boolean>;
318
+ /**
319
+ * Resolves to `true` only when there is no outstanding work left in the queue at all — i.e. there
320
+ * are no pending requests to fetch **and** no requests currently in progress (fetched but not yet
321
+ * handled or reclaimed, including requests locked by other clients sharing the same queue).
322
+ *
323
+ * This is the strong counterpart of {@link isEmpty}: a queue whose only remaining requests are in
324
+ * progress is empty (nothing to fetch) but not finished (that work might still be reclaimed). It is
325
+ * the building block for determining whether crawling is done — though a frontend may still need to
326
+ * account for its own pending background add operations on top of this.
327
+ */
328
+ isFinished(): Promise<boolean>;
329
+ /**
330
+ * Tells the client how long (in seconds) a consumer expects to hold a request fetched via
331
+ * {@link fetchNextRequest} before marking it handled or reclaiming it — typically the consumer's
332
+ * request-processing timeout plus some padding.
333
+ *
334
+ * A client that coordinates consumers via locking uses this to keep the request reserved for at least
335
+ * this long, so that a long-running consumer does not have its request handed out again while it is
336
+ * still being processed. Clients that do not lock may ignore it.
337
+ */
338
+ setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
339
+ /**
340
+ * Extends the lock on a request previously handed out by {@link fetchNextRequest}, for a consumer
341
+ * that needs more time than the sizing hint reserved.
342
+ *
343
+ * @returns `true` when prolonged, `false` when this client does not hold the request locked (or
344
+ * does not lock at all). `false` is information, not an error: the request may be processed twice.
345
+ * Non-locking backends leave this unimplemented.
346
+ */
347
+ extendRequestProcessingTimeSecs?(requestId: string, secs: number): Promise<boolean>;
278
348
  }
279
349
  /**
280
- * Represents a storage capable of working with datasets, KV stores and request queues.
350
+ * Identifies a storage by its ID, name, or alias. At most one may be provided.
351
+ *
352
+ * - `{ id }` — open a pre-existing storage by its unique ID.
353
+ * - `{ name }` — open or create a globally named storage (persists across runs). The name `default`
354
+ * is reserved: it resolves to the default storage, and is emptied on start along with it.
355
+ * - `{ alias }` — open or create a run-scoped unnamed storage identified by this alias.
356
+ * The alias is used locally (e.g. as a directory name or cache key) but the storage
357
+ * itself has no persistent name. Use this for non-default unnamed storages. Like the
358
+ * default storage, an aliased one is emptied on start unless `purgeOnStart` is disabled.
359
+ * - `{}` / omitted — open the default storage.
360
+ */
361
+ export type StorageIdentifier = {
362
+ id: string;
363
+ name?: never;
364
+ alias?: never;
365
+ } | {
366
+ id?: never;
367
+ name: string;
368
+ alias?: never;
369
+ } | {
370
+ id?: never;
371
+ name?: never;
372
+ alias: string;
373
+ } | {
374
+ id?: never;
375
+ name?: never;
376
+ alias?: never;
377
+ };
378
+ /**
379
+ * Represents a storage backend capable of working with datasets, key-value stores and request queues.
380
+ *
381
+ * A new storage backend needs to implement 4 classes:
382
+ * - `StorageBackend` - the factory that creates sub-backends
383
+ * - `DatasetBackend` - operations on a single dataset
384
+ * - `KeyValueStoreBackend` - operations on a single key-value store
385
+ * - `RequestQueueBackend` - operations on a single request queue
386
+ *
387
+ * The `StorageBackend` acts as an async factory: each `create*` method either opens an existing
388
+ * storage or creates a new one, returning a sub-backend bound to that storage instance.
281
389
  */
282
- export interface StorageClient {
283
- datasets(): DatasetCollectionClient;
284
- dataset(id: string): DatasetClient;
285
- keyValueStores(): KeyValueStoreCollectionClient;
286
- keyValueStore(id: string): KeyValueStoreClient;
287
- requestQueues(): RequestQueueCollectionClient;
288
- requestQueue(id: string, options?: RequestQueueOptions): RequestQueueClient;
390
+ export interface StorageBackend {
391
+ /**
392
+ * Create (or open) a dataset backend.
393
+ * If `id` is provided, opens the dataset with that ID.
394
+ * If `name` is provided, opens an existing dataset with that name or creates a new one.
395
+ * If neither is provided, opens or creates the default dataset.
396
+ */
397
+ createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
398
+ /**
399
+ * Create (or open) a key-value store backend.
400
+ * If `id` is provided, opens the key-value store with that ID.
401
+ * If `name` is provided, opens an existing store with that name or creates a new one.
402
+ * If neither is provided, opens or creates the default key-value store.
403
+ */
404
+ createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
405
+ /**
406
+ * Create (or open) a request queue backend.
407
+ * If `id` is provided, opens the request queue with that ID.
408
+ * If `name` is provided, opens an existing queue with that name or creates a new one.
409
+ * If neither is provided, opens or creates the default request queue.
410
+ */
411
+ createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
412
+ /**
413
+ * Check whether a storage with the given ID exists.
414
+ *
415
+ * Used internally to resolve ambiguous `idOrName` strings passed to `Dataset.open()`,
416
+ * `KeyValueStore.open()`, and `RequestQueue.open()`.
417
+ */
418
+ storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
419
+ /**
420
+ * Return an opaque key that uniquely identifies this storage backend instance.
421
+ *
422
+ * The key is used by `StorageInstanceManager` to partition the storage cache per-backend,
423
+ * so that two storages with the same name but backed by different clients
424
+ * (e.g. a local `MemoryStorageBackend` and a cloud `ApifyClient`) are cached as separate instances.
425
+ *
426
+ * When not provided, the fallback uses the client's constructor name, so different
427
+ * `StorageBackend` implementations automatically get separate cache partitions.
428
+ */
429
+ getStorageBackendCacheKey?(): string;
430
+ /**
431
+ * Empty the run-scoped storages — the default one and every alias-keyed one, including any left
432
+ * behind by a previous run. Named storages persist across runs.
433
+ */
289
434
  purge?(): Promise<void>;
290
435
  teardown?(): Promise<void>;
291
- setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;
292
436
  stats?: {
293
437
  rateLimitErrors: number[];
294
438
  };
295
439
  }
296
- //# sourceMappingURL=storages.d.ts.map
package/storages.js CHANGED
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=storages.js.map
@@ -1,8 +1,4 @@
1
- /** @ignore */
2
1
  export type Dictionary<T = any> = Record<PropertyKey, T>;
3
- /** @ignore */
4
2
  export type Constructor<T = unknown> = new (...args: any[]) => T;
5
- /** @ignore */
6
3
  export type Awaitable<T> = T | PromiseLike<T>;
7
4
  export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';
8
- //# sourceMappingURL=utility-types.d.ts.map
package/utility-types.js CHANGED
@@ -1,2 +1 @@
1
1
  export {};
2
- //# sourceMappingURL=utility-types.js.map
package/browser.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,WAAW,MAAM;IACnB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IACrC;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACrC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC;IAChD;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAChC,GAAG,IAAI,MAAM,CAAC;IACd,OAAO,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;CAC5C"}
package/browser.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":""}
@@ -1 +0,0 @@
1
- {"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../src/http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7D,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;AAEnH;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;IAClB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAEhB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAG1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE;QACd,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5E,CAAC;IACF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAmB,SAAQ,WAAW;IACnD,yEAAyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B,6EAA6E;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,+EAA+E;IAC/E,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,CAC1B,gBAAgB,EAAE,QAAQ,EAC1B,cAAc,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,KACvD,IAAI,CAAC;AAEV,MAAM,WAAW,kBAAkB;IAC/B,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAc,SAAQ,kBAAkB;IACrD,UAAU,CAAC,EAAE,eAAe,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CAClF"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"http-client.js","sourceRoot":"","sources":["../src/http-client.ts"],"names":[],"mappings":""}
package/index.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC"}
package/index.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC"}
package/session.d.ts.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,SAAS;IACtB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,mBAAmB,CAAC;IAC/B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;;;;GAKG;AACH,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;;OAGG;IACH,SAAS,IAAI,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,IAAI,OAAO,CAAC;IAErB;;;OAGG;IACH,sBAAsB,IAAI,OAAO,CAAC;IAElC;;;OAGG;IACH,QAAQ,IAAI,OAAO,CAAC;IAEpB;;;OAGG;IACH,QAAQ,IAAI,IAAI,CAAC;IAEjB;;;OAGG;IACH,QAAQ,IAAI,YAAY,CAAC;IAEzB;;;;;;OAMG;IACH,MAAM,IAAI,IAAI,CAAC;IAEf;;;OAGG;IACH,OAAO,IAAI,IAAI,CAAC;IAEhB;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;IAExD;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;IAEjD;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAEjD;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACrC"}
package/session.js.map DELETED
@@ -1 +0,0 @@
1
- {"version":3,"file":"session.js","sourceRoot":"","sources":["../src/session.ts"],"names":[],"mappings":""}