@crawlee/types 4.0.0-beta.67 → 4.0.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/package.json +2 -2
- package/status-message.d.ts +14 -0
- package/status-message.d.ts.map +1 -0
- package/status-message.js +2 -0
- package/status-message.js.map +1 -0
- package/storages.d.ts +123 -86
- package/storages.d.ts.map +1 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
package/index.d.ts
CHANGED
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,mBAAmB,eAAe,CAAC;AACnC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,cAAc,CAAC;AAClC,mBAAmB,kBAAkB,CAAC;AACtC,mBAAmB,cAAc,CAAC;AAClC,mBAAmB,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,eAAe,CAAC;AACnC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,cAAc,CAAC;AAClC,mBAAmB,kBAAkB,CAAC;AACtC,mBAAmB,cAAc,CAAC;AAClC,mBAAmB,aAAa,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/types",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.69",
|
|
4
4
|
"description": "Shared types for the crawlee projects",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,5 +53,5 @@
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
|
-
"gitHead": "
|
|
56
|
+
"gitHead": "e11d3725e54a33960e5425f568dd2d1a8be7cc5b"
|
|
57
57
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Options for setting a crawler run's status message via {@link BasicCrawler.setStatusMessage}.
|
|
3
|
+
*
|
|
4
|
+
* Setting a status message is not a storage concern — the crawler broadcasts it through the event
|
|
5
|
+
* system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their
|
|
6
|
+
* status-reporting backend.
|
|
7
|
+
*/
|
|
8
|
+
export interface SetStatusMessageOptions {
|
|
9
|
+
/** Whether this is the final status message of the run. */
|
|
10
|
+
isStatusMessageTerminal?: boolean;
|
|
11
|
+
/** The log level to log the message with. Defaults to `'DEBUG'`. */
|
|
12
|
+
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=status-message.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"status-message.d.ts","sourceRoot":"","sources":["../src/status-message.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACpC,2DAA2D;IAC3D,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,oEAAoE;IACpE,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;CAClD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"status-message.js","sourceRoot":"","sources":["../src/status-message.ts"],"names":[],"mappings":""}
|
package/storages.d.ts
CHANGED
|
@@ -11,6 +11,13 @@ export interface QueueOperationInfo {
|
|
|
11
11
|
/** The ID of the added request */
|
|
12
12
|
requestId: string;
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* A single page of items returned by {@link DatasetBackend.getData}.
|
|
16
|
+
*
|
|
17
|
+
* Datasets paginate by offset, so a page is self-describing via `total` / `offset` / `limit`: a
|
|
18
|
+
* frontend assembling all pages knows it has reached the end once `offset + items.length >= total`.
|
|
19
|
+
* The cursor-based counterpart for key-value stores is {@link KeyValueStoreListKeysResult}.
|
|
20
|
+
*/
|
|
14
21
|
export interface PaginatedList<Data> {
|
|
15
22
|
/** Total count of entries in the dataset. */
|
|
16
23
|
total: number;
|
|
@@ -25,7 +32,7 @@ export interface PaginatedList<Data> {
|
|
|
25
32
|
/** Dataset entries based on chosen format parameter. */
|
|
26
33
|
items: Data[];
|
|
27
34
|
}
|
|
28
|
-
export interface
|
|
35
|
+
export interface DatasetBackendListOptions {
|
|
29
36
|
desc?: boolean;
|
|
30
37
|
limit?: number;
|
|
31
38
|
offset?: number;
|
|
@@ -37,16 +44,8 @@ export interface DatasetInfo {
|
|
|
37
44
|
modifiedAt: Date;
|
|
38
45
|
accessedAt: Date;
|
|
39
46
|
itemCount: number;
|
|
40
|
-
actId?: string;
|
|
41
|
-
actRunId?: string;
|
|
42
47
|
}
|
|
43
|
-
export interface
|
|
44
|
-
readCount?: number;
|
|
45
|
-
writeCount?: number;
|
|
46
|
-
deleteCount?: number;
|
|
47
|
-
storageBytes?: number;
|
|
48
|
-
}
|
|
49
|
-
export interface DatasetClient<Data extends Dictionary = Dictionary> {
|
|
48
|
+
export interface DatasetBackend<Data extends Dictionary = Dictionary> {
|
|
50
49
|
/**
|
|
51
50
|
* Returns metadata about the dataset (id, name, timestamps, item count, etc.).
|
|
52
51
|
*
|
|
@@ -62,29 +61,42 @@ export interface DatasetClient<Data extends Dictionary = Dictionary> {
|
|
|
62
61
|
/** Add items to the dataset. */
|
|
63
62
|
pushData(items: Data[]): Promise<void>;
|
|
64
63
|
/** Fetch a page of items from the dataset. */
|
|
65
|
-
getData(options?:
|
|
66
|
-
}
|
|
67
|
-
export interface KeyValueStoreStats {
|
|
68
|
-
readCount?: number;
|
|
69
|
-
writeCount?: number;
|
|
70
|
-
deleteCount?: number;
|
|
71
|
-
listCount?: number;
|
|
72
|
-
storageBytes?: number;
|
|
64
|
+
getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Data>>;
|
|
73
65
|
}
|
|
74
66
|
export interface KeyValueStoreInfo {
|
|
75
67
|
id: string;
|
|
76
68
|
name?: string;
|
|
77
|
-
userId?: string;
|
|
78
69
|
createdAt: Date;
|
|
79
70
|
modifiedAt: Date;
|
|
80
71
|
accessedAt: Date;
|
|
81
|
-
actId?: string;
|
|
82
|
-
actRunId?: string;
|
|
83
|
-
stats?: KeyValueStoreStats;
|
|
84
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* The value a serialized record carries on the way *into* a storage backend (`setValue`).
|
|
75
|
+
*
|
|
76
|
+
* The `KeyValueStore` frontend has already serialized by this point, so it is a pre-serialized
|
|
77
|
+
* string, raw bytes (`Buffer` / `ArrayBuffer` / typed array), or a stream the client drains.
|
|
78
|
+
*/
|
|
79
|
+
export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
|
|
80
|
+
/**
|
|
81
|
+
* A record as returned by a storage backend (`getValue`).
|
|
82
|
+
*
|
|
83
|
+
* Storage backends are byte transports: they persist and return bytes verbatim and never serialize
|
|
84
|
+
* or parse. Interpretation is the `KeyValueStore` frontend's job (it parses according to the content
|
|
85
|
+
* type via `parseValue`). The value is therefore always raw bytes — a `Buffer` (Node) or an
|
|
86
|
+
* `ArrayBuffer` (browser backends).
|
|
87
|
+
*/
|
|
85
88
|
export interface KeyValueStoreRecord {
|
|
86
89
|
key: string;
|
|
87
|
-
value:
|
|
90
|
+
value: Buffer | ArrayBuffer;
|
|
91
|
+
contentType?: string;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* A record passed to a storage backend for writing (`setValue`). Like {@link KeyValueStoreRecord} but
|
|
95
|
+
* with the lenient, pre-serialized {@link KeyValueStoreRecordInputValue} for the value.
|
|
96
|
+
*/
|
|
97
|
+
export interface KeyValueStoreInputRecord {
|
|
98
|
+
key: string;
|
|
99
|
+
value: KeyValueStoreRecordInputValue;
|
|
88
100
|
contentType?: string;
|
|
89
101
|
}
|
|
90
102
|
export interface KeyValueStoreListKeysOptions {
|
|
@@ -98,11 +110,37 @@ export interface KeyValueStoreListKeysOptions {
|
|
|
98
110
|
export interface KeyValueStoreItemData {
|
|
99
111
|
key: string;
|
|
100
112
|
size: number;
|
|
113
|
+
/** The MIME content type the record was stored with, if known. */
|
|
114
|
+
contentType: string;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* A single page of keys returned by {@link KeyValueStoreBackend.listKeys}.
|
|
118
|
+
*
|
|
119
|
+
* This mirrors {@link PaginatedList} (the shape returned by {@link DatasetBackend.getData}) so that
|
|
120
|
+
* both listing operations on the storage backend layer return a self-describing page. The difference
|
|
121
|
+
* is the pagination model: datasets are offset-based (`total` / `offset`), whereas key-value stores
|
|
122
|
+
* are cursor-based. A frontend assembling all pages should therefore not guess "is this the last
|
|
123
|
+
* page?" from `items.length < limit` — it should rely on {@link isTruncated} and resume from
|
|
124
|
+
* {@link nextExclusiveStartKey}.
|
|
125
|
+
*/
|
|
126
|
+
export interface KeyValueStoreListKeysResult {
|
|
127
|
+
/** Keys returned on this page. */
|
|
128
|
+
items: KeyValueStoreItemData[];
|
|
129
|
+
/** Number of keys returned on this page (`items.length`). */
|
|
130
|
+
count: number;
|
|
131
|
+
/** Maximum number of keys requested for this page. */
|
|
132
|
+
limit: number;
|
|
133
|
+
/** The `exclusiveStartKey` that produced this page, if any. */
|
|
134
|
+
exclusiveStartKey?: string;
|
|
135
|
+
/** `true` if there are more keys beyond this page. When `true`, {@link nextExclusiveStartKey} is set. */
|
|
136
|
+
isTruncated: boolean;
|
|
137
|
+
/** Cursor to pass as the next call's `exclusiveStartKey`, or `undefined` when {@link isTruncated} is `false`. */
|
|
138
|
+
nextExclusiveStartKey?: string;
|
|
101
139
|
}
|
|
102
140
|
/**
|
|
103
141
|
* Key-value Store client.
|
|
104
142
|
*/
|
|
105
|
-
export interface
|
|
143
|
+
export interface KeyValueStoreBackend {
|
|
106
144
|
/**
|
|
107
145
|
* Returns metadata about the key-value store (id, name, timestamps, etc.).
|
|
108
146
|
*
|
|
@@ -115,45 +153,49 @@ export interface KeyValueStoreClient {
|
|
|
115
153
|
drop(): Promise<void>;
|
|
116
154
|
/** Remove all records from the store but keep the store itself. */
|
|
117
155
|
purge(): Promise<void>;
|
|
118
|
-
/**
|
|
156
|
+
/**
|
|
157
|
+
* Get a record by key. Returns the raw bytes plus content type, or `undefined` if not found.
|
|
158
|
+
*
|
|
159
|
+
* Clients are byte transports and must not parse the body — the `KeyValueStore` frontend
|
|
160
|
+
* interprets it according to the content type.
|
|
161
|
+
*/
|
|
119
162
|
getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
|
|
120
163
|
/** Set a record value. */
|
|
121
|
-
setValue(record:
|
|
164
|
+
setValue(record: KeyValueStoreInputRecord): Promise<void>;
|
|
122
165
|
/** Delete a record by key. */
|
|
123
166
|
deleteValue(key: string): Promise<void>;
|
|
124
|
-
/**
|
|
125
|
-
|
|
167
|
+
/**
|
|
168
|
+
* List a single page of keys in the store. Returns at most `limit` keys starting after
|
|
169
|
+
* `exclusiveStartKey`, wrapped in a self-describing page.
|
|
170
|
+
*
|
|
171
|
+
* Like {@link DatasetBackend.getData}, this returns one page rather than the whole collection;
|
|
172
|
+
* assembling all pages (e.g. for `KeyValueStore.keys()`) is the frontend's job. The result carries
|
|
173
|
+
* a cursor (`isTruncated` / `nextExclusiveStartKey`) so the frontend can paginate deterministically
|
|
174
|
+
* instead of inferring the end from `items.length < limit`.
|
|
175
|
+
*/
|
|
176
|
+
listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
|
|
126
177
|
/** Get the public URL for a record, or `undefined` if unavailable. */
|
|
127
178
|
getPublicUrl(key: string): Promise<string | undefined>;
|
|
128
179
|
/** Check whether a record with the given key exists. */
|
|
129
180
|
recordExists(key: string): Promise<boolean>;
|
|
130
181
|
}
|
|
131
|
-
export interface RequestQueueStats {
|
|
132
|
-
readCount?: number;
|
|
133
|
-
writeCount?: number;
|
|
134
|
-
deleteCount?: number;
|
|
135
|
-
headItemReadCount?: number;
|
|
136
|
-
storageBytes?: number;
|
|
137
|
-
}
|
|
138
182
|
export interface RequestQueueInfo {
|
|
139
183
|
id: string;
|
|
140
184
|
name?: string;
|
|
141
|
-
userId?: string;
|
|
142
185
|
createdAt: Date;
|
|
143
186
|
modifiedAt: Date;
|
|
144
187
|
accessedAt: Date;
|
|
145
|
-
expireAt?: string;
|
|
146
188
|
totalRequestCount: number;
|
|
147
189
|
handledRequestCount: number;
|
|
148
190
|
pendingRequestCount: number;
|
|
149
|
-
actId?: string;
|
|
150
|
-
actRunId?: string;
|
|
151
|
-
hadMultipleClients?: boolean;
|
|
152
|
-
stats?: RequestQueueStats;
|
|
153
191
|
}
|
|
154
|
-
|
|
192
|
+
/**
|
|
193
|
+
* Options for request-queue operations that add or return requests to the queue
|
|
194
|
+
* ({@link RequestQueueBackend.addBatchOfRequests}, {@link RequestQueueBackend.reclaimRequest}).
|
|
195
|
+
*/
|
|
196
|
+
export interface RequestQueueOperationOptions {
|
|
197
|
+
/** Place the affected request(s) at the beginning of the queue so they are processed sooner. */
|
|
155
198
|
forefront?: boolean;
|
|
156
|
-
[k: string]: unknown;
|
|
157
199
|
}
|
|
158
200
|
export interface RequestSchema {
|
|
159
201
|
id?: string;
|
|
@@ -195,7 +237,7 @@ export interface BatchAddRequestsResult {
|
|
|
195
237
|
* locking on the Apify platform) is an internal concern of the implementation and is not exposed on
|
|
196
238
|
* this interface.
|
|
197
239
|
*/
|
|
198
|
-
export interface
|
|
240
|
+
export interface RequestQueueBackend {
|
|
199
241
|
/**
|
|
200
242
|
* Returns metadata about the request queue (id, name, timestamps, request counts, etc.).
|
|
201
243
|
*
|
|
@@ -215,49 +257,49 @@ export interface RequestQueueClient {
|
|
|
215
257
|
* but not re-added. With `forefront`, requests are placed at the beginning of the queue so
|
|
216
258
|
* they are processed sooner.
|
|
217
259
|
*/
|
|
218
|
-
addBatchOfRequests(requests: RequestSchema[], options?:
|
|
260
|
+
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
219
261
|
/**
|
|
220
262
|
* Retrieve a request from the queue by its `uniqueKey`, or `undefined` if it does not exist.
|
|
221
263
|
*/
|
|
222
|
-
getRequest(uniqueKey: string): Promise<
|
|
264
|
+
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
|
|
223
265
|
/**
|
|
224
|
-
* Return the next request in the queue to be processed, or `
|
|
266
|
+
* Return the next request in the queue to be processed, or `undefined` if there are currently no
|
|
225
267
|
* pending requests.
|
|
226
268
|
*
|
|
227
269
|
* The returned request is marked as in-progress; it will not be returned again until it is
|
|
228
270
|
* either reclaimed via {@link reclaimRequest} or marked as handled via {@link markRequestAsHandled}.
|
|
229
271
|
*
|
|
230
|
-
*
|
|
272
|
+
* An `undefined` return value does not mean processing is finished — only that there are no pending
|
|
231
273
|
* requests right now. Use {@link isEmpty} (together with the frontend's knowledge of pending
|
|
232
274
|
* add operations) to determine whether the queue is truly finished.
|
|
233
275
|
*/
|
|
234
|
-
fetchNextRequest(): Promise<
|
|
276
|
+
fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
|
|
235
277
|
/**
|
|
236
278
|
* Mark a request previously returned by {@link fetchNextRequest} as handled.
|
|
237
279
|
*
|
|
238
280
|
* Handled requests are never returned again by {@link fetchNextRequest}. Returns information
|
|
239
|
-
* about the operation, or `
|
|
281
|
+
* about the operation, or `undefined` if the request was not in progress.
|
|
240
282
|
*
|
|
241
|
-
*
|
|
283
|
+
* An `undefined` result is a no-op, not an error: the request is simply not something this client is
|
|
242
284
|
* currently processing, so nothing is changed and the request is never added to the queue as a side
|
|
243
285
|
* effect. (Marking an already-handled request is idempotent and still returns operation info with
|
|
244
|
-
* `wasAlreadyHandled: true` rather than `
|
|
286
|
+
* `wasAlreadyHandled: true` rather than `undefined`.)
|
|
245
287
|
*/
|
|
246
|
-
markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo |
|
|
288
|
+
markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
|
|
247
289
|
/**
|
|
248
290
|
* Reclaim a failed request back to the queue so it can be processed again by a later call to
|
|
249
291
|
* {@link fetchNextRequest}. With `forefront`, the request is returned to the beginning of the
|
|
250
|
-
* queue. Returns information about the operation, or `
|
|
292
|
+
* queue. Returns information about the operation, or `undefined` if the request was not in progress.
|
|
251
293
|
*
|
|
252
294
|
* The request is expected to already be present in the queue (it should have been obtained via
|
|
253
|
-
* {@link fetchNextRequest}); reclaiming releases its lock rather than inserting it.
|
|
295
|
+
* {@link fetchNextRequest}); reclaiming releases its lock rather than inserting it. An `undefined` result
|
|
254
296
|
* is a no-op, not an error: the request is simply not something this client is currently processing,
|
|
255
297
|
* so nothing is changed and the request is never added to the queue as a side effect. Use
|
|
256
|
-
* {@link
|
|
298
|
+
* {@link addBatchOfRequests} to insert a new request.
|
|
257
299
|
*/
|
|
258
|
-
reclaimRequest(request: UpdateRequestSchema, options?:
|
|
300
|
+
reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
|
|
259
301
|
/**
|
|
260
|
-
* Resolves to `true` if the next call to {@link fetchNextRequest} would return `
|
|
302
|
+
* Resolves to `true` if the next call to {@link fetchNextRequest} would return `undefined` — i.e. there
|
|
261
303
|
* are no pending requests to fetch right now.
|
|
262
304
|
*
|
|
263
305
|
* Requests that are currently in progress (fetched but not yet handled or reclaimed, including
|
|
@@ -286,11 +328,7 @@ export interface RequestQueueClient {
|
|
|
286
328
|
* this long, so that a long-running consumer does not have its request handed out again while it is
|
|
287
329
|
* still being processed. Clients that do not lock may ignore it.
|
|
288
330
|
*/
|
|
289
|
-
setExpectedRequestProcessingTimeSecs?(secs: number): void
|
|
290
|
-
}
|
|
291
|
-
export interface SetStatusMessageOptions {
|
|
292
|
-
isStatusMessageTerminal?: boolean;
|
|
293
|
-
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
|
331
|
+
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
|
|
294
332
|
}
|
|
295
333
|
/**
|
|
296
334
|
* Identifies a storage by its ID, name, or alias. At most one may be provided.
|
|
@@ -320,17 +358,17 @@ export type StorageIdentifier = {
|
|
|
320
358
|
alias?: never;
|
|
321
359
|
};
|
|
322
360
|
/**
|
|
323
|
-
* Options for creating a dataset
|
|
361
|
+
* Options for creating a dataset backend via {@link StorageBackend.createDatasetBackend}.
|
|
324
362
|
*/
|
|
325
|
-
export type
|
|
363
|
+
export type CreateDatasetBackendOptions = StorageIdentifier;
|
|
326
364
|
/**
|
|
327
|
-
* Options for creating a key-value store
|
|
365
|
+
* Options for creating a key-value store backend via {@link StorageBackend.createKeyValueStoreBackend}.
|
|
328
366
|
*/
|
|
329
|
-
export type
|
|
367
|
+
export type CreateKeyValueStoreBackendOptions = StorageIdentifier;
|
|
330
368
|
/**
|
|
331
|
-
* Options for creating a request queue
|
|
369
|
+
* Options for creating a request queue backend via {@link StorageBackend.createRequestQueueBackend}.
|
|
332
370
|
*/
|
|
333
|
-
export type
|
|
371
|
+
export type CreateRequestQueueBackendOptions = StorageIdentifier & {
|
|
334
372
|
/**
|
|
335
373
|
* Client key for request locking.
|
|
336
374
|
* TODO: This is an Apify-platform concern and should eventually be pushed down
|
|
@@ -350,36 +388,36 @@ export type CreateRequestQueueClientOptions = StorageIdentifier & {
|
|
|
350
388
|
* Represents a storage backend capable of working with datasets, key-value stores and request queues.
|
|
351
389
|
*
|
|
352
390
|
* A new storage backend needs to implement 4 classes:
|
|
353
|
-
* - `
|
|
354
|
-
* - `
|
|
355
|
-
* - `
|
|
356
|
-
* - `
|
|
391
|
+
* - `StorageBackend` - the factory that creates sub-backends
|
|
392
|
+
* - `DatasetBackend` - operations on a single dataset
|
|
393
|
+
* - `KeyValueStoreBackend` - operations on a single key-value store
|
|
394
|
+
* - `RequestQueueBackend` - operations on a single request queue
|
|
357
395
|
*
|
|
358
|
-
* The `
|
|
359
|
-
* storage or creates a new one, returning a sub-
|
|
396
|
+
* The `StorageBackend` acts as an async factory: each `create*` method either opens an existing
|
|
397
|
+
* storage or creates a new one, returning a sub-backend bound to that storage instance.
|
|
360
398
|
*/
|
|
361
|
-
export interface
|
|
399
|
+
export interface StorageBackend {
|
|
362
400
|
/**
|
|
363
|
-
* Create (or open) a dataset
|
|
401
|
+
* Create (or open) a dataset backend.
|
|
364
402
|
* If `id` is provided, opens the dataset with that ID.
|
|
365
403
|
* If `name` is provided, opens an existing dataset with that name or creates a new one.
|
|
366
404
|
* If neither is provided, opens or creates the default dataset.
|
|
367
405
|
*/
|
|
368
|
-
|
|
406
|
+
createDatasetBackend(options?: CreateDatasetBackendOptions): Promise<DatasetBackend>;
|
|
369
407
|
/**
|
|
370
|
-
* Create (or open) a key-value store
|
|
408
|
+
* Create (or open) a key-value store backend.
|
|
371
409
|
* If `id` is provided, opens the key-value store with that ID.
|
|
372
410
|
* If `name` is provided, opens an existing store with that name or creates a new one.
|
|
373
411
|
* If neither is provided, opens or creates the default key-value store.
|
|
374
412
|
*/
|
|
375
|
-
|
|
413
|
+
createKeyValueStoreBackend(options?: CreateKeyValueStoreBackendOptions): Promise<KeyValueStoreBackend>;
|
|
376
414
|
/**
|
|
377
|
-
* Create (or open) a request queue
|
|
415
|
+
* Create (or open) a request queue backend.
|
|
378
416
|
* If `id` is provided, opens the request queue with that ID.
|
|
379
417
|
* If `name` is provided, opens an existing queue with that name or creates a new one.
|
|
380
418
|
* If neither is provided, opens or creates the default request queue.
|
|
381
419
|
*/
|
|
382
|
-
|
|
420
|
+
createRequestQueueBackend(options?: CreateRequestQueueBackendOptions): Promise<RequestQueueBackend>;
|
|
383
421
|
/**
|
|
384
422
|
* Check whether a storage with the given ID exists.
|
|
385
423
|
*
|
|
@@ -392,15 +430,14 @@ export interface StorageClient {
|
|
|
392
430
|
*
|
|
393
431
|
* The key is used by `StorageInstanceManager` to partition the storage cache per-backend,
|
|
394
432
|
* so that two storages with the same name but backed by different clients
|
|
395
|
-
* (e.g. a local `
|
|
433
|
+
* (e.g. a local `MemoryStorageBackend` and a cloud `ApifyClient`) are cached as separate instances.
|
|
396
434
|
*
|
|
397
435
|
* When not provided, the fallback uses the client's constructor name, so different
|
|
398
|
-
* `
|
|
436
|
+
* `StorageBackend` implementations automatically get separate cache partitions.
|
|
399
437
|
*/
|
|
400
|
-
|
|
438
|
+
getStorageBackendCacheKey?(): string;
|
|
401
439
|
purge?(): Promise<void>;
|
|
402
440
|
teardown?(): Promise<void>;
|
|
403
|
-
setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;
|
|
404
441
|
stats?: {
|
|
405
442
|
rateLimitErrors: number[];
|
|
406
443
|
};
|
package/storages.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storages.d.ts","sourceRoot":"","sources":["../src/storages.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEzE;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI;IAC/B,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"storages.d.ts","sourceRoot":"","sources":["../src/storages.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEzE;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI;IAC/B,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,yBAAyB;IACtC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAChE;;;;;;OAMG;IACH,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;IAEpC,2CAA2C;IAC3C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB,qEAAqE;IACrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvC,8CAA8C;IAC9C,OAAO,CAAC,OAAO,CAAC,EAAE,yBAAyB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;CAC9E;AAED,MAAM,WAAW,iBAAiB;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,MAAM,6BAA6B,GACnC,MAAM,GACN,WAAW,GACX,eAAe,GACf,MAAM,GACN,MAAM,CAAC,cAAc,GACrB,cAAc,CAAC;AAErB;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,6BAA6B,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,2BAA2B;IACxC,kCAAkC;IAClC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6DAA6D;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yGAAyG;IACzG,WAAW,EAAE,OAAO,CAAC;IACrB,iHAAiH;IACjH,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC;;;;;;OAMG;IACH,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAE1C,mDAAmD;IACnD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB,mEAAmE;IACnE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;;;OAKG;IACH,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;IAEhE,0BAA0B;IAC1B,QAAQ,CAAC,MAAM,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1D,8BAA8B;IAC9B,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;;;OAQG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,4BAA4B,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAEvF,sEAAsE;IACtE,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IAEvD,wDAAwD;IACxD,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,4BAA4B;IACzC,gGAAgG;IAChG,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAoB,SAAQ,aAAa;IACtD,EAAE,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,MAAM,WAAW,sBAAsB;IACnC,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC7C;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAChC;;;;;;OAMG;IACH,WAAW,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAEzC,iDAAiD;IACjD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB,oEAAoE;IACpE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;;;;OAMG;IACH,kBAAkB,CACd,QAAQ,EAAE,aAAa,EAAE,EACzB,OAAO,CAAC,EAAE,4BAA4B,GACvC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAEnC;;OAEG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;IAExE;;;;;;;;;;OAUG;IACH,gBAAgB,IAAI,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;IAE7D;;;;;;;;;;OAUG;IACH,oBAAoB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAAC;IAE5F;;;;;;;;;;OAUG;IACH,cAAc,CACV,OAAO,EAAE,mBAAmB,EAC5B,OAAO,CAAC,EAAE,4BAA4B,GACvC,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAAC;IAE3C;;;;;;;;OAQG;IACH,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE5B;;;;;;;;;OASG;IACH,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAE/B;;;;;;;;OAQG;IACH,oCAAoC,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtE;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GACvB;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,EAAE,CAAC,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC;AAElD;;GAEG;AACH,MAAM,MAAM,2BAA2B,GAAG,iBAAiB,CAAC;AAE5D;;GAEG;AACH,MAAM,MAAM,iCAAiC,GAAG,iBAAiB,CAAC;AAElE;;GAEG;AACH,MAAM,MAAM,gCAAgC,GAAG,iBAAiB,GAAG;IAC/D;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,cAAc;IAC3B;;;;;OAKG;IACH,oBAAoB,CAAC,OAAO,CAAC,EAAE,2BAA2B,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACrF;;;;;OAKG;IACH,0BAA0B,CAAC,OAAO,CAAC,EAAE,iCAAiC,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvG;;;;;OAKG;IACH,yBAAyB,CAAC,OAAO,CAAC,EAAE,gCAAgC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpG;;;;;OAKG;IACH,aAAa,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACjG;;;;;;;;;OASG;IACH,yBAAyB,CAAC,IAAI,MAAM,CAAC;IACrC,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,KAAK,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACzC"}
|