@crawlee/fs-storage-native 0.0.1-placeholder.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/index.d.ts +167 -0
  2. package/index.js +772 -0
  3. package/lib.d.ts +25 -0
  4. package/lib.js +93 -0
  5. package/package.json +59 -0
package/index.d.ts ADDED
@@ -0,0 +1,167 @@
1
+ export interface DatasetMetadata {
2
+ id: string;
3
+ name: string | null;
4
+ accessedAt: string;
5
+ createdAt: string;
6
+ modifiedAt: string;
7
+ itemCount: number;
8
+ }
9
+
10
+ export interface DatasetItemsListPage {
11
+ count: number;
12
+ offset: number;
13
+ limit: number;
14
+ total: number;
15
+ desc: boolean;
16
+ items: Record<string, unknown>[];
17
+ }
18
+
19
+ export interface KeyValueStoreMetadata {
20
+ id: string;
21
+ name: string | null;
22
+ accessedAt: string;
23
+ createdAt: string;
24
+ modifiedAt: string;
25
+ }
26
+
27
+ export interface KeyValueStoreRecord {
28
+ key: string;
29
+ contentType: string;
30
+ size: number | null;
31
+ value: Buffer;
32
+ }
33
+
34
+ export interface KeyValueStoreStreamRecord {
35
+ key: string;
36
+ contentType: string;
37
+ size: number | null;
38
+ stream: ReadableStream<Uint8Array>;
39
+ }
40
+
41
+ export interface KeyValueStoreRecordMetadata {
42
+ key: string;
43
+ contentType: string;
44
+ size: number | null;
45
+ }
46
+
47
+ export interface RequestQueueMetadata {
48
+ id: string;
49
+ name: string | null;
50
+ accessedAt: string;
51
+ createdAt: string;
52
+ modifiedAt: string;
53
+ hadMultipleClients: boolean;
54
+ handledRequestCount: number;
55
+ pendingRequestCount: number;
56
+ totalRequestCount: number;
57
+ }
58
+
59
+ export interface ProcessedRequest {
60
+ requestId: string;
61
+ uniqueKey: string;
62
+ wasAlreadyPresent: boolean;
63
+ wasAlreadyHandled: boolean;
64
+ }
65
+
66
+ export interface UnprocessedRequest {
67
+ uniqueKey: string;
68
+ url: string;
69
+ method?: string | null;
70
+ }
71
+
72
+ export interface AddRequestsResponse {
73
+ processedRequests: ProcessedRequest[];
74
+ unprocessedRequests: UnprocessedRequest[];
75
+ }
76
+ export declare class DatasetItemIterator {
77
+ /** Fetch the next item. Returns null when iteration is exhausted. */
78
+ next(): Promise<Record<string, unknown> | null>;
79
+ }
80
+
81
+ export declare class FileSystemDatasetClient {
82
+ static open(
83
+ id?: string | undefined | null,
84
+ name?: string | undefined | null,
85
+ alias?: string | undefined | null,
86
+ storageDir?: string | undefined | null,
87
+ ): Promise<FileSystemDatasetClient>;
88
+ get pathToDataset(): string;
89
+ get pathToMetadata(): string;
90
+ getMetadata(): Promise<DatasetMetadata>;
91
+ dropStorage(): Promise<void>;
92
+ purge(): Promise<void>;
93
+ pushData(data: Record<string, unknown> | Record<string, unknown>[]): Promise<void>;
94
+ getData(
95
+ offset?: number | undefined | null,
96
+ limit?: number | undefined | null,
97
+ desc?: boolean | undefined | null,
98
+ skipEmpty?: boolean | undefined | null,
99
+ ): Promise<DatasetItemsListPage>;
100
+ iterateItems(
101
+ offset?: number | undefined | null,
102
+ limit?: number | undefined | null,
103
+ desc?: boolean | undefined | null,
104
+ skipEmpty?: boolean | undefined | null,
105
+ pageSize?: number | undefined | null,
106
+ ): Promise<DatasetItemIterator>;
107
+ }
108
+
109
+ export declare class FileSystemKeyValueStoreClient {
110
+ static open(
111
+ id?: string | undefined | null,
112
+ name?: string | undefined | null,
113
+ alias?: string | undefined | null,
114
+ storageDir?: string | undefined | null,
115
+ ): Promise<FileSystemKeyValueStoreClient>;
116
+ get pathToKvs(): string;
117
+ get pathToMetadata(): string;
118
+ getMetadata(): Promise<KeyValueStoreMetadata>;
119
+ dropStorage(): Promise<void>;
120
+ purge(): Promise<void>;
121
+ /** Get a record by key. Returns the raw value bytes as a Buffer. */
122
+ getValue(key: string): Promise<KeyValueStoreRecord | null>;
123
+ /** Set a value from a Buffer. */
124
+ setValue(key: string, value: Buffer, contentType?: string | undefined | null): Promise<void>;
125
+ deleteValue(key: string): Promise<void>;
126
+ iterateKeys(
127
+ exclusiveStartKey?: string | undefined | null,
128
+ limit?: number | undefined | null,
129
+ pageSize?: number | undefined | null,
130
+ ): Promise<KvsKeyIterator>;
131
+ getPublicUrl(key: string): Promise<string>;
132
+ recordExists(key: string): Promise<boolean>;
133
+ }
134
+
135
+ export declare class FileSystemRequestQueueClient {
136
+ static open(
137
+ id?: string | undefined | null,
138
+ name?: string | undefined | null,
139
+ alias?: string | undefined | null,
140
+ storageDir?: string | undefined | null,
141
+ ): Promise<FileSystemRequestQueueClient>;
142
+ get pathToRq(): string;
143
+ get pathToMetadata(): string;
144
+ getMetadata(): Promise<RequestQueueMetadata>;
145
+ dropStorage(): Promise<void>;
146
+ purge(): Promise<void>;
147
+ addBatchOfRequests(
148
+ requests: Record<string, unknown>[],
149
+ forefront?: boolean | undefined | null,
150
+ ): Promise<AddRequestsResponse>;
151
+ getRequest(uniqueKey: string): Promise<Record<string, unknown> | null>;
152
+ fetchNextRequest(): Promise<Record<string, unknown> | null>;
153
+ markRequestAsHandled(request: Record<string, unknown>): Promise<ProcessedRequest | null>;
154
+ reclaimRequest(
155
+ request: Record<string, unknown>,
156
+ forefront?: boolean | undefined | null,
157
+ ): Promise<ProcessedRequest | null>;
158
+ isEmpty(): Promise<boolean>;
159
+ isFinished(): Promise<boolean>;
160
+ setExpectedRequestProcessingTime(secs: number): Promise<void>;
161
+ persistState(): Promise<void>;
162
+ }
163
+
164
+ export declare class KvsKeyIterator {
165
+ /** Fetch the next key metadata entry. Returns null when iteration is exhausted. */
166
+ next(): Promise<KeyValueStoreRecordMetadata | null>;
167
+ }
package/index.js ADDED
@@ -0,0 +1,772 @@
1
+ // prettier-ignore
2
+ /* eslint-disable */
3
+ // @ts-nocheck
4
+ /* auto-generated by NAPI-RS */
5
+
6
+ const { readFileSync } = require('node:fs')
7
+ let nativeBinding = null;
8
+ const loadErrors = [];
9
+
10
+ const isMusl = () => {
11
+ let musl = false;
12
+ if (process.platform === 'linux') {
13
+ musl = isMuslFromFilesystem();
14
+ if (musl === null) {
15
+ musl = isMuslFromReport();
16
+ }
17
+ if (musl === null) {
18
+ musl = isMuslFromChildProcess();
19
+ }
20
+ }
21
+ return musl;
22
+ };
23
+
24
+ const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-');
25
+
26
+ const isMuslFromFilesystem = () => {
27
+ try {
28
+ return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl');
29
+ } catch {
30
+ return null;
31
+ }
32
+ };
33
+
34
+ const isMuslFromReport = () => {
35
+ let report = null;
36
+ if (typeof process.report?.getReport === 'function') {
37
+ process.report.excludeNetwork = true;
38
+ report = process.report.getReport();
39
+ }
40
+ if (!report) {
41
+ return null;
42
+ }
43
+ if (report.header && report.header.glibcVersionRuntime) {
44
+ return false;
45
+ }
46
+ if (Array.isArray(report.sharedObjects)) {
47
+ if (report.sharedObjects.some(isFileMusl)) {
48
+ return true;
49
+ }
50
+ }
51
+ return false;
52
+ };
53
+
54
+ const isMuslFromChildProcess = () => {
55
+ try {
56
+ return require('child_process')
57
+ .execSync('ldd --version', { encoding: 'utf8' })
58
+ .includes('musl');
59
+ } catch (e) {
60
+ // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false
61
+ return false;
62
+ }
63
+ };
64
+
65
+ function requireNative() {
66
+ if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
67
+ try {
68
+ return require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
69
+ } catch (err) {
70
+ loadErrors.push(err);
71
+ }
72
+ } else if (process.platform === 'android') {
73
+ if (process.arch === 'arm64') {
74
+ try {
75
+ return require('./fs-storage-native.android-arm64.node');
76
+ } catch (e) {
77
+ loadErrors.push(e);
78
+ }
79
+ try {
80
+ const binding = require('@crawlee/fs-storage-native-android-arm64');
81
+ const bindingPackageVersion =
82
+ require('@crawlee/fs-storage-native-android-arm64/package.json').version;
83
+ if (
84
+ bindingPackageVersion !== '0.1.0' &&
85
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
86
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
87
+ ) {
88
+ throw new Error(
89
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
90
+ );
91
+ }
92
+ return binding;
93
+ } catch (e) {
94
+ loadErrors.push(e);
95
+ }
96
+ } else if (process.arch === 'arm') {
97
+ try {
98
+ return require('./fs-storage-native.android-arm-eabi.node');
99
+ } catch (e) {
100
+ loadErrors.push(e);
101
+ }
102
+ try {
103
+ const binding = require('@crawlee/fs-storage-native-android-arm-eabi');
104
+ const bindingPackageVersion =
105
+ require('@crawlee/fs-storage-native-android-arm-eabi/package.json').version;
106
+ if (
107
+ bindingPackageVersion !== '0.1.0' &&
108
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
109
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
110
+ ) {
111
+ throw new Error(
112
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
113
+ );
114
+ }
115
+ return binding;
116
+ } catch (e) {
117
+ loadErrors.push(e);
118
+ }
119
+ } else {
120
+ loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`));
121
+ }
122
+ } else if (process.platform === 'win32') {
123
+ if (process.arch === 'x64') {
124
+ if (
125
+ process.config?.variables?.shlib_suffix === 'dll.a' ||
126
+ process.config?.variables?.node_target_type === 'shared_library'
127
+ ) {
128
+ try {
129
+ return require('./fs-storage-native.win32-x64-gnu.node');
130
+ } catch (e) {
131
+ loadErrors.push(e);
132
+ }
133
+ try {
134
+ const binding = require('@crawlee/fs-storage-native-win32-x64-gnu');
135
+ const bindingPackageVersion =
136
+ require('@crawlee/fs-storage-native-win32-x64-gnu/package.json').version;
137
+ if (
138
+ bindingPackageVersion !== '0.1.0' &&
139
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
140
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
141
+ ) {
142
+ throw new Error(
143
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
144
+ );
145
+ }
146
+ return binding;
147
+ } catch (e) {
148
+ loadErrors.push(e);
149
+ }
150
+ } else {
151
+ try {
152
+ return require('./fs-storage-native.win32-x64-msvc.node');
153
+ } catch (e) {
154
+ loadErrors.push(e);
155
+ }
156
+ try {
157
+ const binding = require('@crawlee/fs-storage-native-win32-x64-msvc');
158
+ const bindingPackageVersion =
159
+ require('@crawlee/fs-storage-native-win32-x64-msvc/package.json').version;
160
+ if (
161
+ bindingPackageVersion !== '0.1.0' &&
162
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
163
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
164
+ ) {
165
+ throw new Error(
166
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
167
+ );
168
+ }
169
+ return binding;
170
+ } catch (e) {
171
+ loadErrors.push(e);
172
+ }
173
+ }
174
+ } else if (process.arch === 'ia32') {
175
+ try {
176
+ return require('./fs-storage-native.win32-ia32-msvc.node');
177
+ } catch (e) {
178
+ loadErrors.push(e);
179
+ }
180
+ try {
181
+ const binding = require('@crawlee/fs-storage-native-win32-ia32-msvc');
182
+ const bindingPackageVersion =
183
+ require('@crawlee/fs-storage-native-win32-ia32-msvc/package.json').version;
184
+ if (
185
+ bindingPackageVersion !== '0.1.0' &&
186
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
187
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
188
+ ) {
189
+ throw new Error(
190
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
191
+ );
192
+ }
193
+ return binding;
194
+ } catch (e) {
195
+ loadErrors.push(e);
196
+ }
197
+ } else if (process.arch === 'arm64') {
198
+ try {
199
+ return require('./fs-storage-native.win32-arm64-msvc.node');
200
+ } catch (e) {
201
+ loadErrors.push(e);
202
+ }
203
+ try {
204
+ const binding = require('@crawlee/fs-storage-native-win32-arm64-msvc');
205
+ const bindingPackageVersion =
206
+ require('@crawlee/fs-storage-native-win32-arm64-msvc/package.json').version;
207
+ if (
208
+ bindingPackageVersion !== '0.1.0' &&
209
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
210
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
211
+ ) {
212
+ throw new Error(
213
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
214
+ );
215
+ }
216
+ return binding;
217
+ } catch (e) {
218
+ loadErrors.push(e);
219
+ }
220
+ } else {
221
+ loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`));
222
+ }
223
+ } else if (process.platform === 'darwin') {
224
+ try {
225
+ return require('./fs-storage-native.darwin-universal.node');
226
+ } catch (e) {
227
+ loadErrors.push(e);
228
+ }
229
+ try {
230
+ const binding = require('@crawlee/fs-storage-native-darwin-universal');
231
+ const bindingPackageVersion =
232
+ require('@crawlee/fs-storage-native-darwin-universal/package.json').version;
233
+ if (
234
+ bindingPackageVersion !== '0.1.0' &&
235
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
236
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
237
+ ) {
238
+ throw new Error(
239
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
240
+ );
241
+ }
242
+ return binding;
243
+ } catch (e) {
244
+ loadErrors.push(e);
245
+ }
246
+ if (process.arch === 'x64') {
247
+ try {
248
+ return require('./fs-storage-native.darwin-x64.node');
249
+ } catch (e) {
250
+ loadErrors.push(e);
251
+ }
252
+ try {
253
+ const binding = require('@crawlee/fs-storage-native-darwin-x64');
254
+ const bindingPackageVersion =
255
+ require('@crawlee/fs-storage-native-darwin-x64/package.json').version;
256
+ if (
257
+ bindingPackageVersion !== '0.1.0' &&
258
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
259
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
260
+ ) {
261
+ throw new Error(
262
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
263
+ );
264
+ }
265
+ return binding;
266
+ } catch (e) {
267
+ loadErrors.push(e);
268
+ }
269
+ } else if (process.arch === 'arm64') {
270
+ try {
271
+ return require('./fs-storage-native.darwin-arm64.node');
272
+ } catch (e) {
273
+ loadErrors.push(e);
274
+ }
275
+ try {
276
+ const binding = require('@crawlee/fs-storage-native-darwin-arm64');
277
+ const bindingPackageVersion =
278
+ require('@crawlee/fs-storage-native-darwin-arm64/package.json').version;
279
+ if (
280
+ bindingPackageVersion !== '0.1.0' &&
281
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
282
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
283
+ ) {
284
+ throw new Error(
285
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
286
+ );
287
+ }
288
+ return binding;
289
+ } catch (e) {
290
+ loadErrors.push(e);
291
+ }
292
+ } else {
293
+ loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`));
294
+ }
295
+ } else if (process.platform === 'freebsd') {
296
+ if (process.arch === 'x64') {
297
+ try {
298
+ return require('./fs-storage-native.freebsd-x64.node');
299
+ } catch (e) {
300
+ loadErrors.push(e);
301
+ }
302
+ try {
303
+ const binding = require('@crawlee/fs-storage-native-freebsd-x64');
304
+ const bindingPackageVersion =
305
+ require('@crawlee/fs-storage-native-freebsd-x64/package.json').version;
306
+ if (
307
+ bindingPackageVersion !== '0.1.0' &&
308
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
309
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
310
+ ) {
311
+ throw new Error(
312
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
313
+ );
314
+ }
315
+ return binding;
316
+ } catch (e) {
317
+ loadErrors.push(e);
318
+ }
319
+ } else if (process.arch === 'arm64') {
320
+ try {
321
+ return require('./fs-storage-native.freebsd-arm64.node');
322
+ } catch (e) {
323
+ loadErrors.push(e);
324
+ }
325
+ try {
326
+ const binding = require('@crawlee/fs-storage-native-freebsd-arm64');
327
+ const bindingPackageVersion =
328
+ require('@crawlee/fs-storage-native-freebsd-arm64/package.json').version;
329
+ if (
330
+ bindingPackageVersion !== '0.1.0' &&
331
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
332
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
333
+ ) {
334
+ throw new Error(
335
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
336
+ );
337
+ }
338
+ return binding;
339
+ } catch (e) {
340
+ loadErrors.push(e);
341
+ }
342
+ } else {
343
+ loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`));
344
+ }
345
+ } else if (process.platform === 'linux') {
346
+ if (process.arch === 'x64') {
347
+ if (isMusl()) {
348
+ try {
349
+ return require('./fs-storage-native.linux-x64-musl.node');
350
+ } catch (e) {
351
+ loadErrors.push(e);
352
+ }
353
+ try {
354
+ const binding = require('@crawlee/fs-storage-native-linux-x64-musl');
355
+ const bindingPackageVersion =
356
+ require('@crawlee/fs-storage-native-linux-x64-musl/package.json').version;
357
+ if (
358
+ bindingPackageVersion !== '0.1.0' &&
359
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
360
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
361
+ ) {
362
+ throw new Error(
363
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
364
+ );
365
+ }
366
+ return binding;
367
+ } catch (e) {
368
+ loadErrors.push(e);
369
+ }
370
+ } else {
371
+ try {
372
+ return require('./fs-storage-native.linux-x64-gnu.node');
373
+ } catch (e) {
374
+ loadErrors.push(e);
375
+ }
376
+ try {
377
+ const binding = require('@crawlee/fs-storage-native-linux-x64-gnu');
378
+ const bindingPackageVersion =
379
+ require('@crawlee/fs-storage-native-linux-x64-gnu/package.json').version;
380
+ if (
381
+ bindingPackageVersion !== '0.1.0' &&
382
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
383
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
384
+ ) {
385
+ throw new Error(
386
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
387
+ );
388
+ }
389
+ return binding;
390
+ } catch (e) {
391
+ loadErrors.push(e);
392
+ }
393
+ }
394
+ } else if (process.arch === 'arm64') {
395
+ if (isMusl()) {
396
+ try {
397
+ return require('./fs-storage-native.linux-arm64-musl.node');
398
+ } catch (e) {
399
+ loadErrors.push(e);
400
+ }
401
+ try {
402
+ const binding = require('@crawlee/fs-storage-native-linux-arm64-musl');
403
+ const bindingPackageVersion =
404
+ require('@crawlee/fs-storage-native-linux-arm64-musl/package.json').version;
405
+ if (
406
+ bindingPackageVersion !== '0.1.0' &&
407
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
408
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
409
+ ) {
410
+ throw new Error(
411
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
412
+ );
413
+ }
414
+ return binding;
415
+ } catch (e) {
416
+ loadErrors.push(e);
417
+ }
418
+ } else {
419
+ try {
420
+ return require('./fs-storage-native.linux-arm64-gnu.node');
421
+ } catch (e) {
422
+ loadErrors.push(e);
423
+ }
424
+ try {
425
+ const binding = require('@crawlee/fs-storage-native-linux-arm64-gnu');
426
+ const bindingPackageVersion =
427
+ require('@crawlee/fs-storage-native-linux-arm64-gnu/package.json').version;
428
+ if (
429
+ bindingPackageVersion !== '0.1.0' &&
430
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
431
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
432
+ ) {
433
+ throw new Error(
434
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
435
+ );
436
+ }
437
+ return binding;
438
+ } catch (e) {
439
+ loadErrors.push(e);
440
+ }
441
+ }
442
+ } else if (process.arch === 'arm') {
443
+ if (isMusl()) {
444
+ try {
445
+ return require('./fs-storage-native.linux-arm-musleabihf.node');
446
+ } catch (e) {
447
+ loadErrors.push(e);
448
+ }
449
+ try {
450
+ const binding = require('@crawlee/fs-storage-native-linux-arm-musleabihf');
451
+ const bindingPackageVersion =
452
+ require('@crawlee/fs-storage-native-linux-arm-musleabihf/package.json').version;
453
+ if (
454
+ bindingPackageVersion !== '0.1.0' &&
455
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
456
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
457
+ ) {
458
+ throw new Error(
459
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
460
+ );
461
+ }
462
+ return binding;
463
+ } catch (e) {
464
+ loadErrors.push(e);
465
+ }
466
+ } else {
467
+ try {
468
+ return require('./fs-storage-native.linux-arm-gnueabihf.node');
469
+ } catch (e) {
470
+ loadErrors.push(e);
471
+ }
472
+ try {
473
+ const binding = require('@crawlee/fs-storage-native-linux-arm-gnueabihf');
474
+ const bindingPackageVersion =
475
+ require('@crawlee/fs-storage-native-linux-arm-gnueabihf/package.json').version;
476
+ if (
477
+ bindingPackageVersion !== '0.1.0' &&
478
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
479
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
480
+ ) {
481
+ throw new Error(
482
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
483
+ );
484
+ }
485
+ return binding;
486
+ } catch (e) {
487
+ loadErrors.push(e);
488
+ }
489
+ }
490
+ } else if (process.arch === 'loong64') {
491
+ if (isMusl()) {
492
+ try {
493
+ return require('./fs-storage-native.linux-loong64-musl.node');
494
+ } catch (e) {
495
+ loadErrors.push(e);
496
+ }
497
+ try {
498
+ const binding = require('@crawlee/fs-storage-native-linux-loong64-musl');
499
+ const bindingPackageVersion =
500
+ require('@crawlee/fs-storage-native-linux-loong64-musl/package.json').version;
501
+ if (
502
+ bindingPackageVersion !== '0.1.0' &&
503
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
504
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
505
+ ) {
506
+ throw new Error(
507
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
508
+ );
509
+ }
510
+ return binding;
511
+ } catch (e) {
512
+ loadErrors.push(e);
513
+ }
514
+ } else {
515
+ try {
516
+ return require('./fs-storage-native.linux-loong64-gnu.node');
517
+ } catch (e) {
518
+ loadErrors.push(e);
519
+ }
520
+ try {
521
+ const binding = require('@crawlee/fs-storage-native-linux-loong64-gnu');
522
+ const bindingPackageVersion =
523
+ require('@crawlee/fs-storage-native-linux-loong64-gnu/package.json').version;
524
+ if (
525
+ bindingPackageVersion !== '0.1.0' &&
526
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
527
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
528
+ ) {
529
+ throw new Error(
530
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
531
+ );
532
+ }
533
+ return binding;
534
+ } catch (e) {
535
+ loadErrors.push(e);
536
+ }
537
+ }
538
+ } else if (process.arch === 'riscv64') {
539
+ if (isMusl()) {
540
+ try {
541
+ return require('./fs-storage-native.linux-riscv64-musl.node');
542
+ } catch (e) {
543
+ loadErrors.push(e);
544
+ }
545
+ try {
546
+ const binding = require('@crawlee/fs-storage-native-linux-riscv64-musl');
547
+ const bindingPackageVersion =
548
+ require('@crawlee/fs-storage-native-linux-riscv64-musl/package.json').version;
549
+ if (
550
+ bindingPackageVersion !== '0.1.0' &&
551
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
552
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
553
+ ) {
554
+ throw new Error(
555
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
556
+ );
557
+ }
558
+ return binding;
559
+ } catch (e) {
560
+ loadErrors.push(e);
561
+ }
562
+ } else {
563
+ try {
564
+ return require('./fs-storage-native.linux-riscv64-gnu.node');
565
+ } catch (e) {
566
+ loadErrors.push(e);
567
+ }
568
+ try {
569
+ const binding = require('@crawlee/fs-storage-native-linux-riscv64-gnu');
570
+ const bindingPackageVersion =
571
+ require('@crawlee/fs-storage-native-linux-riscv64-gnu/package.json').version;
572
+ if (
573
+ bindingPackageVersion !== '0.1.0' &&
574
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
575
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
576
+ ) {
577
+ throw new Error(
578
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
579
+ );
580
+ }
581
+ return binding;
582
+ } catch (e) {
583
+ loadErrors.push(e);
584
+ }
585
+ }
586
+ } else if (process.arch === 'ppc64') {
587
+ try {
588
+ return require('./fs-storage-native.linux-ppc64-gnu.node');
589
+ } catch (e) {
590
+ loadErrors.push(e);
591
+ }
592
+ try {
593
+ const binding = require('@crawlee/fs-storage-native-linux-ppc64-gnu');
594
+ const bindingPackageVersion =
595
+ require('@crawlee/fs-storage-native-linux-ppc64-gnu/package.json').version;
596
+ if (
597
+ bindingPackageVersion !== '0.1.0' &&
598
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
599
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
600
+ ) {
601
+ throw new Error(
602
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
603
+ );
604
+ }
605
+ return binding;
606
+ } catch (e) {
607
+ loadErrors.push(e);
608
+ }
609
+ } else if (process.arch === 's390x') {
610
+ try {
611
+ return require('./fs-storage-native.linux-s390x-gnu.node');
612
+ } catch (e) {
613
+ loadErrors.push(e);
614
+ }
615
+ try {
616
+ const binding = require('@crawlee/fs-storage-native-linux-s390x-gnu');
617
+ const bindingPackageVersion =
618
+ require('@crawlee/fs-storage-native-linux-s390x-gnu/package.json').version;
619
+ if (
620
+ bindingPackageVersion !== '0.1.0' &&
621
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
622
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
623
+ ) {
624
+ throw new Error(
625
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
626
+ );
627
+ }
628
+ return binding;
629
+ } catch (e) {
630
+ loadErrors.push(e);
631
+ }
632
+ } else {
633
+ loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`));
634
+ }
635
+ } else if (process.platform === 'openharmony') {
636
+ if (process.arch === 'arm64') {
637
+ try {
638
+ return require('./fs-storage-native.openharmony-arm64.node');
639
+ } catch (e) {
640
+ loadErrors.push(e);
641
+ }
642
+ try {
643
+ const binding = require('@crawlee/fs-storage-native-openharmony-arm64');
644
+ const bindingPackageVersion =
645
+ require('@crawlee/fs-storage-native-openharmony-arm64/package.json').version;
646
+ if (
647
+ bindingPackageVersion !== '0.1.0' &&
648
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
649
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
650
+ ) {
651
+ throw new Error(
652
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
653
+ );
654
+ }
655
+ return binding;
656
+ } catch (e) {
657
+ loadErrors.push(e);
658
+ }
659
+ } else if (process.arch === 'x64') {
660
+ try {
661
+ return require('./fs-storage-native.openharmony-x64.node');
662
+ } catch (e) {
663
+ loadErrors.push(e);
664
+ }
665
+ try {
666
+ const binding = require('@crawlee/fs-storage-native-openharmony-x64');
667
+ const bindingPackageVersion =
668
+ require('@crawlee/fs-storage-native-openharmony-x64/package.json').version;
669
+ if (
670
+ bindingPackageVersion !== '0.1.0' &&
671
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
672
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
673
+ ) {
674
+ throw new Error(
675
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
676
+ );
677
+ }
678
+ return binding;
679
+ } catch (e) {
680
+ loadErrors.push(e);
681
+ }
682
+ } else if (process.arch === 'arm') {
683
+ try {
684
+ return require('./fs-storage-native.openharmony-arm.node');
685
+ } catch (e) {
686
+ loadErrors.push(e);
687
+ }
688
+ try {
689
+ const binding = require('@crawlee/fs-storage-native-openharmony-arm');
690
+ const bindingPackageVersion =
691
+ require('@crawlee/fs-storage-native-openharmony-arm/package.json').version;
692
+ if (
693
+ bindingPackageVersion !== '0.1.0' &&
694
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK &&
695
+ process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0'
696
+ ) {
697
+ throw new Error(
698
+ `Native binding package version mismatch, expected 0.1.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`,
699
+ );
700
+ }
701
+ return binding;
702
+ } catch (e) {
703
+ loadErrors.push(e);
704
+ }
705
+ } else {
706
+ loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`));
707
+ }
708
+ } else {
709
+ loadErrors.push(
710
+ new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`),
711
+ );
712
+ }
713
+ }
714
+
715
+ nativeBinding = requireNative();
716
+
717
+ if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
718
+ let wasiBinding = null;
719
+ let wasiBindingError = null;
720
+ try {
721
+ wasiBinding = require('./fs-storage-native.wasi.cjs');
722
+ nativeBinding = wasiBinding;
723
+ } catch (err) {
724
+ if (process.env.NAPI_RS_FORCE_WASI) {
725
+ wasiBindingError = err;
726
+ }
727
+ }
728
+ if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
729
+ try {
730
+ wasiBinding = require('@crawlee/fs-storage-native-wasm32-wasi');
731
+ nativeBinding = wasiBinding;
732
+ } catch (err) {
733
+ if (process.env.NAPI_RS_FORCE_WASI) {
734
+ if (!wasiBindingError) {
735
+ wasiBindingError = err;
736
+ } else {
737
+ wasiBindingError.cause = err;
738
+ }
739
+ loadErrors.push(err);
740
+ }
741
+ }
742
+ }
743
+ if (process.env.NAPI_RS_FORCE_WASI === 'error' && !wasiBinding) {
744
+ const error = new Error('WASI binding not found and NAPI_RS_FORCE_WASI is set to error');
745
+ error.cause = wasiBindingError;
746
+ throw error;
747
+ }
748
+ }
749
+
750
+ if (!nativeBinding) {
751
+ if (loadErrors.length > 0) {
752
+ throw new Error(
753
+ `Cannot find native binding. ` +
754
+ `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
755
+ 'Please try `npm i` again after removing both package-lock.json and node_modules directory.',
756
+ {
757
+ cause: loadErrors.reduce((err, cur) => {
758
+ cur.cause = err;
759
+ return cur;
760
+ }),
761
+ },
762
+ );
763
+ }
764
+ throw new Error(`Failed to load native binding`);
765
+ }
766
+
767
+ module.exports = nativeBinding;
768
+ module.exports.DatasetItemIterator = nativeBinding.DatasetItemIterator;
769
+ module.exports.FileSystemDatasetClient = nativeBinding.FileSystemDatasetClient;
770
+ module.exports.FileSystemKeyValueStoreClient = nativeBinding.FileSystemKeyValueStoreClient;
771
+ module.exports.FileSystemRequestQueueClient = nativeBinding.FileSystemRequestQueueClient;
772
+ module.exports.KvsKeyIterator = nativeBinding.KvsKeyIterator;
package/lib.d.ts ADDED
@@ -0,0 +1,25 @@
1
+ export * from './index';
2
+
3
+ import { DatasetItemIterator, FileSystemKeyValueStoreClient, KvsKeyIterator } from './index';
4
+ import type { KeyValueStoreRecordMetadata, KeyValueStoreStreamRecord } from './index';
5
+
6
+ declare module './index' {
7
+ interface DatasetItemIterator {
8
+ [Symbol.asyncIterator](): AsyncIterator<Record<string, unknown>>;
9
+ }
10
+
11
+ interface KvsKeyIterator {
12
+ [Symbol.asyncIterator](): AsyncIterator<KeyValueStoreRecordMetadata>;
13
+ }
14
+
15
+ interface FileSystemKeyValueStoreClient {
16
+ /** Get a value as a ReadableStream of bytes. Returns null if the key doesn't exist. */
17
+ getValueStream(key: string): Promise<KeyValueStoreStreamRecord | null>;
18
+ /** Set a value from a ReadableStream. Consumes the entire stream and writes atomically. */
19
+ setValueStream(
20
+ key: string,
21
+ stream: ReadableStream<Uint8Array>,
22
+ contentType?: string | null,
23
+ ): Promise<void>;
24
+ }
25
+ }
package/lib.js ADDED
@@ -0,0 +1,93 @@
1
+ const { createReadStream, createWriteStream } = require('fs');
2
+ const { unlink } = require('fs/promises');
3
+ const { Readable, Writable } = require('stream');
4
+
5
+ const native = require('./index.js');
6
+
7
+ // Add Symbol.asyncIterator to DatasetItemIterator so users can write:
8
+ // for await (const item of client.iterateItems()) { ... }
9
+ native.DatasetItemIterator.prototype[Symbol.asyncIterator] = function () {
10
+ return {
11
+ next: async () => {
12
+ const value = await this.next();
13
+ if (value === null) {
14
+ return { done: true, value: undefined };
15
+ }
16
+ return { done: false, value };
17
+ },
18
+ };
19
+ };
20
+
21
+ // Same for KvsKeyIterator.
22
+ native.KvsKeyIterator.prototype[Symbol.asyncIterator] = function () {
23
+ return {
24
+ next: async () => {
25
+ const value = await this.next();
26
+ if (value === null) {
27
+ return { done: true, value: undefined };
28
+ }
29
+ return { done: false, value };
30
+ },
31
+ };
32
+ };
33
+
34
+ // Wrap getValue to convert the byte-array value to a real Buffer.
35
+ const origGetValue = native.FileSystemKeyValueStoreClient.prototype.getValue;
36
+ native.FileSystemKeyValueStoreClient.prototype.getValue = async function (...args) {
37
+ const record = await origGetValue.apply(this, args);
38
+ if (record) {
39
+ record.value = Buffer.from(record.value);
40
+ }
41
+ return record;
42
+ };
43
+
44
+ // getValueStream: returns { key, contentType, size, stream } or null.
45
+ // The stream is a Web ReadableStream<Uint8Array> created from the file on disk.
46
+ native.FileSystemKeyValueStoreClient.prototype.getValueStream = async function (key) {
47
+ const info = await this._getValueFileInfo(key);
48
+ if (info === null) {
49
+ return null;
50
+ }
51
+
52
+ const nodeStream = createReadStream(info.filePath);
53
+ const stream = Readable.toWeb(nodeStream);
54
+
55
+ return {
56
+ key: info.key,
57
+ contentType: info.contentType,
58
+ size: info.size,
59
+ stream,
60
+ };
61
+ };
62
+
63
+ // setValueStream: pipes a ReadableStream directly to a temp file on disk,
64
+ // then atomically renames it into place. No buffering in memory.
65
+ native.FileSystemKeyValueStoreClient.prototype.setValueStream = async function (
66
+ key,
67
+ stream,
68
+ contentType,
69
+ ) {
70
+ const tempPath = this._getTempFilePath();
71
+ const ws = createWriteStream(tempPath);
72
+ const writable = Writable.toWeb(ws);
73
+
74
+ let size = 0;
75
+ const transform = new TransformStream({
76
+ transform(chunk, controller) {
77
+ size += chunk.byteLength;
78
+ controller.enqueue(chunk);
79
+ },
80
+ });
81
+
82
+ try {
83
+ await stream.pipeThrough(transform).pipeTo(writable);
84
+ } catch (err) {
85
+ await unlink(tempPath).catch(() => {});
86
+ throw err;
87
+ }
88
+
89
+ const ct = contentType ?? 'application/octet-stream';
90
+ return this._finalizeStreamedValue(key, tempPath, size, ct);
91
+ };
92
+
93
+ module.exports = native;
package/package.json ADDED
@@ -0,0 +1,59 @@
1
+ {
2
+ "name": "@crawlee/fs-storage-native",
3
+ "version": "0.0.1-placeholder.0",
4
+ "description": "Low-level Rust-powered napi-rs bindings for Crawlee's filesystem storage. Not intended for direct use in crawlers — depend on @crawlee/fs-storage instead.",
5
+ "homepage": "https://github.com/apify/crawlee-storage",
6
+ "license": "Apache-2.0",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/apify/crawlee-storage.git"
10
+ },
11
+ "files": [
12
+ "index.js",
13
+ "index.d.ts",
14
+ "lib.js",
15
+ "lib.d.ts"
16
+ ],
17
+ "main": "lib.js",
18
+ "types": "lib.d.ts",
19
+ "scripts": {
20
+ "artifacts": "napi artifacts --output-dir ../artifacts --npm-dir npm",
21
+ "build": "napi build --platform --release --no-const-enum",
22
+ "build:debug": "napi build --platform --no-const-enum",
23
+ "prepublishOnly": "napi prepublish -t npm --no-gh-release",
24
+ "copy-version": "napi version",
25
+ "test": "vitest run",
26
+ "lint": "oxlint",
27
+ "lint:fix": "oxlint --fix",
28
+ "fmt": "oxfmt",
29
+ "fmt:check": "oxfmt --check"
30
+ },
31
+ "devDependencies": {
32
+ "@napi-rs/cli": "^3",
33
+ "@types/node": "^22",
34
+ "oxfmt": "^0.44.0",
35
+ "oxlint": "^1.59.0",
36
+ "oxlint-tsgolint": "^0.23.0",
37
+ "typescript": "^5",
38
+ "vitest": "^3"
39
+ },
40
+ "optionalDependencies": {
41
+ "@crawlee/fs-storage-native-darwin-arm64": "0.0.1-placeholder.0",
42
+ "@crawlee/fs-storage-native-darwin-x64": "0.0.1-placeholder.0",
43
+ "@crawlee/fs-storage-native-linux-x64-gnu": "0.0.1-placeholder.0",
44
+ "@crawlee/fs-storage-native-win32-x64-msvc": "0.0.1-placeholder.0"
45
+ },
46
+ "napi": {
47
+ "binaryName": "fs-storage-native",
48
+ "dtsHeaderFile": "dts-header.d.ts",
49
+ "targets": [
50
+ "x86_64-unknown-linux-gnu",
51
+ "x86_64-apple-darwin",
52
+ "aarch64-apple-darwin",
53
+ "x86_64-pc-windows-msvc"
54
+ ]
55
+ },
56
+ "engines": {
57
+ "node": ">= 20"
58
+ }
59
+ }