@crawlee/fs-storage-native 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +42 -78
- package/index.js +555 -729
- package/lib.js +11 -15
- package/package.json +14 -7
package/lib.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { createReadStream, createWriteStream } from 'fs';
|
|
2
|
+
import { unlink } from 'fs/promises';
|
|
3
|
+
import { Readable, Writable } from 'stream';
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
import { DatasetItemIterator, FileSystemKeyValueStoreClient, KvsKeyIterator } from './index.js';
|
|
6
6
|
|
|
7
7
|
// Add Symbol.asyncIterator to DatasetItemIterator so users can write:
|
|
8
8
|
// for await (const item of client.iterateItems()) { ... }
|
|
9
|
-
|
|
9
|
+
DatasetItemIterator.prototype[Symbol.asyncIterator] = function () {
|
|
10
10
|
return {
|
|
11
11
|
next: async () => {
|
|
12
12
|
const value = await this.next();
|
|
@@ -19,7 +19,7 @@ native.DatasetItemIterator.prototype[Symbol.asyncIterator] = function () {
|
|
|
19
19
|
};
|
|
20
20
|
|
|
21
21
|
// Same for KvsKeyIterator.
|
|
22
|
-
|
|
22
|
+
KvsKeyIterator.prototype[Symbol.asyncIterator] = function () {
|
|
23
23
|
return {
|
|
24
24
|
next: async () => {
|
|
25
25
|
const value = await this.next();
|
|
@@ -32,8 +32,8 @@ native.KvsKeyIterator.prototype[Symbol.asyncIterator] = function () {
|
|
|
32
32
|
};
|
|
33
33
|
|
|
34
34
|
// Wrap getValue to convert the byte-array value to a real Buffer.
|
|
35
|
-
const origGetValue =
|
|
36
|
-
|
|
35
|
+
const origGetValue = FileSystemKeyValueStoreClient.prototype.getValue;
|
|
36
|
+
FileSystemKeyValueStoreClient.prototype.getValue = async function (...args) {
|
|
37
37
|
const record = await origGetValue.apply(this, args);
|
|
38
38
|
if (record) {
|
|
39
39
|
record.value = Buffer.from(record.value);
|
|
@@ -43,7 +43,7 @@ native.FileSystemKeyValueStoreClient.prototype.getValue = async function (...arg
|
|
|
43
43
|
|
|
44
44
|
// getValueStream: returns { key, contentType, size, stream } or null.
|
|
45
45
|
// The stream is a Web ReadableStream<Uint8Array> created from the file on disk.
|
|
46
|
-
|
|
46
|
+
FileSystemKeyValueStoreClient.prototype.getValueStream = async function (key) {
|
|
47
47
|
const info = await this._getValueFileInfo(key);
|
|
48
48
|
if (info === null) {
|
|
49
49
|
return null;
|
|
@@ -62,11 +62,7 @@ native.FileSystemKeyValueStoreClient.prototype.getValueStream = async function (
|
|
|
62
62
|
|
|
63
63
|
// setValueStream: pipes a ReadableStream directly to a temp file on disk,
|
|
64
64
|
// then atomically renames it into place. No buffering in memory.
|
|
65
|
-
|
|
66
|
-
key,
|
|
67
|
-
stream,
|
|
68
|
-
contentType,
|
|
69
|
-
) {
|
|
65
|
+
FileSystemKeyValueStoreClient.prototype.setValueStream = async function (key, stream, contentType) {
|
|
70
66
|
const tempPath = this._getTempFilePath();
|
|
71
67
|
const ws = createWriteStream(tempPath);
|
|
72
68
|
const writable = Writable.toWeb(ws);
|
|
@@ -90,4 +86,4 @@ native.FileSystemKeyValueStoreClient.prototype.setValueStream = async function (
|
|
|
90
86
|
return this._finalizeStreamedValue(key, tempPath, size, ct);
|
|
91
87
|
};
|
|
92
88
|
|
|
93
|
-
|
|
89
|
+
export * from './index.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/fs-storage-native",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Low-level Rust-powered napi-rs bindings for Crawlee's filesystem storage. Not intended for direct use in crawlers — depend on @crawlee/fs-storage instead.",
|
|
5
5
|
"homepage": "https://github.com/apify/crawlee-storage",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -14,12 +14,19 @@
|
|
|
14
14
|
"lib.js",
|
|
15
15
|
"lib.d.ts"
|
|
16
16
|
],
|
|
17
|
+
"type": "module",
|
|
17
18
|
"main": "lib.js",
|
|
18
19
|
"types": "lib.d.ts",
|
|
20
|
+
"exports": {
|
|
21
|
+
".": {
|
|
22
|
+
"types": "./lib.d.ts",
|
|
23
|
+
"import": "./lib.js"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
19
26
|
"scripts": {
|
|
20
27
|
"artifacts": "napi artifacts --output-dir ../artifacts --npm-dir npm",
|
|
21
|
-
"build": "napi build --platform --release --no-const-enum",
|
|
22
|
-
"build:debug": "napi build --platform --no-const-enum",
|
|
28
|
+
"build": "napi build --platform --release --esm --no-const-enum",
|
|
29
|
+
"build:debug": "napi build --platform --esm --no-const-enum",
|
|
23
30
|
"prepublishOnly": "napi prepublish -t npm --no-gh-release",
|
|
24
31
|
"copy-version": "napi version",
|
|
25
32
|
"test": "vitest run",
|
|
@@ -38,10 +45,10 @@
|
|
|
38
45
|
"vitest": "^3"
|
|
39
46
|
},
|
|
40
47
|
"optionalDependencies": {
|
|
41
|
-
"@crawlee/fs-storage-native-darwin-arm64": "0.1.
|
|
42
|
-
"@crawlee/fs-storage-native-darwin-x64": "0.1.
|
|
43
|
-
"@crawlee/fs-storage-native-linux-x64-gnu": "0.1.
|
|
44
|
-
"@crawlee/fs-storage-native-win32-x64-msvc": "0.1.
|
|
48
|
+
"@crawlee/fs-storage-native-darwin-arm64": "0.1.4",
|
|
49
|
+
"@crawlee/fs-storage-native-darwin-x64": "0.1.4",
|
|
50
|
+
"@crawlee/fs-storage-native-linux-x64-gnu": "0.1.4",
|
|
51
|
+
"@crawlee/fs-storage-native-win32-x64-msvc": "0.1.4"
|
|
45
52
|
},
|
|
46
53
|
"napi": {
|
|
47
54
|
"binaryName": "fs-storage-native",
|