@remix-run/file-storage 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -35
- package/dist/fs.d.ts +2 -0
- package/dist/fs.d.ts.map +1 -0
- package/dist/fs.js +1 -0
- package/dist/lib/backends/fs.d.ts +17 -0
- package/dist/lib/backends/fs.d.ts.map +1 -0
- package/dist/lib/backends/fs.js +165 -0
- package/dist/lib/backends/memory.d.ts +7 -0
- package/dist/lib/backends/memory.d.ts.map +1 -0
- package/dist/lib/backends/memory.js +69 -0
- package/dist/lib/file-storage.d.ts.map +1 -1
- package/dist/memory.d.ts +1 -1
- package/dist/memory.d.ts.map +1 -1
- package/dist/memory.js +1 -1
- package/package.json +4 -4
- package/src/fs.ts +1 -0
- package/src/lib/backends/fs.ts +195 -0
- package/src/lib/backends/memory.ts +76 -0
- package/src/lib/file-storage.ts +0 -5
- package/src/memory.ts +1 -1
- package/dist/lib/local-file-storage.d.ts +0 -26
- package/dist/lib/local-file-storage.d.ts.map +0 -1
- package/dist/lib/local-file-storage.js +0 -158
- package/dist/lib/memory-file-storage.d.ts +0 -14
- package/dist/lib/memory-file-storage.d.ts.map +0 -1
- package/dist/lib/memory-file-storage.js +0 -63
- package/dist/local.d.ts +0 -2
- package/dist/local.d.ts.map +0 -1
- package/dist/local.js +0 -1
- package/src/lib/local-file-storage.ts +0 -189
- package/src/lib/memory-file-storage.ts +0 -75
- package/src/local.ts +0 -1
package/README.md
CHANGED
|
@@ -23,10 +23,12 @@ npm install @remix-run/file-storage
|
|
|
23
23
|
|
|
24
24
|
## Usage
|
|
25
25
|
|
|
26
|
+
### File System
|
|
27
|
+
|
|
26
28
|
```ts
|
|
27
|
-
import {
|
|
29
|
+
import { createFsFileStorage } from '@remix-run/file-storage/fs'
|
|
28
30
|
|
|
29
|
-
let storage =
|
|
31
|
+
let storage = createFsFileStorage('./user/files')
|
|
30
32
|
|
|
31
33
|
let file = new File(['hello world'], 'hello.txt', { type: 'text/plain' })
|
|
32
34
|
let key = 'hello-key'
|
|
@@ -44,39 +46,6 @@ fileFromStorage.type // 'text/plain'
|
|
|
44
46
|
await storage.remove(key)
|
|
45
47
|
```
|
|
46
48
|
|
|
47
|
-
The `FileStorage` interface allows you to implement your own file storage for custom storage backends:
|
|
48
|
-
|
|
49
|
-
```ts
|
|
50
|
-
import { type FileStorage } from '@remix-run/file-storage'
|
|
51
|
-
|
|
52
|
-
class CustomFileStorage implements FileStorage {
|
|
53
|
-
/**
|
|
54
|
-
* Returns `true` if a file with the given key exists, `false` otherwise.
|
|
55
|
-
*/
|
|
56
|
-
has(key: string): boolean | Promise<boolean> {
|
|
57
|
-
// ...
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Puts a file in storage at the given key.
|
|
61
|
-
*/
|
|
62
|
-
set(key: string, file: File): void | Promise<void> {
|
|
63
|
-
// ...
|
|
64
|
-
}
|
|
65
|
-
/**
|
|
66
|
-
* Returns the file with the given key, or `null` if no such key exists.
|
|
67
|
-
*/
|
|
68
|
-
get(key: string): File | null | Promise<File | null> {
|
|
69
|
-
// ...
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Removes the file with the given key from storage.
|
|
73
|
-
*/
|
|
74
|
-
remove(key: string): void | Promise<void> {
|
|
75
|
-
// ...
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
```
|
|
79
|
-
|
|
80
49
|
## Related Packages
|
|
81
50
|
|
|
82
51
|
- [`form-data-parser`](https://github.com/remix-run/remix/tree/main/packages/form-data-parser) - Pairs well with this library for storing `FileUpload` objects received in `multipart/form-data` requests
|
package/dist/fs.d.ts
ADDED
package/dist/fs.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../src/fs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA"}
|
package/dist/fs.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createFsFileStorage } from "./lib/backends/fs.js";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { FileStorage } from '../file-storage.ts';
|
|
2
|
+
/**
|
|
3
|
+
* Creates a `FileStorage` that is backed by a filesystem directory using node:fs.
|
|
4
|
+
*
|
|
5
|
+
* Important: No attempt is made to avoid overwriting existing files, so the directory used should
|
|
6
|
+
* be a new directory solely dedicated to this storage object.
|
|
7
|
+
*
|
|
8
|
+
* Note: Keys have no correlation to file names on disk, so they may be any string including
|
|
9
|
+
* characters that are not valid in file names. Additionally, individual `File` names have no
|
|
10
|
+
* correlation to names of files on disk, so multiple files with the same name may be stored in the
|
|
11
|
+
* same storage object.
|
|
12
|
+
*
|
|
13
|
+
* @param directory The directory where files are stored
|
|
14
|
+
* @returns A new file storage backed by a filesystem directory
|
|
15
|
+
*/
|
|
16
|
+
export declare function createFsFileStorage(directory: string): FileStorage;
|
|
17
|
+
//# sourceMappingURL=fs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs.d.ts","sourceRoot":"","sources":["../../../src/lib/backends/fs.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAyC,MAAM,oBAAoB,CAAA;AAI5F;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CA4JlE"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as fsp from 'node:fs/promises';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { openFile, writeFile } from '@remix-run/fs';
|
|
5
|
+
/**
|
|
6
|
+
* Creates a `FileStorage` that is backed by a filesystem directory using node:fs.
|
|
7
|
+
*
|
|
8
|
+
* Important: No attempt is made to avoid overwriting existing files, so the directory used should
|
|
9
|
+
* be a new directory solely dedicated to this storage object.
|
|
10
|
+
*
|
|
11
|
+
* Note: Keys have no correlation to file names on disk, so they may be any string including
|
|
12
|
+
* characters that are not valid in file names. Additionally, individual `File` names have no
|
|
13
|
+
* correlation to names of files on disk, so multiple files with the same name may be stored in the
|
|
14
|
+
* same storage object.
|
|
15
|
+
*
|
|
16
|
+
* @param directory The directory where files are stored
|
|
17
|
+
* @returns A new file storage backed by a filesystem directory
|
|
18
|
+
*/
|
|
19
|
+
export function createFsFileStorage(directory) {
|
|
20
|
+
let rootDir = path.resolve(directory);
|
|
21
|
+
try {
|
|
22
|
+
let stats = fs.statSync(rootDir);
|
|
23
|
+
if (!stats.isDirectory()) {
|
|
24
|
+
throw new Error(`Path "${rootDir}" is not a directory`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (!isNoEntityError(error)) {
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
31
|
+
fs.mkdirSync(rootDir, { recursive: true });
|
|
32
|
+
}
|
|
33
|
+
async function getPaths(key) {
|
|
34
|
+
let hash = await computeHash(key);
|
|
35
|
+
let directory = path.join(rootDir, hash.slice(0, 2));
|
|
36
|
+
return {
|
|
37
|
+
directory,
|
|
38
|
+
filePath: path.join(directory, `${hash}.dat`),
|
|
39
|
+
metaPath: path.join(directory, `${hash}.meta.json`),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
async function putFile(key, file) {
|
|
43
|
+
let { directory, filePath, metaPath } = await getPaths(key);
|
|
44
|
+
// Ensure directory exists
|
|
45
|
+
await fsp.mkdir(directory, { recursive: true });
|
|
46
|
+
await writeFile(filePath, file);
|
|
47
|
+
let meta = {
|
|
48
|
+
key,
|
|
49
|
+
lastModified: file.lastModified,
|
|
50
|
+
name: file.name,
|
|
51
|
+
type: file.type,
|
|
52
|
+
};
|
|
53
|
+
await fsp.writeFile(metaPath, JSON.stringify(meta));
|
|
54
|
+
let metaData = await readMetadata(metaPath);
|
|
55
|
+
return openFile(filePath, {
|
|
56
|
+
lastModified: metaData.lastModified,
|
|
57
|
+
name: metaData.name,
|
|
58
|
+
type: metaData.type,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
async get(key) {
|
|
63
|
+
let { filePath, metaPath } = await getPaths(key);
|
|
64
|
+
try {
|
|
65
|
+
let meta = await readMetadata(metaPath);
|
|
66
|
+
return openFile(filePath, {
|
|
67
|
+
lastModified: meta.lastModified,
|
|
68
|
+
name: meta.name,
|
|
69
|
+
type: meta.type,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
if (!isNoEntityError(error)) {
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
async has(key) {
|
|
80
|
+
let { metaPath } = await getPaths(key);
|
|
81
|
+
try {
|
|
82
|
+
await fsp.access(metaPath);
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
async list(options) {
|
|
90
|
+
let { cursor, includeMetadata = false, limit = 32, prefix } = options ?? {};
|
|
91
|
+
let files = [];
|
|
92
|
+
let foundCursor = cursor === undefined;
|
|
93
|
+
let nextCursor;
|
|
94
|
+
let lastHash;
|
|
95
|
+
outerLoop: for await (let subdir of await fsp.opendir(rootDir)) {
|
|
96
|
+
if (!subdir.isDirectory())
|
|
97
|
+
continue;
|
|
98
|
+
for await (let file of await fsp.opendir(path.join(rootDir, subdir.name))) {
|
|
99
|
+
if (!file.isFile() || !file.name.endsWith('.meta.json'))
|
|
100
|
+
continue;
|
|
101
|
+
let hash = file.name.slice(0, -10); // Remove ".meta.json"
|
|
102
|
+
if (foundCursor) {
|
|
103
|
+
let meta = await readMetadata(path.join(rootDir, subdir.name, file.name));
|
|
104
|
+
if (prefix != null && !meta.key.startsWith(prefix)) {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (files.length >= limit) {
|
|
108
|
+
nextCursor = lastHash;
|
|
109
|
+
break outerLoop;
|
|
110
|
+
}
|
|
111
|
+
if (includeMetadata) {
|
|
112
|
+
let size = (await fsp.stat(path.join(rootDir, subdir.name, `${hash}.dat`))).size;
|
|
113
|
+
files.push({ ...meta, size });
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
files.push({ key: meta.key });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
else if (hash === cursor) {
|
|
120
|
+
foundCursor = true;
|
|
121
|
+
}
|
|
122
|
+
lastHash = hash;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return {
|
|
126
|
+
cursor: nextCursor,
|
|
127
|
+
files,
|
|
128
|
+
};
|
|
129
|
+
},
|
|
130
|
+
put(key, file) {
|
|
131
|
+
return putFile(key, file);
|
|
132
|
+
},
|
|
133
|
+
async remove(key) {
|
|
134
|
+
let { directory, filePath, metaPath } = await getPaths(key);
|
|
135
|
+
try {
|
|
136
|
+
await Promise.all([fsp.unlink(filePath), fsp.unlink(metaPath)]);
|
|
137
|
+
// Check if directory is empty and remove it if so
|
|
138
|
+
let files = await fsp.readdir(directory);
|
|
139
|
+
if (files.length === 0) {
|
|
140
|
+
await fsp.rmdir(directory);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
if (!isNoEntityError(error)) {
|
|
145
|
+
throw error;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
},
|
|
149
|
+
async set(key, file) {
|
|
150
|
+
await putFile(key, file);
|
|
151
|
+
},
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
async function readMetadata(metaPath) {
|
|
155
|
+
return JSON.parse(await fsp.readFile(metaPath, 'utf-8'));
|
|
156
|
+
}
|
|
157
|
+
async function computeHash(key, algorithm = 'SHA-256') {
|
|
158
|
+
let digest = await crypto.subtle.digest(algorithm, new TextEncoder().encode(key));
|
|
159
|
+
return Array.from(new Uint8Array(digest))
|
|
160
|
+
.map((b) => b.toString(16).padStart(2, '0'))
|
|
161
|
+
.join('');
|
|
162
|
+
}
|
|
163
|
+
function isNoEntityError(obj) {
|
|
164
|
+
return obj instanceof Error && 'code' in obj && obj.code === 'ENOENT';
|
|
165
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FileStorage } from '../file-storage.ts';
|
|
2
|
+
/**
|
|
3
|
+
* Creates a simple, in-memory implementation of the `FileStorage` interface.
|
|
4
|
+
* @returns A new in-memory file storage instance
|
|
5
|
+
*/
|
|
6
|
+
export declare function createMemoryFileStorage(): FileStorage;
|
|
7
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/lib/backends/memory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAA2B,MAAM,oBAAoB,CAAA;AAE9E;;;GAGG;AACH,wBAAgB,uBAAuB,IAAI,WAAW,CAqErD"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a simple, in-memory implementation of the `FileStorage` interface.
|
|
3
|
+
* @returns A new in-memory file storage instance
|
|
4
|
+
*/
|
|
5
|
+
export function createMemoryFileStorage() {
|
|
6
|
+
let map = new Map();
|
|
7
|
+
async function putFile(key, file) {
|
|
8
|
+
let buffer = await file.arrayBuffer();
|
|
9
|
+
let newFile = new File([buffer], file.name, {
|
|
10
|
+
lastModified: file.lastModified,
|
|
11
|
+
type: file.type,
|
|
12
|
+
});
|
|
13
|
+
map.set(key, newFile);
|
|
14
|
+
return newFile;
|
|
15
|
+
}
|
|
16
|
+
return {
|
|
17
|
+
get(key) {
|
|
18
|
+
return map.get(key) ?? null;
|
|
19
|
+
},
|
|
20
|
+
has(key) {
|
|
21
|
+
return map.has(key);
|
|
22
|
+
},
|
|
23
|
+
list(options) {
|
|
24
|
+
let { cursor, includeMetadata = false, limit = Infinity, prefix } = options ?? {};
|
|
25
|
+
let files = [];
|
|
26
|
+
let foundCursor = cursor === undefined;
|
|
27
|
+
let nextCursor;
|
|
28
|
+
for (let [key, file] of map.entries()) {
|
|
29
|
+
if (foundCursor) {
|
|
30
|
+
if (prefix != null && !key.startsWith(prefix)) {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (files.length >= limit) {
|
|
34
|
+
nextCursor = files[files.length - 1]?.key;
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
if (includeMetadata) {
|
|
38
|
+
files.push({
|
|
39
|
+
key,
|
|
40
|
+
lastModified: file.lastModified,
|
|
41
|
+
name: file.name,
|
|
42
|
+
size: file.size,
|
|
43
|
+
type: file.type,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
files.push({ key });
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
else if (key === cursor) {
|
|
51
|
+
foundCursor = true;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
cursor: nextCursor,
|
|
56
|
+
files,
|
|
57
|
+
};
|
|
58
|
+
},
|
|
59
|
+
put(key, file) {
|
|
60
|
+
return putFile(key, file);
|
|
61
|
+
},
|
|
62
|
+
remove(key) {
|
|
63
|
+
map.delete(key);
|
|
64
|
+
},
|
|
65
|
+
async set(key, file) {
|
|
66
|
+
await putFile(key, file);
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-storage.d.ts","sourceRoot":"","sources":["../../src/lib/file-storage.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"file-storage.d.ts","sourceRoot":"","sources":["../../src/lib/file-storage.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;IACpD;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0DG;IACH,IAAI,CAAC,CAAC,SAAS,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;IAChF;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAClD;;;;OAIG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACzC;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACnD;AAED,MAAM,WAAW,OAAO;IACtB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,WAAW,YAAa,SAAQ,OAAO;IAC3C;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,UAAU,CAAC,CAAC,SAAS,WAAW;IAC/C;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,KAAK,EAAE,CAAC,CAAC,SAAS;QAAE,eAAe,EAAE,IAAI,CAAA;KAAE,GAAG,YAAY,GAAG,OAAO,CAAC,EAAE,CAAA;CACxE"}
|
package/dist/memory.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { createMemoryFileStorage } from './lib/backends/memory.ts';
|
|
2
2
|
//# sourceMappingURL=memory.d.ts.map
|
package/dist/memory.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../src/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../src/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAA"}
|
package/dist/memory.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { createMemoryFileStorage } from "./lib/backends/memory.js";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remix-run/file-storage",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.13.0",
|
|
4
4
|
"description": "Key/value storage for JavaScript File objects",
|
|
5
5
|
"author": "Michael Jackson <mjijackson@gmail.com>",
|
|
6
6
|
"repository": {
|
|
@@ -23,9 +23,9 @@
|
|
|
23
23
|
"types": "./dist/index.d.ts",
|
|
24
24
|
"default": "./dist/index.js"
|
|
25
25
|
},
|
|
26
|
-
"./
|
|
27
|
-
"types": "./dist/
|
|
28
|
-
"default": "./dist/
|
|
26
|
+
"./fs": {
|
|
27
|
+
"types": "./dist/fs.d.ts",
|
|
28
|
+
"default": "./dist/fs.js"
|
|
29
29
|
},
|
|
30
30
|
"./memory": {
|
|
31
31
|
"types": "./dist/memory.d.ts",
|
package/src/fs.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { createFsFileStorage } from './lib/backends/fs.ts'
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import * as fs from 'node:fs'
|
|
2
|
+
import * as fsp from 'node:fs/promises'
|
|
3
|
+
import * as path from 'node:path'
|
|
4
|
+
import { openFile, writeFile } from '@remix-run/fs'
|
|
5
|
+
|
|
6
|
+
import type { FileStorage, FileMetadata, ListOptions, ListResult } from '../file-storage.ts'
|
|
7
|
+
|
|
8
|
+
type MetadataJson = Omit<FileMetadata, 'size'>
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Creates a `FileStorage` that is backed by a filesystem directory using node:fs.
|
|
12
|
+
*
|
|
13
|
+
* Important: No attempt is made to avoid overwriting existing files, so the directory used should
|
|
14
|
+
* be a new directory solely dedicated to this storage object.
|
|
15
|
+
*
|
|
16
|
+
* Note: Keys have no correlation to file names on disk, so they may be any string including
|
|
17
|
+
* characters that are not valid in file names. Additionally, individual `File` names have no
|
|
18
|
+
* correlation to names of files on disk, so multiple files with the same name may be stored in the
|
|
19
|
+
* same storage object.
|
|
20
|
+
*
|
|
21
|
+
* @param directory The directory where files are stored
|
|
22
|
+
* @returns A new file storage backed by a filesystem directory
|
|
23
|
+
*/
|
|
24
|
+
export function createFsFileStorage(directory: string): FileStorage {
|
|
25
|
+
let rootDir = path.resolve(directory)
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
let stats = fs.statSync(rootDir)
|
|
29
|
+
|
|
30
|
+
if (!stats.isDirectory()) {
|
|
31
|
+
throw new Error(`Path "${rootDir}" is not a directory`)
|
|
32
|
+
}
|
|
33
|
+
} catch (error) {
|
|
34
|
+
if (!isNoEntityError(error)) {
|
|
35
|
+
throw error
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
fs.mkdirSync(rootDir, { recursive: true })
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function getPaths(
|
|
42
|
+
key: string,
|
|
43
|
+
): Promise<{ directory: string; filePath: string; metaPath: string }> {
|
|
44
|
+
let hash = await computeHash(key)
|
|
45
|
+
let directory = path.join(rootDir, hash.slice(0, 2))
|
|
46
|
+
|
|
47
|
+
return {
|
|
48
|
+
directory,
|
|
49
|
+
filePath: path.join(directory, `${hash}.dat`),
|
|
50
|
+
metaPath: path.join(directory, `${hash}.meta.json`),
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function putFile(key: string, file: File): Promise<File> {
|
|
55
|
+
let { directory, filePath, metaPath } = await getPaths(key)
|
|
56
|
+
|
|
57
|
+
// Ensure directory exists
|
|
58
|
+
await fsp.mkdir(directory, { recursive: true })
|
|
59
|
+
|
|
60
|
+
await writeFile(filePath, file)
|
|
61
|
+
|
|
62
|
+
let meta: MetadataJson = {
|
|
63
|
+
key,
|
|
64
|
+
lastModified: file.lastModified,
|
|
65
|
+
name: file.name,
|
|
66
|
+
type: file.type,
|
|
67
|
+
}
|
|
68
|
+
await fsp.writeFile(metaPath, JSON.stringify(meta))
|
|
69
|
+
|
|
70
|
+
let metaData = await readMetadata(metaPath)
|
|
71
|
+
|
|
72
|
+
return openFile(filePath, {
|
|
73
|
+
lastModified: metaData.lastModified,
|
|
74
|
+
name: metaData.name,
|
|
75
|
+
type: metaData.type,
|
|
76
|
+
})
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
async get(key: string): Promise<File | null> {
|
|
81
|
+
let { filePath, metaPath } = await getPaths(key)
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
let meta = await readMetadata(metaPath)
|
|
85
|
+
|
|
86
|
+
return openFile(filePath, {
|
|
87
|
+
lastModified: meta.lastModified,
|
|
88
|
+
name: meta.name,
|
|
89
|
+
type: meta.type,
|
|
90
|
+
})
|
|
91
|
+
} catch (error) {
|
|
92
|
+
if (!isNoEntityError(error)) {
|
|
93
|
+
throw error
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return null
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
async has(key: string): Promise<boolean> {
|
|
100
|
+
let { metaPath } = await getPaths(key)
|
|
101
|
+
|
|
102
|
+
try {
|
|
103
|
+
await fsp.access(metaPath)
|
|
104
|
+
return true
|
|
105
|
+
} catch {
|
|
106
|
+
return false
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
async list<opts extends ListOptions>(options?: opts): Promise<ListResult<opts>> {
|
|
110
|
+
let { cursor, includeMetadata = false, limit = 32, prefix } = options ?? {}
|
|
111
|
+
|
|
112
|
+
let files: any[] = []
|
|
113
|
+
let foundCursor = cursor === undefined
|
|
114
|
+
let nextCursor: string | undefined
|
|
115
|
+
let lastHash: string | undefined
|
|
116
|
+
|
|
117
|
+
outerLoop: for await (let subdir of await fsp.opendir(rootDir)) {
|
|
118
|
+
if (!subdir.isDirectory()) continue
|
|
119
|
+
|
|
120
|
+
for await (let file of await fsp.opendir(path.join(rootDir, subdir.name))) {
|
|
121
|
+
if (!file.isFile() || !file.name.endsWith('.meta.json')) continue
|
|
122
|
+
|
|
123
|
+
let hash = file.name.slice(0, -10) // Remove ".meta.json"
|
|
124
|
+
|
|
125
|
+
if (foundCursor) {
|
|
126
|
+
let meta = await readMetadata(path.join(rootDir, subdir.name, file.name))
|
|
127
|
+
|
|
128
|
+
if (prefix != null && !meta.key.startsWith(prefix)) {
|
|
129
|
+
continue
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (files.length >= limit) {
|
|
133
|
+
nextCursor = lastHash
|
|
134
|
+
break outerLoop
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (includeMetadata) {
|
|
138
|
+
let size = (await fsp.stat(path.join(rootDir, subdir.name, `${hash}.dat`))).size
|
|
139
|
+
files.push({ ...meta, size })
|
|
140
|
+
} else {
|
|
141
|
+
files.push({ key: meta.key })
|
|
142
|
+
}
|
|
143
|
+
} else if (hash === cursor) {
|
|
144
|
+
foundCursor = true
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
lastHash = hash
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
cursor: nextCursor,
|
|
153
|
+
files,
|
|
154
|
+
}
|
|
155
|
+
},
|
|
156
|
+
put(key: string, file: File): Promise<File> {
|
|
157
|
+
return putFile(key, file)
|
|
158
|
+
},
|
|
159
|
+
async remove(key: string): Promise<void> {
|
|
160
|
+
let { directory, filePath, metaPath } = await getPaths(key)
|
|
161
|
+
|
|
162
|
+
try {
|
|
163
|
+
await Promise.all([fsp.unlink(filePath), fsp.unlink(metaPath)])
|
|
164
|
+
|
|
165
|
+
// Check if directory is empty and remove it if so
|
|
166
|
+
let files = await fsp.readdir(directory)
|
|
167
|
+
if (files.length === 0) {
|
|
168
|
+
await fsp.rmdir(directory)
|
|
169
|
+
}
|
|
170
|
+
} catch (error) {
|
|
171
|
+
if (!isNoEntityError(error)) {
|
|
172
|
+
throw error
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
},
|
|
176
|
+
async set(key: string, file: File): Promise<void> {
|
|
177
|
+
await putFile(key, file)
|
|
178
|
+
},
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
async function readMetadata(metaPath: string): Promise<MetadataJson> {
|
|
183
|
+
return JSON.parse(await fsp.readFile(metaPath, 'utf-8'))
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function computeHash(key: string, algorithm = 'SHA-256'): Promise<string> {
|
|
187
|
+
let digest = await crypto.subtle.digest(algorithm, new TextEncoder().encode(key))
|
|
188
|
+
return Array.from(new Uint8Array(digest))
|
|
189
|
+
.map((b) => b.toString(16).padStart(2, '0'))
|
|
190
|
+
.join('')
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function isNoEntityError(obj: unknown): obj is NodeJS.ErrnoException & { code: 'ENOENT' } {
|
|
194
|
+
return obj instanceof Error && 'code' in obj && (obj as NodeJS.ErrnoException).code === 'ENOENT'
|
|
195
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { FileStorage, ListOptions, ListResult } from '../file-storage.ts'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Creates a simple, in-memory implementation of the `FileStorage` interface.
|
|
5
|
+
* @returns A new in-memory file storage instance
|
|
6
|
+
*/
|
|
7
|
+
export function createMemoryFileStorage(): FileStorage {
|
|
8
|
+
let map = new Map<string, File>()
|
|
9
|
+
|
|
10
|
+
async function putFile(key: string, file: File): Promise<File> {
|
|
11
|
+
let buffer = await file.arrayBuffer()
|
|
12
|
+
let newFile = new File([buffer], file.name, {
|
|
13
|
+
lastModified: file.lastModified,
|
|
14
|
+
type: file.type,
|
|
15
|
+
})
|
|
16
|
+
map.set(key, newFile)
|
|
17
|
+
return newFile
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
get(key: string): File | null {
|
|
22
|
+
return map.get(key) ?? null
|
|
23
|
+
},
|
|
24
|
+
has(key: string): boolean {
|
|
25
|
+
return map.has(key)
|
|
26
|
+
},
|
|
27
|
+
list<opts extends ListOptions>(options?: opts): ListResult<opts> {
|
|
28
|
+
let { cursor, includeMetadata = false, limit = Infinity, prefix } = options ?? {}
|
|
29
|
+
|
|
30
|
+
let files: any[] = []
|
|
31
|
+
let foundCursor = cursor === undefined
|
|
32
|
+
let nextCursor: string | undefined
|
|
33
|
+
|
|
34
|
+
for (let [key, file] of map.entries()) {
|
|
35
|
+
if (foundCursor) {
|
|
36
|
+
if (prefix != null && !key.startsWith(prefix)) {
|
|
37
|
+
continue
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (files.length >= limit) {
|
|
41
|
+
nextCursor = files[files.length - 1]?.key
|
|
42
|
+
break
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (includeMetadata) {
|
|
46
|
+
files.push({
|
|
47
|
+
key,
|
|
48
|
+
lastModified: file.lastModified,
|
|
49
|
+
name: file.name,
|
|
50
|
+
size: file.size,
|
|
51
|
+
type: file.type,
|
|
52
|
+
})
|
|
53
|
+
} else {
|
|
54
|
+
files.push({ key })
|
|
55
|
+
}
|
|
56
|
+
} else if (key === cursor) {
|
|
57
|
+
foundCursor = true
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return {
|
|
62
|
+
cursor: nextCursor,
|
|
63
|
+
files,
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
put(key: string, file: File): Promise<File> {
|
|
67
|
+
return putFile(key, file)
|
|
68
|
+
},
|
|
69
|
+
remove(key: string): void {
|
|
70
|
+
map.delete(key)
|
|
71
|
+
},
|
|
72
|
+
async set(key: string, file: File): Promise<void> {
|
|
73
|
+
await putFile(key, file)
|
|
74
|
+
},
|
|
75
|
+
}
|
|
76
|
+
}
|
package/src/lib/file-storage.ts
CHANGED
|
@@ -8,14 +8,12 @@ export interface FileStorage {
|
|
|
8
8
|
* @returns The file with the given key, or `null` if no such key exists
|
|
9
9
|
*/
|
|
10
10
|
get(key: string): File | null | Promise<File | null>
|
|
11
|
-
|
|
12
11
|
/**
|
|
13
12
|
* Check if a file with the given key exists.
|
|
14
13
|
* @param key The key to look up
|
|
15
14
|
* @returns `true` if a file with the given key exists, `false` otherwise
|
|
16
15
|
*/
|
|
17
16
|
has(key: string): boolean | Promise<boolean>
|
|
18
|
-
|
|
19
17
|
/**
|
|
20
18
|
* List the files in storage.
|
|
21
19
|
*
|
|
@@ -76,7 +74,6 @@ export interface FileStorage {
|
|
|
76
74
|
* @returns An object with an array of `files` and an optional `cursor` property
|
|
77
75
|
*/
|
|
78
76
|
list<T extends ListOptions>(options?: T): ListResult<T> | Promise<ListResult<T>>
|
|
79
|
-
|
|
80
77
|
/**
|
|
81
78
|
* Put a [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File) in storage and return
|
|
82
79
|
* a new file backed by this storage.
|
|
@@ -85,14 +82,12 @@ export interface FileStorage {
|
|
|
85
82
|
* @returns A new File object backed by this storage
|
|
86
83
|
*/
|
|
87
84
|
put(key: string, file: File): File | Promise<File>
|
|
88
|
-
|
|
89
85
|
/**
|
|
90
86
|
* Remove the file with the given key from storage.
|
|
91
87
|
* @param key The key to remove
|
|
92
88
|
* @returns A promise that resolves when the file has been removed
|
|
93
89
|
*/
|
|
94
90
|
remove(key: string): void | Promise<void>
|
|
95
|
-
|
|
96
91
|
/**
|
|
97
92
|
* Put a [`File`](https://developer.mozilla.org/en-US/docs/Web/API/File) in storage at the given
|
|
98
93
|
* key.
|
package/src/memory.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { createMemoryFileStorage } from './lib/backends/memory.ts'
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import type { FileStorage, ListOptions, ListResult } from './file-storage.ts';
|
|
2
|
-
/**
|
|
3
|
-
* A `FileStorage` that is backed by a directory on the local filesystem.
|
|
4
|
-
*
|
|
5
|
-
* Important: No attempt is made to avoid overwriting existing files, so the directory used should
|
|
6
|
-
* be a new directory solely dedicated to this storage object.
|
|
7
|
-
*
|
|
8
|
-
* Note: Keys have no correlation to file names on disk, so they may be any string including
|
|
9
|
-
* characters that are not valid in file names. Additionally, individual `File` names have no
|
|
10
|
-
* correlation to names of files on disk, so multiple files with the same name may be stored in the
|
|
11
|
-
* same storage object.
|
|
12
|
-
*/
|
|
13
|
-
export declare class LocalFileStorage implements FileStorage {
|
|
14
|
-
#private;
|
|
15
|
-
/**
|
|
16
|
-
* @param directory The directory where files are stored
|
|
17
|
-
*/
|
|
18
|
-
constructor(directory: string);
|
|
19
|
-
get(key: string): Promise<File | null>;
|
|
20
|
-
has(key: string): Promise<boolean>;
|
|
21
|
-
list<T extends ListOptions>(options?: T): Promise<ListResult<T>>;
|
|
22
|
-
put(key: string, file: File): Promise<File>;
|
|
23
|
-
remove(key: string): Promise<void>;
|
|
24
|
-
set(key: string, file: File): Promise<void>;
|
|
25
|
-
}
|
|
26
|
-
//# sourceMappingURL=local-file-storage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"local-file-storage.d.ts","sourceRoot":"","sources":["../../src/lib/local-file-storage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAgB,WAAW,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAI3F;;;;;;;;;;GAUG;AACH,qBAAa,gBAAiB,YAAW,WAAW;;IAGlD;;OAEG;gBACS,SAAS,EAAE,MAAM;IAkBvB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAoBtC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWlC,IAAI,CAAC,CAAC,SAAS,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAgDhE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CA2BlD"}
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
import * as fs from 'node:fs';
|
|
2
|
-
import * as fsp from 'node:fs/promises';
|
|
3
|
-
import * as path from 'node:path';
|
|
4
|
-
import { openFile, writeFile } from '@remix-run/fs';
|
|
5
|
-
/**
|
|
6
|
-
* A `FileStorage` that is backed by a directory on the local filesystem.
|
|
7
|
-
*
|
|
8
|
-
* Important: No attempt is made to avoid overwriting existing files, so the directory used should
|
|
9
|
-
* be a new directory solely dedicated to this storage object.
|
|
10
|
-
*
|
|
11
|
-
* Note: Keys have no correlation to file names on disk, so they may be any string including
|
|
12
|
-
* characters that are not valid in file names. Additionally, individual `File` names have no
|
|
13
|
-
* correlation to names of files on disk, so multiple files with the same name may be stored in the
|
|
14
|
-
* same storage object.
|
|
15
|
-
*/
|
|
16
|
-
export class LocalFileStorage {
|
|
17
|
-
#dirname;
|
|
18
|
-
/**
|
|
19
|
-
* @param directory The directory where files are stored
|
|
20
|
-
*/
|
|
21
|
-
constructor(directory) {
|
|
22
|
-
this.#dirname = path.resolve(directory);
|
|
23
|
-
try {
|
|
24
|
-
let stats = fs.statSync(this.#dirname);
|
|
25
|
-
if (!stats.isDirectory()) {
|
|
26
|
-
throw new Error(`Path "${this.#dirname}" is not a directory`);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
catch (error) {
|
|
30
|
-
if (!isNoEntityError(error)) {
|
|
31
|
-
throw error;
|
|
32
|
-
}
|
|
33
|
-
fs.mkdirSync(this.#dirname, { recursive: true });
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
async get(key) {
|
|
37
|
-
let { filePath, metaPath } = await this.#getPaths(key);
|
|
38
|
-
try {
|
|
39
|
-
let meta = await readMetadata(metaPath);
|
|
40
|
-
return openFile(filePath, {
|
|
41
|
-
lastModified: meta.lastModified,
|
|
42
|
-
name: meta.name,
|
|
43
|
-
type: meta.type,
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
catch (error) {
|
|
47
|
-
if (!isNoEntityError(error)) {
|
|
48
|
-
throw error;
|
|
49
|
-
}
|
|
50
|
-
return null;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
async has(key) {
|
|
54
|
-
let { metaPath } = await this.#getPaths(key);
|
|
55
|
-
try {
|
|
56
|
-
await fsp.access(metaPath);
|
|
57
|
-
return true;
|
|
58
|
-
}
|
|
59
|
-
catch {
|
|
60
|
-
return false;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
async list(options) {
|
|
64
|
-
let { cursor, includeMetadata = false, limit = 32, prefix } = options ?? {};
|
|
65
|
-
let files = [];
|
|
66
|
-
let foundCursor = cursor === undefined;
|
|
67
|
-
let nextCursor;
|
|
68
|
-
let lastHash;
|
|
69
|
-
outerLoop: for await (let subdir of await fsp.opendir(this.#dirname)) {
|
|
70
|
-
if (!subdir.isDirectory())
|
|
71
|
-
continue;
|
|
72
|
-
for await (let file of await fsp.opendir(path.join(this.#dirname, subdir.name))) {
|
|
73
|
-
if (!file.isFile() || !file.name.endsWith('.meta.json'))
|
|
74
|
-
continue;
|
|
75
|
-
let hash = file.name.slice(0, -10); // Remove ".meta.json"
|
|
76
|
-
if (foundCursor) {
|
|
77
|
-
let meta = await readMetadata(path.join(this.#dirname, subdir.name, file.name));
|
|
78
|
-
if (prefix != null && !meta.key.startsWith(prefix)) {
|
|
79
|
-
continue;
|
|
80
|
-
}
|
|
81
|
-
if (files.length >= limit) {
|
|
82
|
-
nextCursor = lastHash;
|
|
83
|
-
break outerLoop;
|
|
84
|
-
}
|
|
85
|
-
if (includeMetadata) {
|
|
86
|
-
let size = (await fsp.stat(path.join(this.#dirname, subdir.name, `${hash}.dat`))).size;
|
|
87
|
-
files.push({ ...meta, size });
|
|
88
|
-
}
|
|
89
|
-
else {
|
|
90
|
-
files.push({ key: meta.key });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
else if (hash === cursor) {
|
|
94
|
-
foundCursor = true;
|
|
95
|
-
}
|
|
96
|
-
lastHash = hash;
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
return {
|
|
100
|
-
cursor: nextCursor,
|
|
101
|
-
files,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
async put(key, file) {
|
|
105
|
-
await this.set(key, file);
|
|
106
|
-
return (await this.get(key));
|
|
107
|
-
}
|
|
108
|
-
async remove(key) {
|
|
109
|
-
let { directory, filePath, metaPath } = await this.#getPaths(key);
|
|
110
|
-
try {
|
|
111
|
-
await Promise.all([fsp.unlink(filePath), fsp.unlink(metaPath)]);
|
|
112
|
-
// Check if directory is empty and remove it if so
|
|
113
|
-
let files = await fsp.readdir(directory);
|
|
114
|
-
if (files.length === 0) {
|
|
115
|
-
await fsp.rmdir(directory);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
catch (error) {
|
|
119
|
-
if (!isNoEntityError(error)) {
|
|
120
|
-
throw error;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
async set(key, file) {
|
|
125
|
-
let { directory, filePath, metaPath } = await this.#getPaths(key);
|
|
126
|
-
// Ensure directory exists
|
|
127
|
-
await fsp.mkdir(directory, { recursive: true });
|
|
128
|
-
await writeFile(filePath, file);
|
|
129
|
-
let meta = {
|
|
130
|
-
key,
|
|
131
|
-
lastModified: file.lastModified,
|
|
132
|
-
name: file.name,
|
|
133
|
-
type: file.type,
|
|
134
|
-
};
|
|
135
|
-
await fsp.writeFile(metaPath, JSON.stringify(meta));
|
|
136
|
-
}
|
|
137
|
-
async #getPaths(key) {
|
|
138
|
-
let hash = await computeHash(key);
|
|
139
|
-
let directory = path.join(this.#dirname, hash.slice(0, 2));
|
|
140
|
-
return {
|
|
141
|
-
directory,
|
|
142
|
-
filePath: path.join(directory, `${hash}.dat`),
|
|
143
|
-
metaPath: path.join(directory, `${hash}.meta.json`),
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
async function readMetadata(metaPath) {
|
|
148
|
-
return JSON.parse(await fsp.readFile(metaPath, 'utf-8'));
|
|
149
|
-
}
|
|
150
|
-
async function computeHash(key, algorithm = 'SHA-256') {
|
|
151
|
-
let digest = await crypto.subtle.digest(algorithm, new TextEncoder().encode(key));
|
|
152
|
-
return Array.from(new Uint8Array(digest))
|
|
153
|
-
.map((b) => b.toString(16).padStart(2, '0'))
|
|
154
|
-
.join('');
|
|
155
|
-
}
|
|
156
|
-
function isNoEntityError(obj) {
|
|
157
|
-
return obj instanceof Error && 'code' in obj && obj.code === 'ENOENT';
|
|
158
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { FileStorage, ListOptions, ListResult } from './file-storage.ts';
|
|
2
|
-
/**
|
|
3
|
-
* A simple, in-memory implementation of the `FileStorage` interface.
|
|
4
|
-
*/
|
|
5
|
-
export declare class MemoryFileStorage implements FileStorage {
|
|
6
|
-
#private;
|
|
7
|
-
get(key: string): File | null;
|
|
8
|
-
has(key: string): boolean;
|
|
9
|
-
list<T extends ListOptions>(options?: T): ListResult<T>;
|
|
10
|
-
put(key: string, file: File): Promise<File>;
|
|
11
|
-
remove(key: string): void;
|
|
12
|
-
set(key: string, file: File): Promise<void>;
|
|
13
|
-
}
|
|
14
|
-
//# sourceMappingURL=memory-file-storage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"memory-file-storage.d.ts","sourceRoot":"","sources":["../../src/lib/memory-file-storage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAE7E;;GAEG;AACH,qBAAa,iBAAkB,YAAW,WAAW;;IAGnD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAI7B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAIzB,IAAI,CAAC,CAAC,SAAS,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;IAwCjD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjD,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAInB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;CASlD"}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* A simple, in-memory implementation of the `FileStorage` interface.
|
|
3
|
-
*/
|
|
4
|
-
export class MemoryFileStorage {
|
|
5
|
-
#map = new Map();
|
|
6
|
-
get(key) {
|
|
7
|
-
return this.#map.get(key) ?? null;
|
|
8
|
-
}
|
|
9
|
-
has(key) {
|
|
10
|
-
return this.#map.has(key);
|
|
11
|
-
}
|
|
12
|
-
list(options) {
|
|
13
|
-
let { cursor, includeMetadata = false, limit = Infinity, prefix } = options ?? {};
|
|
14
|
-
let files = [];
|
|
15
|
-
let foundCursor = cursor === undefined;
|
|
16
|
-
let nextCursor;
|
|
17
|
-
for (let [key, file] of this.#map.entries()) {
|
|
18
|
-
if (foundCursor) {
|
|
19
|
-
if (prefix != null && !key.startsWith(prefix)) {
|
|
20
|
-
continue;
|
|
21
|
-
}
|
|
22
|
-
if (files.length >= limit) {
|
|
23
|
-
nextCursor = files[files.length - 1]?.key;
|
|
24
|
-
break;
|
|
25
|
-
}
|
|
26
|
-
if (includeMetadata) {
|
|
27
|
-
files.push({
|
|
28
|
-
key,
|
|
29
|
-
lastModified: file.lastModified,
|
|
30
|
-
name: file.name,
|
|
31
|
-
size: file.size,
|
|
32
|
-
type: file.type,
|
|
33
|
-
});
|
|
34
|
-
}
|
|
35
|
-
else {
|
|
36
|
-
files.push({ key });
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
else if (key === cursor) {
|
|
40
|
-
foundCursor = true;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
return {
|
|
44
|
-
cursor: nextCursor,
|
|
45
|
-
files,
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
async put(key, file) {
|
|
49
|
-
await this.set(key, file);
|
|
50
|
-
return this.get(key);
|
|
51
|
-
}
|
|
52
|
-
remove(key) {
|
|
53
|
-
this.#map.delete(key);
|
|
54
|
-
}
|
|
55
|
-
async set(key, file) {
|
|
56
|
-
let buffer = await file.arrayBuffer();
|
|
57
|
-
let newFile = new File([buffer], file.name, {
|
|
58
|
-
lastModified: file.lastModified,
|
|
59
|
-
type: file.type,
|
|
60
|
-
});
|
|
61
|
-
this.#map.set(key, newFile);
|
|
62
|
-
}
|
|
63
|
-
}
|
package/dist/local.d.ts
DELETED
package/dist/local.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../src/local.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA"}
|
package/dist/local.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { LocalFileStorage } from "./lib/local-file-storage.js";
|
|
@@ -1,189 +0,0 @@
|
|
|
1
|
-
import * as fs from 'node:fs'
|
|
2
|
-
import * as fsp from 'node:fs/promises'
|
|
3
|
-
import * as path from 'node:path'
|
|
4
|
-
import { openFile, writeFile } from '@remix-run/fs'
|
|
5
|
-
|
|
6
|
-
import type { FileStorage, FileMetadata, ListOptions, ListResult } from './file-storage.ts'
|
|
7
|
-
|
|
8
|
-
type MetadataJson = Omit<FileMetadata, 'size'>
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* A `FileStorage` that is backed by a directory on the local filesystem.
|
|
12
|
-
*
|
|
13
|
-
* Important: No attempt is made to avoid overwriting existing files, so the directory used should
|
|
14
|
-
* be a new directory solely dedicated to this storage object.
|
|
15
|
-
*
|
|
16
|
-
* Note: Keys have no correlation to file names on disk, so they may be any string including
|
|
17
|
-
* characters that are not valid in file names. Additionally, individual `File` names have no
|
|
18
|
-
* correlation to names of files on disk, so multiple files with the same name may be stored in the
|
|
19
|
-
* same storage object.
|
|
20
|
-
*/
|
|
21
|
-
export class LocalFileStorage implements FileStorage {
|
|
22
|
-
#dirname: string
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* @param directory The directory where files are stored
|
|
26
|
-
*/
|
|
27
|
-
constructor(directory: string) {
|
|
28
|
-
this.#dirname = path.resolve(directory)
|
|
29
|
-
|
|
30
|
-
try {
|
|
31
|
-
let stats = fs.statSync(this.#dirname)
|
|
32
|
-
|
|
33
|
-
if (!stats.isDirectory()) {
|
|
34
|
-
throw new Error(`Path "${this.#dirname}" is not a directory`)
|
|
35
|
-
}
|
|
36
|
-
} catch (error) {
|
|
37
|
-
if (!isNoEntityError(error)) {
|
|
38
|
-
throw error
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
fs.mkdirSync(this.#dirname, { recursive: true })
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
async get(key: string): Promise<File | null> {
|
|
46
|
-
let { filePath, metaPath } = await this.#getPaths(key)
|
|
47
|
-
|
|
48
|
-
try {
|
|
49
|
-
let meta = await readMetadata(metaPath)
|
|
50
|
-
|
|
51
|
-
return openFile(filePath, {
|
|
52
|
-
lastModified: meta.lastModified,
|
|
53
|
-
name: meta.name,
|
|
54
|
-
type: meta.type,
|
|
55
|
-
})
|
|
56
|
-
} catch (error) {
|
|
57
|
-
if (!isNoEntityError(error)) {
|
|
58
|
-
throw error
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
return null
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
async has(key: string): Promise<boolean> {
|
|
66
|
-
let { metaPath } = await this.#getPaths(key)
|
|
67
|
-
|
|
68
|
-
try {
|
|
69
|
-
await fsp.access(metaPath)
|
|
70
|
-
return true
|
|
71
|
-
} catch {
|
|
72
|
-
return false
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
async list<T extends ListOptions>(options?: T): Promise<ListResult<T>> {
|
|
77
|
-
let { cursor, includeMetadata = false, limit = 32, prefix } = options ?? {}
|
|
78
|
-
|
|
79
|
-
let files: any[] = []
|
|
80
|
-
let foundCursor = cursor === undefined
|
|
81
|
-
let nextCursor: string | undefined
|
|
82
|
-
let lastHash: string | undefined
|
|
83
|
-
|
|
84
|
-
outerLoop: for await (let subdir of await fsp.opendir(this.#dirname)) {
|
|
85
|
-
if (!subdir.isDirectory()) continue
|
|
86
|
-
|
|
87
|
-
for await (let file of await fsp.opendir(path.join(this.#dirname, subdir.name))) {
|
|
88
|
-
if (!file.isFile() || !file.name.endsWith('.meta.json')) continue
|
|
89
|
-
|
|
90
|
-
let hash = file.name.slice(0, -10) // Remove ".meta.json"
|
|
91
|
-
|
|
92
|
-
if (foundCursor) {
|
|
93
|
-
let meta = await readMetadata(path.join(this.#dirname, subdir.name, file.name))
|
|
94
|
-
|
|
95
|
-
if (prefix != null && !meta.key.startsWith(prefix)) {
|
|
96
|
-
continue
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
if (files.length >= limit) {
|
|
100
|
-
nextCursor = lastHash
|
|
101
|
-
break outerLoop
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
if (includeMetadata) {
|
|
105
|
-
let size = (await fsp.stat(path.join(this.#dirname, subdir.name, `${hash}.dat`))).size
|
|
106
|
-
files.push({ ...meta, size })
|
|
107
|
-
} else {
|
|
108
|
-
files.push({ key: meta.key })
|
|
109
|
-
}
|
|
110
|
-
} else if (hash === cursor) {
|
|
111
|
-
foundCursor = true
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
lastHash = hash
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
return {
|
|
119
|
-
cursor: nextCursor,
|
|
120
|
-
files,
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async put(key: string, file: File): Promise<File> {
|
|
125
|
-
await this.set(key, file)
|
|
126
|
-
return (await this.get(key))!
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
async remove(key: string): Promise<void> {
|
|
130
|
-
let { directory, filePath, metaPath } = await this.#getPaths(key)
|
|
131
|
-
|
|
132
|
-
try {
|
|
133
|
-
await Promise.all([fsp.unlink(filePath), fsp.unlink(metaPath)])
|
|
134
|
-
|
|
135
|
-
// Check if directory is empty and remove it if so
|
|
136
|
-
let files = await fsp.readdir(directory)
|
|
137
|
-
if (files.length === 0) {
|
|
138
|
-
await fsp.rmdir(directory)
|
|
139
|
-
}
|
|
140
|
-
} catch (error) {
|
|
141
|
-
if (!isNoEntityError(error)) {
|
|
142
|
-
throw error
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async set(key: string, file: File): Promise<void> {
|
|
148
|
-
let { directory, filePath, metaPath } = await this.#getPaths(key)
|
|
149
|
-
|
|
150
|
-
// Ensure directory exists
|
|
151
|
-
await fsp.mkdir(directory, { recursive: true })
|
|
152
|
-
|
|
153
|
-
await writeFile(filePath, file)
|
|
154
|
-
|
|
155
|
-
let meta: MetadataJson = {
|
|
156
|
-
key,
|
|
157
|
-
lastModified: file.lastModified,
|
|
158
|
-
name: file.name,
|
|
159
|
-
type: file.type,
|
|
160
|
-
}
|
|
161
|
-
await fsp.writeFile(metaPath, JSON.stringify(meta))
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
async #getPaths(key: string): Promise<{ directory: string; filePath: string; metaPath: string }> {
|
|
165
|
-
let hash = await computeHash(key)
|
|
166
|
-
let directory = path.join(this.#dirname, hash.slice(0, 2))
|
|
167
|
-
|
|
168
|
-
return {
|
|
169
|
-
directory,
|
|
170
|
-
filePath: path.join(directory, `${hash}.dat`),
|
|
171
|
-
metaPath: path.join(directory, `${hash}.meta.json`),
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
async function readMetadata(metaPath: string): Promise<MetadataJson> {
|
|
177
|
-
return JSON.parse(await fsp.readFile(metaPath, 'utf-8'))
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
async function computeHash(key: string, algorithm = 'SHA-256'): Promise<string> {
|
|
181
|
-
let digest = await crypto.subtle.digest(algorithm, new TextEncoder().encode(key))
|
|
182
|
-
return Array.from(new Uint8Array(digest))
|
|
183
|
-
.map((b) => b.toString(16).padStart(2, '0'))
|
|
184
|
-
.join('')
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function isNoEntityError(obj: unknown): obj is NodeJS.ErrnoException & { code: 'ENOENT' } {
|
|
188
|
-
return obj instanceof Error && 'code' in obj && (obj as NodeJS.ErrnoException).code === 'ENOENT'
|
|
189
|
-
}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import type { FileStorage, ListOptions, ListResult } from './file-storage.ts'
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* A simple, in-memory implementation of the `FileStorage` interface.
|
|
5
|
-
*/
|
|
6
|
-
export class MemoryFileStorage implements FileStorage {
|
|
7
|
-
#map = new Map<string, File>()
|
|
8
|
-
|
|
9
|
-
get(key: string): File | null {
|
|
10
|
-
return this.#map.get(key) ?? null
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
has(key: string): boolean {
|
|
14
|
-
return this.#map.has(key)
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
list<T extends ListOptions>(options?: T): ListResult<T> {
|
|
18
|
-
let { cursor, includeMetadata = false, limit = Infinity, prefix } = options ?? {}
|
|
19
|
-
|
|
20
|
-
let files: any[] = []
|
|
21
|
-
let foundCursor = cursor === undefined
|
|
22
|
-
let nextCursor: string | undefined
|
|
23
|
-
|
|
24
|
-
for (let [key, file] of this.#map.entries()) {
|
|
25
|
-
if (foundCursor) {
|
|
26
|
-
if (prefix != null && !key.startsWith(prefix)) {
|
|
27
|
-
continue
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
if (files.length >= limit) {
|
|
31
|
-
nextCursor = files[files.length - 1]?.key
|
|
32
|
-
break
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
if (includeMetadata) {
|
|
36
|
-
files.push({
|
|
37
|
-
key,
|
|
38
|
-
lastModified: file.lastModified,
|
|
39
|
-
name: file.name,
|
|
40
|
-
size: file.size,
|
|
41
|
-
type: file.type,
|
|
42
|
-
})
|
|
43
|
-
} else {
|
|
44
|
-
files.push({ key })
|
|
45
|
-
}
|
|
46
|
-
} else if (key === cursor) {
|
|
47
|
-
foundCursor = true
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
return {
|
|
52
|
-
cursor: nextCursor,
|
|
53
|
-
files,
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
async put(key: string, file: File): Promise<File> {
|
|
58
|
-
await this.set(key, file)
|
|
59
|
-
return this.get(key)!
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
remove(key: string): void {
|
|
63
|
-
this.#map.delete(key)
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
async set(key: string, file: File): Promise<void> {
|
|
67
|
-
let buffer = await file.arrayBuffer()
|
|
68
|
-
let newFile = new File([buffer], file.name, {
|
|
69
|
-
lastModified: file.lastModified,
|
|
70
|
-
type: file.type,
|
|
71
|
-
})
|
|
72
|
-
|
|
73
|
-
this.#map.set(key, newFile)
|
|
74
|
-
}
|
|
75
|
-
}
|
package/src/local.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { LocalFileStorage } from './lib/local-file-storage.ts'
|