@thi.ng/block-fs 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -1
- package/README.md +9 -2
- package/api.d.ts +17 -0
- package/directory.js +15 -10
- package/entry.js +1 -1
- package/fs.d.ts +96 -4
- package/fs.js +171 -76
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
-
- **Last updated**: 2025-04-
|
|
3
|
+
- **Last updated**: 2025-04-02T10:24:13Z
|
|
4
4
|
- **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
|
|
5
5
|
|
|
6
6
|
All notable changes to this project will be documented in this file.
|
|
@@ -11,6 +11,18 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
|
|
|
11
11
|
**Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
|
|
12
12
|
and/or version bumps of transitive dependencies.
|
|
13
13
|
|
|
14
|
+
## [0.2.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.2.0) (2025-04-02)
|
|
15
|
+
|
|
16
|
+
#### 🚀 Features
|
|
17
|
+
|
|
18
|
+
- add path separator option, various refactoring ([875465e](https://github.com/thi-ng/umbrella/commit/875465e))
|
|
19
|
+
- add `BlockFSOpts.separator`
|
|
20
|
+
- rename `.readFileRaw()` => `.readBlocks()`
|
|
21
|
+
- rename `.writeFileRaw()` => `.writeBlocks()`
|
|
22
|
+
- add additional internal safety checks
|
|
23
|
+
- internal refactoring (`this` destructuring)
|
|
24
|
+
- add docs
|
|
25
|
+
|
|
14
26
|
## [0.1.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.1.0) (2025-04-01)
|
|
15
27
|
|
|
16
28
|
#### 🚀 Features
|
package/README.md
CHANGED
|
@@ -53,6 +53,7 @@ The package also provides an hierarchical filesystem layer with pluggable
|
|
|
53
53
|
storage providers and other customizable aspects. The default implementation
|
|
54
54
|
supports:
|
|
55
55
|
|
|
56
|
+
- 8 - 32bit block IDs
|
|
56
57
|
- arbitrarily nested directories
|
|
57
58
|
- filenames of max. 31 bytes (UTF-8) per directory level
|
|
58
59
|
- max. 32 owner IDs
|
|
@@ -73,6 +74,9 @@ The filesystem stores a [bitfield](https://thi.ng/bitfield) of block allocations
|
|
|
73
74
|
in the first N blocks. The number of blocks used depends on configured block
|
|
74
75
|
size and the max. number of blocks in the storage backend.
|
|
75
76
|
|
|
77
|
+
Blocks can be reserved for custom purposes by calling
|
|
78
|
+
[`.allocateBlocks()`](https://docs.thi.ng/umbrella/block-fs/classes/BlockFS.html#allocateblocks).
|
|
79
|
+
|
|
76
80
|
#### Root directory
|
|
77
81
|
|
|
78
82
|
The root directory starts in block N, directly after the block allocation table.
|
|
@@ -130,7 +134,7 @@ For Node.js REPL:
|
|
|
130
134
|
const bf = await import("@thi.ng/block-fs");
|
|
131
135
|
```
|
|
132
136
|
|
|
133
|
-
Package sizes (brotli'd, pre-treeshake): ESM: 4.
|
|
137
|
+
Package sizes (brotli'd, pre-treeshake): ESM: 4.32 KB
|
|
134
138
|
|
|
135
139
|
## Dependencies
|
|
136
140
|
|
|
@@ -198,7 +202,10 @@ console.log(await fs.readFile("/deeply/nested/paths/are-ok"));
|
|
|
198
202
|
|
|
199
203
|
// iterate all files & directory entries in root dir
|
|
200
204
|
for await (let entry of fs.root.tree()) {
|
|
201
|
-
|
|
205
|
+
// entry.path is absolute path
|
|
206
|
+
// entry.size is always a bigint
|
|
207
|
+
// entry.ctime/mtime is UNIX epoch
|
|
208
|
+
console.log(entry.path, entry.size, new Date(entry.ctime));
|
|
202
209
|
}
|
|
203
210
|
|
|
204
211
|
// /hello 0n 2025-04-01T20:18:55.916Z
|
package/api.d.ts
CHANGED
|
@@ -18,18 +18,35 @@ export interface IBlockStorage {
|
|
|
18
18
|
}
|
|
19
19
|
export interface BlockStorageOpts {
|
|
20
20
|
logger?: ILogger;
|
|
21
|
+
/**
|
|
22
|
+
* Max. number of blocks. Will be rounded up to a multiple of 8.
|
|
23
|
+
*/
|
|
21
24
|
numBlocks: number;
|
|
25
|
+
/**
|
|
26
|
+
* Block size in bytes. Must be a power of 2. For usage with
|
|
27
|
+
* {@link BlockFS}, requires a minimum size of 128 bytes (though in practice
|
|
28
|
+
* should be larger to minimize overhead per block).
|
|
29
|
+
*/
|
|
22
30
|
blockSize: Pow2;
|
|
23
31
|
}
|
|
24
32
|
export interface EntrySpec {
|
|
33
|
+
/** Entry type (file or directory) */
|
|
25
34
|
type: EntryType;
|
|
35
|
+
/** TODO currently still unused */
|
|
26
36
|
locked?: boolean;
|
|
37
|
+
/** Owner ID */
|
|
27
38
|
owner?: number;
|
|
39
|
+
/** Entry name */
|
|
28
40
|
name: string;
|
|
41
|
+
/** Only used for files. File size as bigint */
|
|
29
42
|
size?: bigint;
|
|
43
|
+
/** Entry creation timestamp */
|
|
30
44
|
ctime?: number;
|
|
45
|
+
/** Entry modification timestamp */
|
|
31
46
|
mtime?: number;
|
|
47
|
+
/** Data start block ID */
|
|
32
48
|
start: number;
|
|
49
|
+
/** Data end block ID */
|
|
33
50
|
end?: number;
|
|
34
51
|
}
|
|
35
52
|
export declare enum EntryType {
|
package/directory.js
CHANGED
|
@@ -30,18 +30,22 @@ class Directory {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
async traverse() {
|
|
33
|
+
const {
|
|
34
|
+
fs,
|
|
35
|
+
fs: { sentinelID, storage }
|
|
36
|
+
} = this;
|
|
33
37
|
const blocks = [];
|
|
34
38
|
const entries = [];
|
|
35
39
|
let blockID = this.entry.start;
|
|
36
40
|
while (true) {
|
|
37
41
|
blocks.push(blockID);
|
|
38
|
-
const block = await
|
|
39
|
-
let { next, size } =
|
|
40
|
-
if (!next) next =
|
|
42
|
+
const block = await storage.loadBlock(blockID);
|
|
43
|
+
let { next, size } = fs.getBlockMeta(block);
|
|
44
|
+
if (!next) next = sentinelID;
|
|
41
45
|
for (let i = 0; i < size; i++) {
|
|
42
46
|
entries.push(this.defEntry(i, blockID, block));
|
|
43
47
|
}
|
|
44
|
-
if (next ===
|
|
48
|
+
if (next === sentinelID) break;
|
|
45
49
|
blockID = next;
|
|
46
50
|
}
|
|
47
51
|
return { blocks, entries };
|
|
@@ -52,16 +56,17 @@ class Directory {
|
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
58
|
async mkdir(name) {
|
|
59
|
+
const fs = this.fs;
|
|
55
60
|
const length = utf8Length(name);
|
|
56
|
-
if (!length || length >
|
|
61
|
+
if (!length || length > fs.opts.entry.maxLength)
|
|
57
62
|
illegalArgs(`invalid name: '${name}'`);
|
|
58
63
|
const traversed = await this.traverse();
|
|
59
64
|
this.ensureUniqueName(name, traversed.entries);
|
|
60
|
-
const block = (await
|
|
61
|
-
const data = await
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
await
|
|
65
|
+
const block = (await fs.allocateBlocks(1))[0];
|
|
66
|
+
const data = await fs.storage.loadBlock(block);
|
|
67
|
+
fs.setBlockMeta(data, fs.sentinelID, 0);
|
|
68
|
+
fs.setBlockLink(data, this.entry.start, fs.dataStartBlockID);
|
|
69
|
+
await fs.storage.saveBlock(block, data);
|
|
65
70
|
return this.addEntry(
|
|
66
71
|
{
|
|
67
72
|
name,
|
package/entry.js
CHANGED
package/fs.d.ts
CHANGED
|
@@ -3,6 +3,15 @@ import type { ILogger } from "@thi.ng/logger";
|
|
|
3
3
|
import { EntryType, type IBlockStorage, type IDirectory, type IEntry } from "./api.js";
|
|
4
4
|
import { Lock } from "./lock.js";
|
|
5
5
|
export interface BlockFSOpts {
|
|
6
|
+
/**
|
|
7
|
+
* Path separator.
|
|
8
|
+
*
|
|
9
|
+
* @defaultValue `/`
|
|
10
|
+
*/
|
|
11
|
+
separator: string;
|
|
12
|
+
/**
|
|
13
|
+
* Logger instance to use.
|
|
14
|
+
*/
|
|
6
15
|
logger: ILogger;
|
|
7
16
|
/**
|
|
8
17
|
* Customizable {@link IDirectory} factory function. By default creates
|
|
@@ -66,16 +75,99 @@ export declare class BlockFS {
|
|
|
66
75
|
root: IDirectory;
|
|
67
76
|
constructor(storage: IBlockStorage, opts?: Partial<BlockFSOpts>);
|
|
68
77
|
init(): Promise<this>;
|
|
78
|
+
/**
|
|
79
|
+
* Attempts to find the {@link IEntry} for given `path` and returns it if
|
|
80
|
+
* successful. Throws error if no such path exists.
|
|
81
|
+
*
|
|
82
|
+
* @remarks
|
|
83
|
+
* Also see {@link BlockFS.ensureEntryForPath}.
|
|
84
|
+
*
|
|
85
|
+
* @param path
|
|
86
|
+
*/
|
|
69
87
|
entryForPath(path: string): Promise<IEntry>;
|
|
88
|
+
/**
|
|
89
|
+
* Attempts to find or to create the {@link IEntry} for given `path` and
|
|
90
|
+
* entry type (file or directory) and then returns it if successful. Throws
|
|
91
|
+
* error if the path exists, but does not match the expected type or if the
|
|
92
|
+
* entry could not be created for other reasons.
|
|
93
|
+
*
|
|
94
|
+
* @remarks
|
|
95
|
+
* Also see {@link BlockFS.entryForPath}.
|
|
96
|
+
*
|
|
97
|
+
* @param path
|
|
98
|
+
* @param type
|
|
99
|
+
*/
|
|
70
100
|
ensureEntryForPath(path: string, type: EntryType): Promise<IEntry>;
|
|
101
|
+
/**
|
|
102
|
+
* Attempts to allocate a number of free blocks required for storing the
|
|
103
|
+
* given number of `bytes`. If successful, marks the blocks as used in the
|
|
104
|
+
* allocation table and then returns list of their IDs, otherwise throws an
|
|
105
|
+
* error if there're insufficient blocks available.
|
|
106
|
+
*
|
|
107
|
+
* @param bytes
|
|
108
|
+
*/
|
|
71
109
|
allocateBlocks(bytes: number): Promise<number[]>;
|
|
110
|
+
/**
|
|
111
|
+
* Marks the given block IDs as free/unused in the block allocation table
|
|
112
|
+
* and deletes/clears the blocks.
|
|
113
|
+
*
|
|
114
|
+
* @param ids
|
|
115
|
+
*/
|
|
72
116
|
freeBlocks(ids: number[]): Promise<void>;
|
|
117
|
+
/**
|
|
118
|
+
* Same as POSIX `mkdirp`. Attempts to create given directory, including any
|
|
119
|
+
* missing intermediate ones defined by `path`.
|
|
120
|
+
*
|
|
121
|
+
* @param path
|
|
122
|
+
*/
|
|
73
123
|
mkdir(path: string): Promise<IEntry>;
|
|
74
|
-
|
|
124
|
+
/**
|
|
125
|
+
* Async iterator. Reads block list for given `path` (file path or start
|
|
126
|
+
* block ID) and yields data contents (byte arrays) of each block.
|
|
127
|
+
*
|
|
128
|
+
* @remarks
|
|
129
|
+
* Also see other read methods:
|
|
130
|
+
*
|
|
131
|
+
* - {@link BlockFS.readFile}
|
|
132
|
+
* - {@link BlockFS.readText}
|
|
133
|
+
* - {@link BlockFS.readJSON}
|
|
134
|
+
*
|
|
135
|
+
* @param path
|
|
136
|
+
*/
|
|
137
|
+
readBlocks(path: string | number): AsyncGenerator<Uint8Array<ArrayBufferLike>, void, unknown>;
|
|
138
|
+
/**
|
|
139
|
+
* Fully reads given file into a single byte buffer and returns it.
|
|
140
|
+
*
|
|
141
|
+
* @param path
|
|
142
|
+
*/
|
|
75
143
|
readFile(path: string | number): Promise<Uint8Array<ArrayBuffer>>;
|
|
144
|
+
/**
|
|
145
|
+
* Fully reads given file into a single UTF-8 byte buffer, then decodes and
|
|
146
|
+
* returns it as string.
|
|
147
|
+
*
|
|
148
|
+
* @param path
|
|
149
|
+
*/
|
|
76
150
|
readText(path: string | number): Promise<string>;
|
|
77
|
-
|
|
78
|
-
|
|
151
|
+
/**
|
|
152
|
+
* Fully reads given file into a single UTF-8 byte buffer, then decodes it
|
|
153
|
+
* as JSON and returns result.
|
|
154
|
+
*
|
|
155
|
+
* @param path
|
|
156
|
+
*/
|
|
157
|
+
readJSON<T>(path: string | number): Promise<T>;
|
|
158
|
+
/**
|
|
159
|
+
* Takes an array of block IDs (or `null`) and a `data` byte array. Writes
|
|
160
|
+
* chunks of data into given blocks and connecting each block as linked
|
|
161
|
+
* list. Returns object of start/end block IDs and data size.
|
|
162
|
+
*
|
|
163
|
+
* @remarks
|
|
164
|
+
* If `blocks` is null, blocks are automatically allocated via
|
|
165
|
+
* {@link BlockFS.allocateBlocks}.
|
|
166
|
+
*
|
|
167
|
+
* @param blocks
|
|
168
|
+
* @param data
|
|
169
|
+
*/
|
|
170
|
+
writeBlocks(blocks: number[] | null, data: Uint8Array): Promise<{
|
|
79
171
|
start: number;
|
|
80
172
|
end: number;
|
|
81
173
|
size: number;
|
|
@@ -85,7 +177,7 @@ export declare class BlockFS {
|
|
|
85
177
|
end: number;
|
|
86
178
|
size: number;
|
|
87
179
|
}>;
|
|
88
|
-
|
|
180
|
+
appendBlocks(blockID: number, data: Uint8Array): Promise<{
|
|
89
181
|
start: number;
|
|
90
182
|
end: number;
|
|
91
183
|
size: number;
|
package/fs.js
CHANGED
|
@@ -3,6 +3,7 @@ import { defBitField } from "@thi.ng/bitfield/bitfield";
|
|
|
3
3
|
import { isString } from "@thi.ng/checks/is-string";
|
|
4
4
|
import { assert } from "@thi.ng/errors/assert";
|
|
5
5
|
import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
|
|
6
|
+
import { illegalState } from "@thi.ng/errors/illegal-state";
|
|
6
7
|
import { NULL_LOGGER } from "@thi.ng/logger/null";
|
|
7
8
|
import {
|
|
8
9
|
EntryType
|
|
@@ -22,9 +23,10 @@ class BlockFS {
|
|
|
22
23
|
constructor(storage, opts) {
|
|
23
24
|
this.storage = storage;
|
|
24
25
|
this.opts = {
|
|
25
|
-
logger: NULL_LOGGER,
|
|
26
26
|
directory: (fs, entry) => new Directory(fs, entry),
|
|
27
27
|
entry: DEFAULT_ENTRY,
|
|
28
|
+
logger: NULL_LOGGER,
|
|
29
|
+
separator: "/",
|
|
28
30
|
...opts
|
|
29
31
|
};
|
|
30
32
|
assert(
|
|
@@ -67,60 +69,84 @@ class BlockFS {
|
|
|
67
69
|
/** Root directory */
|
|
68
70
|
root;
|
|
69
71
|
async init() {
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
const { blockIndex, storage, opts, rootDirBlockID } = this;
|
|
73
|
+
const indexSize = blockIndex.data.length;
|
|
74
|
+
const blockSize = storage.blockSize;
|
|
75
|
+
for (let i = 0; i < rootDirBlockID; i++) {
|
|
76
|
+
const data = await storage.loadBlock(i);
|
|
74
77
|
const offset = i * blockSize;
|
|
75
78
|
if (offset + data.length > indexSize) {
|
|
76
|
-
|
|
79
|
+
blockIndex.data.set(
|
|
77
80
|
data.subarray(0, indexSize - offset),
|
|
78
81
|
offset
|
|
79
82
|
);
|
|
80
83
|
} else {
|
|
81
|
-
|
|
84
|
+
blockIndex.data.set(data, offset);
|
|
82
85
|
}
|
|
83
86
|
}
|
|
84
|
-
|
|
85
|
-
const rootEntry =
|
|
87
|
+
blockIndex.fill(1, 0, this.dataStartBlockID);
|
|
88
|
+
const rootEntry = opts.entry.factory(
|
|
86
89
|
this,
|
|
87
90
|
null,
|
|
88
|
-
|
|
89
|
-
new Uint8Array(
|
|
91
|
+
rootDirBlockID,
|
|
92
|
+
new Uint8Array(opts.entry.size),
|
|
90
93
|
0
|
|
91
94
|
);
|
|
92
95
|
rootEntry.set({
|
|
93
96
|
name: "",
|
|
94
97
|
type: EntryType.DIR,
|
|
95
|
-
start:
|
|
98
|
+
start: rootDirBlockID,
|
|
96
99
|
owner: 0
|
|
97
100
|
});
|
|
98
|
-
this.root =
|
|
101
|
+
this.root = opts.directory(this, rootEntry);
|
|
99
102
|
return this;
|
|
100
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* Attempts to find the {@link IEntry} for given `path` and returns it if
|
|
106
|
+
* successful. Throws error if no such path exists.
|
|
107
|
+
*
|
|
108
|
+
* @remarks
|
|
109
|
+
* Also see {@link BlockFS.ensureEntryForPath}.
|
|
110
|
+
*
|
|
111
|
+
* @param path
|
|
112
|
+
*/
|
|
101
113
|
async entryForPath(path) {
|
|
102
114
|
let dir = this.root;
|
|
103
|
-
|
|
115
|
+
const { directory, separator } = this.opts;
|
|
116
|
+
if (path[0] === separator) path = path.substring(1);
|
|
104
117
|
if (path === "") return dir.entry;
|
|
105
|
-
const $path = path.split(
|
|
106
|
-
for (let i = 0
|
|
118
|
+
const $path = path.split(separator);
|
|
119
|
+
for (let i = 0, len = $path.length - 1; i <= len; i++) {
|
|
107
120
|
let entry = await dir.findName($path[i]);
|
|
108
121
|
if (!entry) break;
|
|
109
|
-
if (i ===
|
|
122
|
+
if (i === len) return entry;
|
|
110
123
|
if (!entry.isDirectory()) illegalArgs(path);
|
|
111
|
-
dir =
|
|
124
|
+
dir = directory(this, entry);
|
|
112
125
|
}
|
|
113
126
|
illegalArgs(path);
|
|
114
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* Attempts to find or to create the {@link IEntry} for given `path` and
|
|
130
|
+
* entry type (file or directory) and then returns it if successful. Throws
|
|
131
|
+
* error if the path exists, but does not match the expected type or if the
|
|
132
|
+
* entry could not be created for other reasons.
|
|
133
|
+
*
|
|
134
|
+
* @remarks
|
|
135
|
+
* Also see {@link BlockFS.entryForPath}.
|
|
136
|
+
*
|
|
137
|
+
* @param path
|
|
138
|
+
* @param type
|
|
139
|
+
*/
|
|
115
140
|
async ensureEntryForPath(path, type) {
|
|
116
141
|
let dir = this.root;
|
|
117
|
-
|
|
142
|
+
const { directory, separator } = this.opts;
|
|
143
|
+
if (path[0] === separator) path = path.substring(1);
|
|
118
144
|
if (path === "") return dir.entry;
|
|
119
|
-
const $path = path.split(
|
|
120
|
-
for (let i = 0
|
|
145
|
+
const $path = path.split(separator);
|
|
146
|
+
for (let i = 0, len = $path.length - 1; i <= len; i++) {
|
|
121
147
|
let entry = await dir.findName($path[i]);
|
|
122
148
|
if (!entry) {
|
|
123
|
-
if (i <
|
|
149
|
+
if (i < len) {
|
|
124
150
|
entry = await dir.mkdir($path[i]);
|
|
125
151
|
} else {
|
|
126
152
|
return await dir.addEntry({
|
|
@@ -130,7 +156,7 @@ class BlockFS {
|
|
|
130
156
|
});
|
|
131
157
|
}
|
|
132
158
|
}
|
|
133
|
-
if (i ===
|
|
159
|
+
if (i === len) {
|
|
134
160
|
if (entry.type !== type)
|
|
135
161
|
illegalArgs(
|
|
136
162
|
`path exists, but is not a ${EntryType[type]}: ${path}`
|
|
@@ -138,32 +164,53 @@ class BlockFS {
|
|
|
138
164
|
return entry;
|
|
139
165
|
}
|
|
140
166
|
if (!entry.isDirectory()) illegalArgs(path);
|
|
141
|
-
dir =
|
|
167
|
+
dir = directory(this, entry);
|
|
142
168
|
}
|
|
143
169
|
illegalArgs(path);
|
|
144
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* Attempts to allocate a number of free blocks required for storing the
|
|
173
|
+
* given number of `bytes`. If successful, marks the blocks as used in the
|
|
174
|
+
* allocation table and then returns list of their IDs, otherwise throws an
|
|
175
|
+
* error if there're insufficient blocks available.
|
|
176
|
+
*
|
|
177
|
+
* @param bytes
|
|
178
|
+
*/
|
|
145
179
|
async allocateBlocks(bytes) {
|
|
146
|
-
const
|
|
180
|
+
const {
|
|
181
|
+
blockDataSize,
|
|
182
|
+
blockIndex,
|
|
183
|
+
dataStartBlockID,
|
|
184
|
+
lock,
|
|
185
|
+
sentinelID
|
|
186
|
+
} = this;
|
|
187
|
+
const lockID = await lock.acquire();
|
|
147
188
|
try {
|
|
148
189
|
const ids = [];
|
|
149
|
-
let last =
|
|
190
|
+
let last = dataStartBlockID;
|
|
150
191
|
while (bytes > 0) {
|
|
151
|
-
const next =
|
|
152
|
-
if (next < 0 || next >=
|
|
192
|
+
const next = blockIndex.firstZero(last);
|
|
193
|
+
if (next < 0 || next >= sentinelID) {
|
|
153
194
|
throw new Error(
|
|
154
195
|
`insufficient free blocks for storing ${bytes} bytes`
|
|
155
196
|
);
|
|
156
197
|
}
|
|
157
198
|
ids.push(next);
|
|
158
199
|
last = next + 1;
|
|
159
|
-
bytes -=
|
|
200
|
+
bytes -= blockDataSize;
|
|
160
201
|
}
|
|
161
202
|
await this.updateBlockIndex(ids, 1);
|
|
162
203
|
return ids;
|
|
163
204
|
} finally {
|
|
164
|
-
await
|
|
205
|
+
await lock.release(lockID);
|
|
165
206
|
}
|
|
166
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Marks the given block IDs as free/unused in the block allocation table
|
|
210
|
+
* and deletes/clears the blocks.
|
|
211
|
+
*
|
|
212
|
+
* @param ids
|
|
213
|
+
*/
|
|
167
214
|
async freeBlocks(ids) {
|
|
168
215
|
const lockID = await this.lock.acquire();
|
|
169
216
|
try {
|
|
@@ -173,54 +220,99 @@ class BlockFS {
|
|
|
173
220
|
await this.lock.release(lockID);
|
|
174
221
|
}
|
|
175
222
|
}
|
|
223
|
+
/**
|
|
224
|
+
* Same as POSIX `mkdirp`. Attempts to create given directory, including any
|
|
225
|
+
* missing intermediate ones defined by `path`.
|
|
226
|
+
*
|
|
227
|
+
* @param path
|
|
228
|
+
*/
|
|
176
229
|
async mkdir(path) {
|
|
177
230
|
return this.ensureEntryForPath(path, EntryType.DIR);
|
|
178
231
|
}
|
|
179
|
-
|
|
232
|
+
/**
|
|
233
|
+
* Async iterator. Reads block list for given `path` (file path or start
|
|
234
|
+
* block ID) and yields data contents (byte arrays) of each block.
|
|
235
|
+
*
|
|
236
|
+
* @remarks
|
|
237
|
+
* Also see other read methods:
|
|
238
|
+
*
|
|
239
|
+
* - {@link BlockFS.readFile}
|
|
240
|
+
* - {@link BlockFS.readText}
|
|
241
|
+
* - {@link BlockFS.readJSON}
|
|
242
|
+
*
|
|
243
|
+
* @param path
|
|
244
|
+
*/
|
|
245
|
+
async *readBlocks(path) {
|
|
180
246
|
let blockID = isString(path) ? (await this.entryForPath(path)).start : path;
|
|
247
|
+
const { blockDataOffset, blockIndex, sentinelID, storage } = this;
|
|
181
248
|
while (true) {
|
|
182
|
-
if (!
|
|
249
|
+
if (!blockIndex.at(blockID)) {
|
|
183
250
|
throw new Error(`invalid block ref: ${blockID}`);
|
|
184
251
|
}
|
|
185
|
-
const bytes = await
|
|
252
|
+
const bytes = await storage.loadBlock(blockID);
|
|
186
253
|
const { next, size } = this.getBlockMeta(bytes);
|
|
187
|
-
yield bytes.subarray(
|
|
188
|
-
|
|
189
|
-
this.blockDataOffset + size
|
|
190
|
-
);
|
|
191
|
-
if (next === this.sentinelID) return;
|
|
254
|
+
yield bytes.subarray(blockDataOffset, blockDataOffset + size);
|
|
255
|
+
if (next === sentinelID) return;
|
|
192
256
|
blockID = next;
|
|
193
257
|
}
|
|
194
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* Fully reads given file into a single byte buffer and returns it.
|
|
261
|
+
*
|
|
262
|
+
* @param path
|
|
263
|
+
*/
|
|
195
264
|
async readFile(path) {
|
|
196
265
|
const buffer = [];
|
|
197
|
-
for await (let block of this.
|
|
266
|
+
for await (let block of this.readBlocks(path)) buffer.push(...block);
|
|
198
267
|
return new Uint8Array(buffer);
|
|
199
268
|
}
|
|
269
|
+
/**
|
|
270
|
+
* Fully reads given file into a single UTF-8 byte buffer, then decodes and
|
|
271
|
+
* returns it as string.
|
|
272
|
+
*
|
|
273
|
+
* @param path
|
|
274
|
+
*/
|
|
200
275
|
async readText(path) {
|
|
201
276
|
return new TextDecoder().decode(await this.readFile(path));
|
|
202
277
|
}
|
|
278
|
+
/**
|
|
279
|
+
* Fully reads given file into a single UTF-8 byte buffer, then decodes it
|
|
280
|
+
* as JSON and returns result.
|
|
281
|
+
*
|
|
282
|
+
* @param path
|
|
283
|
+
*/
|
|
203
284
|
async readJSON(path) {
|
|
204
285
|
return JSON.parse(await this.readText(path));
|
|
205
286
|
}
|
|
206
|
-
|
|
287
|
+
/**
|
|
288
|
+
* Takes an array of block IDs (or `null`) and a `data` byte array. Writes
|
|
289
|
+
* chunks of data into given blocks and connecting each block as linked
|
|
290
|
+
* list. Returns object of start/end block IDs and data size.
|
|
291
|
+
*
|
|
292
|
+
* @remarks
|
|
293
|
+
* If `blocks` is null, blocks are automatically allocated via
|
|
294
|
+
* {@link BlockFS.allocateBlocks}.
|
|
295
|
+
*
|
|
296
|
+
* @param blocks
|
|
297
|
+
* @param data
|
|
298
|
+
*/
|
|
299
|
+
async writeBlocks(blocks, data) {
|
|
300
|
+
const { blockDataOffset, blockDataSize, sentinelID, storage } = this;
|
|
207
301
|
if (!blocks) blocks = await this.allocateBlocks(data.length);
|
|
208
302
|
let offset = 0;
|
|
209
303
|
for (let i = 0, numBlocks = blocks.length - 1; i <= numBlocks; i++) {
|
|
304
|
+
if (offset >= data.length)
|
|
305
|
+
illegalState(`too many blocks, EOF already reached`);
|
|
210
306
|
const id = blocks[i];
|
|
211
|
-
const block = await
|
|
212
|
-
i < numBlocks ? this.setBlockMeta(block, blocks[i + 1],
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
const chunk = data.subarray(offset, offset + this.blockDataSize);
|
|
218
|
-
block.set(chunk, this.blockDataOffset);
|
|
219
|
-
if (chunk.length < this.blockDataSize) {
|
|
220
|
-
block.fill(0, this.blockDataOffset + chunk.length);
|
|
307
|
+
const block = await storage.loadBlock(id);
|
|
308
|
+
i < numBlocks ? this.setBlockMeta(block, blocks[i + 1], blockDataSize) : this.setBlockMeta(block, sentinelID, data.length - offset);
|
|
309
|
+
const chunk = data.subarray(offset, offset + blockDataSize);
|
|
310
|
+
block.set(chunk, blockDataOffset);
|
|
311
|
+
if (chunk.length < blockDataSize) {
|
|
312
|
+
block.fill(0, blockDataOffset + chunk.length);
|
|
221
313
|
}
|
|
222
|
-
await
|
|
223
|
-
offset +=
|
|
314
|
+
await storage.saveBlock(id, block);
|
|
315
|
+
offset += blockDataSize;
|
|
224
316
|
}
|
|
225
317
|
return {
|
|
226
318
|
start: blocks[0],
|
|
@@ -230,7 +322,7 @@ class BlockFS {
|
|
|
230
322
|
}
|
|
231
323
|
async writeFile(path, data) {
|
|
232
324
|
if (isString(data)) data = new TextEncoder().encode(data);
|
|
233
|
-
if (!path) return this.
|
|
325
|
+
if (!path) return this.writeBlocks(null, data);
|
|
234
326
|
const entry = await this.ensureEntryForPath(path, EntryType.FILE);
|
|
235
327
|
let blocks = await this.blockList(entry.start);
|
|
236
328
|
const overflow = data.length - blocks.length * this.blockDataSize;
|
|
@@ -241,37 +333,38 @@ class BlockFS {
|
|
|
241
333
|
await this.freeBlocks(blocks.slice(needed));
|
|
242
334
|
blocks = blocks.slice(0, needed);
|
|
243
335
|
}
|
|
244
|
-
blocks.sort();
|
|
336
|
+
blocks.sort((a, b) => a - b);
|
|
245
337
|
entry.start = blocks[0];
|
|
246
338
|
entry.end = blocks[blocks.length - 1];
|
|
247
339
|
entry.size = BigInt(data.length);
|
|
248
340
|
entry.mtime = Date.now();
|
|
249
341
|
await entry.save();
|
|
250
|
-
return this.
|
|
342
|
+
return this.writeBlocks(blocks, data);
|
|
251
343
|
}
|
|
252
|
-
async
|
|
253
|
-
if (blockID === this.sentinelID) return this.
|
|
344
|
+
async appendBlocks(blockID, data) {
|
|
345
|
+
if (blockID === this.sentinelID) return this.writeBlocks(null, data);
|
|
346
|
+
const { blockDataOffset, blockDataSize, storage } = this;
|
|
254
347
|
const blocks = await this.blockList(blockID);
|
|
255
348
|
const lastBlockID = blocks[blocks.length - 1];
|
|
256
|
-
const lastBlock = await
|
|
349
|
+
const lastBlock = await storage.loadBlock(lastBlockID);
|
|
257
350
|
const currLength = decodeBytes(
|
|
258
351
|
lastBlock,
|
|
259
352
|
this.blockIDBytes,
|
|
260
353
|
this.blockDataSizeBytes
|
|
261
354
|
);
|
|
262
|
-
const remaining =
|
|
263
|
-
lastBlock.fill(0, 0,
|
|
355
|
+
const remaining = blockDataSize - currLength;
|
|
356
|
+
lastBlock.fill(0, 0, blockDataOffset);
|
|
264
357
|
lastBlock.set(
|
|
265
358
|
data.subarray(0, remaining),
|
|
266
|
-
|
|
359
|
+
blockDataOffset + currLength
|
|
267
360
|
);
|
|
268
361
|
let newEndBlockID;
|
|
269
362
|
if (data.length > remaining) {
|
|
270
|
-
const { start, end } = await this.
|
|
363
|
+
const { start, end } = await this.writeBlocks(
|
|
271
364
|
null,
|
|
272
365
|
data.subarray(remaining)
|
|
273
366
|
);
|
|
274
|
-
this.setBlockMeta(lastBlock, start,
|
|
367
|
+
this.setBlockMeta(lastBlock, start, blockDataSize);
|
|
275
368
|
newEndBlockID = end;
|
|
276
369
|
} else {
|
|
277
370
|
this.setBlockMeta(
|
|
@@ -281,14 +374,14 @@ class BlockFS {
|
|
|
281
374
|
);
|
|
282
375
|
newEndBlockID = lastBlockID;
|
|
283
376
|
}
|
|
284
|
-
await
|
|
377
|
+
await storage.saveBlock(lastBlockID, lastBlock);
|
|
285
378
|
return { start: blockID, end: newEndBlockID };
|
|
286
379
|
}
|
|
287
380
|
async appendFile(path, data) {
|
|
288
381
|
if (isString(data)) data = new TextEncoder().encode(data);
|
|
289
|
-
if (!isString(path)) return this.
|
|
382
|
+
if (!isString(path)) return this.appendBlocks(path, data);
|
|
290
383
|
const entry = await this.ensureEntryForPath(path, EntryType.FILE);
|
|
291
|
-
const { start, end } = await this.
|
|
384
|
+
const { start, end } = await this.appendBlocks(entry.end, data);
|
|
292
385
|
if (entry.start === this.sentinelID) entry.start = start;
|
|
293
386
|
entry.end = end;
|
|
294
387
|
entry.size += BigInt(data.length);
|
|
@@ -308,33 +401,35 @@ class BlockFS {
|
|
|
308
401
|
}
|
|
309
402
|
}
|
|
310
403
|
async blockList(blockID) {
|
|
404
|
+
const { blockIDBytes, sentinelID, storage } = this;
|
|
311
405
|
const blocks = [];
|
|
312
|
-
if (blockID ===
|
|
406
|
+
if (blockID === sentinelID) return blocks;
|
|
313
407
|
while (true) {
|
|
314
408
|
blocks.push(blockID);
|
|
315
|
-
const block = await
|
|
316
|
-
const nextID = decodeBytes(block, 0,
|
|
317
|
-
if (nextID ===
|
|
409
|
+
const block = await storage.loadBlock(blockID);
|
|
410
|
+
const nextID = decodeBytes(block, 0, blockIDBytes);
|
|
411
|
+
if (nextID === sentinelID) break;
|
|
318
412
|
blockID = nextID;
|
|
319
413
|
}
|
|
320
414
|
return blocks;
|
|
321
415
|
}
|
|
322
416
|
async updateBlockIndex(ids, state) {
|
|
323
|
-
const
|
|
417
|
+
const { blockIndex, storage, tmp } = this;
|
|
418
|
+
const blockSize = storage.blockSize;
|
|
324
419
|
const updatedBlocks = /* @__PURE__ */ new Set();
|
|
325
420
|
for (let id of ids) {
|
|
326
|
-
|
|
421
|
+
blockIndex.setAt(id, state);
|
|
327
422
|
updatedBlocks.add((id >>> 3) / blockSize | 0);
|
|
328
423
|
}
|
|
329
424
|
for (let id of updatedBlocks) {
|
|
330
425
|
this.opts.logger.debug("update block index", id);
|
|
331
|
-
const chunk =
|
|
426
|
+
const chunk = blockIndex.data.subarray(
|
|
332
427
|
id * blockSize,
|
|
333
428
|
(id + 1) * blockSize
|
|
334
429
|
);
|
|
335
|
-
|
|
336
|
-
if (chunk.length < blockSize)
|
|
337
|
-
await
|
|
430
|
+
tmp.set(chunk);
|
|
431
|
+
if (chunk.length < blockSize) tmp.fill(0, chunk.length);
|
|
432
|
+
await storage.saveBlock(id, tmp);
|
|
338
433
|
}
|
|
339
434
|
}
|
|
340
435
|
/** @internal */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thi.ng/block-fs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Customizable block-based storage, adapters & file system layer",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./index.js",
|
|
@@ -118,5 +118,5 @@
|
|
|
118
118
|
"status": "alpha",
|
|
119
119
|
"year": 2024
|
|
120
120
|
},
|
|
121
|
-
"gitHead": "
|
|
121
|
+
"gitHead": "de166de7bb358998c797090a49bf55bcb7c325ba\n"
|
|
122
122
|
}
|