@thi.ng/block-fs 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/README.md +120 -1
- package/bin/blockfs +20 -0
- package/cli.d.ts +23 -0
- package/cli.js +214 -0
- package/directory.js +1 -1
- package/fs.d.ts +13 -0
- package/fs.js +18 -1
- package/package.json +13 -2
- package/storage/file.d.ts +6 -0
- package/storage/file.js +6 -9
- package/storage/memory.d.ts +11 -1
- package/storage/memory.js +15 -9
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
-
- **Last updated**: 2025-04-
|
|
3
|
+
- **Last updated**: 2025-04-02T18:47:59Z
|
|
4
4
|
- **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
|
|
5
5
|
|
|
6
6
|
All notable changes to this project will be documented in this file.
|
|
@@ -11,6 +11,30 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
|
|
|
11
11
|
**Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
|
|
12
12
|
and/or version bumps of transitive dependencies.
|
|
13
13
|
|
|
14
|
+
## [0.3.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.3.0) (2025-04-02)
|
|
15
|
+
|
|
16
|
+
#### 🚀 Features
|
|
17
|
+
|
|
18
|
+
- add `BlockFS.readAsObjectURL()` ([551327c](https://github.com/thi-ng/umbrella/commit/551327c))
|
|
19
|
+
- add mem storage opts, add logging ([27016d6](https://github.com/thi-ng/umbrella/commit/27016d6))
|
|
20
|
+
- add `MemoryBlockStorage` support for pre-loaded buffers
|
|
21
|
+
- add logging
|
|
22
|
+
- add docs
|
|
23
|
+
- add CLI app wrapper ([68abe74](https://github.com/thi-ng/umbrella/commit/68abe74))
|
|
24
|
+
- add CLI app wrapper with these commands:
|
|
25
|
+
- `convert`: convert file tree into single BlockFS blob
|
|
26
|
+
- `list`: list file tree of a BlockFS blob
|
|
27
|
+
- update deps
|
|
28
|
+
- improve tree display (`list` cmd) ([a23866c](https://github.com/thi-ng/umbrella/commit/a23866c))
|
|
29
|
+
|
|
30
|
+
#### 🩹 Bug fixes
|
|
31
|
+
|
|
32
|
+
- fix parent dir linkage ([a121e76](https://github.com/thi-ng/umbrella/commit/a121e76))
|
|
33
|
+
|
|
34
|
+
#### ♻️ Refactoring
|
|
35
|
+
|
|
36
|
+
- update sentinel block ID ([51a1e44](https://github.com/thi-ng/umbrella/commit/51a1e44))
|
|
37
|
+
|
|
14
38
|
## [0.2.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.2.0) (2025-04-02)
|
|
15
39
|
|
|
16
40
|
#### 🚀 Features
|
package/README.md
CHANGED
|
@@ -21,6 +21,9 @@
|
|
|
21
21
|
- [Root directory](#root-directory)
|
|
22
22
|
- [Directory entries](#directory-entries)
|
|
23
23
|
- [File blocks](#file-blocks)
|
|
24
|
+
- [Command line app](#command-line-app)
|
|
25
|
+
- [Convert file tree into single BlockFS blob](#convert-file-tree-into-single-blockfs-blob)
|
|
26
|
+
- [List file tree of a BlockFS blob](#list-file-tree-of-a-blockfs-blob)
|
|
24
27
|
- [Status](#status)
|
|
25
28
|
- [Installation](#installation)
|
|
26
29
|
- [Dependencies](#dependencies)
|
|
@@ -102,6 +105,121 @@ the max. number of blocks in the storage backend.
|
|
|
102
105
|
|
|
103
106
|
TODO diagram
|
|
104
107
|
|
|
108
|
+
### Command line app
|
|
109
|
+
|
|
110
|
+
The package includes a mult-command CLI app with the following operations:
|
|
111
|
+
|
|
112
|
+
#### Convert file tree into single BlockFS blob
|
|
113
|
+
|
|
114
|
+
The `convert` command is used to bundle an entire file tree from the host system
|
|
115
|
+
into a single binary blob based on `BlockFS` with configured block size. The
|
|
116
|
+
file tree MUST fit into the RAM available to `bun` (or `node`).
|
|
117
|
+
|
|
118
|
+
Once bundled, the binary blob can then be used together with
|
|
119
|
+
[`MemoryBlockStorage`](https://docs.thi.ng/umbrella/block-fs/classes/MemoryBlockStorage.html)
|
|
120
|
+
and [`BlockFS`](https://docs.thi.ng/umbrella/block-fs/classes/BlockFS.html) for
|
|
121
|
+
other purposes (e.g. distributed with your web app to provide a virtual
|
|
122
|
+
filesystem).
|
|
123
|
+
|
|
124
|
+
Example usage to bundle the source directory of this package:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
npx @thi.ng/block-fs convert -o dummy.dat packages/block-fs/src/
|
|
128
|
+
|
|
129
|
+
# [INFO] blockfs: number of files: 11
|
|
130
|
+
# [INFO] blockfs: number of directories: 2
|
|
131
|
+
# [INFO] blockfs: total file size: 40341
|
|
132
|
+
# [INFO] blockfs: number of blocks: 56
|
|
133
|
+
# [INFO] blockfs: writing file: dummy.dat
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
General usage:
|
|
137
|
+
|
|
138
|
+
```text
|
|
139
|
+
npx @thi.ng/block-fs convert --help
|
|
140
|
+
|
|
141
|
+
Usage: blockfs <cmd> [opts] input
|
|
142
|
+
|
|
143
|
+
Flags:
|
|
144
|
+
|
|
145
|
+
-v, --verbose Display extra process information
|
|
146
|
+
|
|
147
|
+
Main:
|
|
148
|
+
|
|
149
|
+
-bs BYTES, --block-size BYTES Block size (default: 1024)
|
|
150
|
+
-n INT, --num-blocks INT Number of blocks (multiple of 8)
|
|
151
|
+
-o STR, --out STR [required] Output file path
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
#### List file tree of a BlockFS blob
|
|
155
|
+
|
|
156
|
+
The `list` command is used to list the files & directories stored in a binary blob created via the `convert` command. Several output options (e.g. `tree`-like output) are supported.
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
npx @thi.ng/block-fs list dummy.dat
|
|
160
|
+
# /api.ts
|
|
161
|
+
# /cli.ts
|
|
162
|
+
# /directory.ts
|
|
163
|
+
# /entry.ts
|
|
164
|
+
# /fs.ts
|
|
165
|
+
# /index.ts
|
|
166
|
+
# /lock.ts
|
|
167
|
+
# /storage
|
|
168
|
+
# /storage/astorage.ts
|
|
169
|
+
# /storage/file.ts
|
|
170
|
+
# /storage/memory.ts
|
|
171
|
+
# /utils.ts
|
|
172
|
+
|
|
173
|
+
npx @thi.ng/block-fs list --tree dummy.dat
|
|
174
|
+
# ├── api.ts
|
|
175
|
+
# ├── cli.ts
|
|
176
|
+
# ├── directory.ts
|
|
177
|
+
# ├── entry.ts
|
|
178
|
+
# ├── fs.ts
|
|
179
|
+
# ├── index.ts
|
|
180
|
+
# ├── lock.ts
|
|
181
|
+
# ├── storage
|
|
182
|
+
# │ ├── astorage.ts
|
|
183
|
+
# │ ├── file.ts
|
|
184
|
+
# │ └── memory.ts
|
|
185
|
+
# └── utils.ts
|
|
186
|
+
|
|
187
|
+
# display file sizes & modification times
|
|
188
|
+
npx @thi.ng/block-fs list --tree --all dummy.dat
|
|
189
|
+
# ├── api.ts 2204 2025-04-02T10:22:55.573Z
|
|
190
|
+
# ├── cli.ts 6799 2025-04-02T18:07:58.895Z
|
|
191
|
+
# ├── directory.ts 3994 2025-04-02T13:47:00.108Z
|
|
192
|
+
# ├── entry.ts 4130 2025-04-02T10:22:55.574Z
|
|
193
|
+
# ├── fs.ts 16377 2025-04-02T13:46:36.608Z
|
|
194
|
+
# ├── index.ts 317 2025-04-01T21:38:08.232Z
|
|
195
|
+
# ├── lock.ts 1501 2025-04-01T21:38:08.232Z
|
|
196
|
+
# ├── storage 2025-04-02T18:33:47.389Z
|
|
197
|
+
# │ ├── astorage.ts 1205 2025-04-02T10:22:55.574Z
|
|
198
|
+
# │ ├── file.ts 1780 2025-04-02T14:25:12.461Z
|
|
199
|
+
# │ └── memory.ts 1802 2025-04-02T14:26:02.163Z
|
|
200
|
+
# └── utils.ts 418 2025-04-02T10:22:55.574Z
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
General usage:
|
|
204
|
+
|
|
205
|
+
```text
|
|
206
|
+
npx @thi.ng/block-fs list --help
|
|
207
|
+
|
|
208
|
+
Usage: blockfs <cmd> [opts] input
|
|
209
|
+
|
|
210
|
+
Flags:
|
|
211
|
+
|
|
212
|
+
-a, --all Display all attribs
|
|
213
|
+
-t, --tree List files as tree
|
|
214
|
+
-v, --verbose Display extra process information
|
|
215
|
+
-m, --with-mtime Display modified times
|
|
216
|
+
-s, --with-size Display file sizes
|
|
217
|
+
|
|
218
|
+
Main:
|
|
219
|
+
|
|
220
|
+
-bs BYTES, --block-size BYTES Block size (default: 1024)
|
|
221
|
+
```
|
|
222
|
+
|
|
105
223
|
## Status
|
|
106
224
|
|
|
107
225
|
**ALPHA** - bleeding edge / work-in-progress
|
|
@@ -134,11 +252,12 @@ For Node.js REPL:
|
|
|
134
252
|
const bf = await import("@thi.ng/block-fs");
|
|
135
253
|
```
|
|
136
254
|
|
|
137
|
-
Package sizes (brotli'd, pre-treeshake): ESM: 4.
|
|
255
|
+
Package sizes (brotli'd, pre-treeshake): ESM: 4.43 KB
|
|
138
256
|
|
|
139
257
|
## Dependencies
|
|
140
258
|
|
|
141
259
|
- [@thi.ng/api](https://github.com/thi-ng/umbrella/tree/develop/packages/api)
|
|
260
|
+
- [@thi.ng/args](https://github.com/thi-ng/umbrella/tree/develop/packages/args)
|
|
142
261
|
- [@thi.ng/binary](https://github.com/thi-ng/umbrella/tree/develop/packages/binary)
|
|
143
262
|
- [@thi.ng/bitfield](https://github.com/thi-ng/umbrella/tree/develop/packages/bitfield)
|
|
144
263
|
- [@thi.ng/checks](https://github.com/thi-ng/umbrella/tree/develop/packages/checks)
|
package/bin/blockfs
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
|
|
3
|
+
# https://stackoverflow.com/a/246128/294515
|
|
4
|
+
SOURCE="${BASH_SOURCE[0]}"
|
|
5
|
+
while [ -L "$SOURCE" ]; do
|
|
6
|
+
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
|
|
7
|
+
SOURCE="$(readlink "$SOURCE")"
|
|
8
|
+
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
|
|
9
|
+
done
|
|
10
|
+
DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
|
|
11
|
+
DIR="$(dirname $DIR)"
|
|
12
|
+
|
|
13
|
+
# prefer using bun
|
|
14
|
+
if [ -x "$(command -v bun)" ]; then
|
|
15
|
+
CMD=bun
|
|
16
|
+
else
|
|
17
|
+
CMD=node
|
|
18
|
+
fi
|
|
19
|
+
|
|
20
|
+
/usr/bin/env $CMD "$DIR/cli.js" "$DIR" "$@"
|
package/cli.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type Command, type CommandCtx } from "@thi.ng/args";
|
|
2
|
+
interface CLIOpts {
|
|
3
|
+
verbose: boolean;
|
|
4
|
+
}
|
|
5
|
+
interface ConvertOpts extends CLIOpts {
|
|
6
|
+
numBlocks?: number;
|
|
7
|
+
blockSize: number;
|
|
8
|
+
include?: string[];
|
|
9
|
+
out: string;
|
|
10
|
+
}
|
|
11
|
+
interface ListOpts extends CLIOpts {
|
|
12
|
+
blockSize: number;
|
|
13
|
+
all: boolean;
|
|
14
|
+
tree: boolean;
|
|
15
|
+
withMtime: boolean;
|
|
16
|
+
withSize: boolean;
|
|
17
|
+
}
|
|
18
|
+
export interface AppCtx<T extends CLIOpts> extends CommandCtx<T, CLIOpts> {
|
|
19
|
+
}
|
|
20
|
+
export declare const CONVERT: Command<ConvertOpts, CLIOpts, AppCtx<ConvertOpts>>;
|
|
21
|
+
export declare const LIST: Command<ListOpts, CLIOpts, AppCtx<ListOpts>>;
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=cli.d.ts.map
|
package/cli.js
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import {
|
|
2
|
+
cliApp,
|
|
3
|
+
flag,
|
|
4
|
+
int,
|
|
5
|
+
string,
|
|
6
|
+
strings
|
|
7
|
+
} from "@thi.ng/args";
|
|
8
|
+
import { align, isPow2 } from "@thi.ng/binary";
|
|
9
|
+
import { illegalArgs } from "@thi.ng/errors";
|
|
10
|
+
import { files, readBinary, readJSON, writeFile } from "@thi.ng/file-io";
|
|
11
|
+
import { LogLevel } from "@thi.ng/logger";
|
|
12
|
+
import { statSync } from "node:fs";
|
|
13
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
14
|
+
import { Entry } from "./entry.js";
|
|
15
|
+
import { BlockFS } from "./fs.js";
|
|
16
|
+
import { MemoryBlockStorage } from "./storage/memory.js";
|
|
17
|
+
const PKG = readJSON(join(process.argv[2], "package.json"));
|
|
18
|
+
const ARG_BLOCKSIZE = {
|
|
19
|
+
blockSize: int({
|
|
20
|
+
alias: "bs",
|
|
21
|
+
desc: "Block size",
|
|
22
|
+
hint: "BYTES",
|
|
23
|
+
default: 1024,
|
|
24
|
+
coerce: (x) => {
|
|
25
|
+
const size = +x;
|
|
26
|
+
if (!isPow2(size)) illegalArgs("block size must be a power of 2");
|
|
27
|
+
return size;
|
|
28
|
+
}
|
|
29
|
+
})
|
|
30
|
+
};
|
|
31
|
+
const collectFiles = (ctx) => {
|
|
32
|
+
const root = resolve(ctx.inputs[0]);
|
|
33
|
+
const filtered = [];
|
|
34
|
+
const dirs = /* @__PURE__ */ new Set();
|
|
35
|
+
let total = 0;
|
|
36
|
+
for (let f of files(root)) {
|
|
37
|
+
const stats = statSync(f);
|
|
38
|
+
if (stats.isFile()) {
|
|
39
|
+
const dest = relative(root, f);
|
|
40
|
+
filtered.push({
|
|
41
|
+
src: f,
|
|
42
|
+
dest,
|
|
43
|
+
size: stats.size,
|
|
44
|
+
ctime: stats.ctimeMs,
|
|
45
|
+
mtime: stats.mtimeMs
|
|
46
|
+
});
|
|
47
|
+
dirs.add(dirname(dest));
|
|
48
|
+
total += stats.size;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return { files: filtered, dirs: [...dirs], size: total };
|
|
52
|
+
};
|
|
53
|
+
const computeBlockCount = (collected, blockSize, numBlocks) => {
|
|
54
|
+
let blocks = collected.dirs.length;
|
|
55
|
+
const blockIDBytes = numBlocks ? align(Math.ceil(Math.log2(numBlocks)), 8) >> 3 : align(Math.ceil(Math.log2(collected.size / blockSize + blocks)), 8) >> 3;
|
|
56
|
+
const blockDataSizeBytes = align(Math.ceil(Math.log2(blockSize)), 8) >> 3;
|
|
57
|
+
const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
|
|
58
|
+
blocks += Math.ceil(
|
|
59
|
+
(collected.files.length + collected.dirs.length) * Entry.SIZE / blockDataSize
|
|
60
|
+
);
|
|
61
|
+
for (let f of collected.files) {
|
|
62
|
+
blocks += Math.ceil(f.size / blockDataSize);
|
|
63
|
+
}
|
|
64
|
+
const blockIDBytes2 = align(Math.ceil(Math.log2(blocks)), 8) >> 3;
|
|
65
|
+
return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, blocks) : blocks;
|
|
66
|
+
};
|
|
67
|
+
const CONVERT = {
|
|
68
|
+
opts: {
|
|
69
|
+
...ARG_BLOCKSIZE,
|
|
70
|
+
numBlocks: int({
|
|
71
|
+
alias: "n",
|
|
72
|
+
desc: "Number of blocks (multiple of 8)",
|
|
73
|
+
optional: true
|
|
74
|
+
}),
|
|
75
|
+
out: string({
|
|
76
|
+
alias: "o",
|
|
77
|
+
desc: "Output file path",
|
|
78
|
+
optional: false
|
|
79
|
+
}),
|
|
80
|
+
include: strings({
|
|
81
|
+
alias: "i",
|
|
82
|
+
desc: "Only include file extensions",
|
|
83
|
+
hint: "EXT"
|
|
84
|
+
})
|
|
85
|
+
},
|
|
86
|
+
desc: "Convert file tree into single BlockFS blob",
|
|
87
|
+
fn: async (ctx) => {
|
|
88
|
+
const collected = collectFiles(ctx);
|
|
89
|
+
const numBlocks = align(
|
|
90
|
+
ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize),
|
|
91
|
+
8
|
|
92
|
+
);
|
|
93
|
+
ctx.logger.info("number of files:", collected.files.length);
|
|
94
|
+
ctx.logger.info("number of directories:", collected.dirs.length);
|
|
95
|
+
ctx.logger.info("total file size:", collected.size);
|
|
96
|
+
ctx.logger.info("number of blocks:", numBlocks);
|
|
97
|
+
const storage = new MemoryBlockStorage({
|
|
98
|
+
numBlocks,
|
|
99
|
+
blockSize: ctx.opts.blockSize,
|
|
100
|
+
logger: ctx.logger
|
|
101
|
+
});
|
|
102
|
+
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
103
|
+
await bfs.init();
|
|
104
|
+
for (let f of collected.files) {
|
|
105
|
+
ctx.logger.debug("writing file:", f.dest);
|
|
106
|
+
await bfs.writeFile(f.dest, readBinary(f.src));
|
|
107
|
+
const entry = await bfs.entryForPath(f.dest);
|
|
108
|
+
entry.ctime = f.ctime;
|
|
109
|
+
entry.mtime = f.mtime;
|
|
110
|
+
await entry.save();
|
|
111
|
+
}
|
|
112
|
+
writeFile(ctx.opts.out, storage.buffer, void 0, ctx.logger);
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
const LIST = {
|
|
116
|
+
opts: {
|
|
117
|
+
...ARG_BLOCKSIZE,
|
|
118
|
+
all: flag({
|
|
119
|
+
alias: "a",
|
|
120
|
+
desc: "Display all attribs"
|
|
121
|
+
}),
|
|
122
|
+
tree: flag({
|
|
123
|
+
alias: "t",
|
|
124
|
+
desc: "List files as tree"
|
|
125
|
+
}),
|
|
126
|
+
withMtime: flag({
|
|
127
|
+
alias: "m",
|
|
128
|
+
desc: "Display modified times"
|
|
129
|
+
}),
|
|
130
|
+
withSize: flag({
|
|
131
|
+
alias: "s",
|
|
132
|
+
desc: "Display file sizes"
|
|
133
|
+
})
|
|
134
|
+
},
|
|
135
|
+
desc: "List file tree of a BlockFS blob",
|
|
136
|
+
fn: async (ctx) => {
|
|
137
|
+
if (ctx.opts.all) {
|
|
138
|
+
ctx.opts.withMtime = ctx.opts.withSize = true;
|
|
139
|
+
}
|
|
140
|
+
const buffer = readBinary(ctx.inputs[0]);
|
|
141
|
+
const storage = new MemoryBlockStorage({
|
|
142
|
+
numBlocks: buffer.length / ctx.opts.blockSize >>> 0,
|
|
143
|
+
blockSize: ctx.opts.blockSize,
|
|
144
|
+
logger: ctx.logger,
|
|
145
|
+
buffer
|
|
146
|
+
});
|
|
147
|
+
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
148
|
+
await bfs.init();
|
|
149
|
+
const tree = [];
|
|
150
|
+
for await (let entry of bfs.root.tree()) {
|
|
151
|
+
const path = entry.path;
|
|
152
|
+
const depth = path.split("/").length - 2;
|
|
153
|
+
tree.push([path, depth, entry]);
|
|
154
|
+
}
|
|
155
|
+
tree.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
156
|
+
const rows = [];
|
|
157
|
+
const maxWidths = [0, 0, 0];
|
|
158
|
+
for (let i = 0, num = tree.length - 1; i <= num; i++) {
|
|
159
|
+
const f = tree[i];
|
|
160
|
+
const isLast = i === num || f[1] > tree[i + 1][1];
|
|
161
|
+
const row = ctx.opts.tree ? ["\u2502 ".repeat(f[1]) + (isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ") + f[2].name] : [f[0]];
|
|
162
|
+
if (ctx.opts.withSize) {
|
|
163
|
+
row.push(f[2].isDirectory() ? "" : String(Number(f[2].size)));
|
|
164
|
+
}
|
|
165
|
+
if (ctx.opts.withMtime) {
|
|
166
|
+
row.push(new Date(f[2].mtime).toISOString());
|
|
167
|
+
}
|
|
168
|
+
rows.push(row);
|
|
169
|
+
for (let i2 = 0; i2 < row.length; i2++) {
|
|
170
|
+
maxWidths[i2] = Math.max(maxWidths[i2], row[i2].length);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
for (let row of rows) {
|
|
174
|
+
console.log(row.map((x, i) => x.padEnd(maxWidths[i])).join(" "));
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
cliApp({
|
|
179
|
+
opts: {
|
|
180
|
+
verbose: flag({
|
|
181
|
+
alias: "v",
|
|
182
|
+
desc: "Display extra process information"
|
|
183
|
+
})
|
|
184
|
+
},
|
|
185
|
+
commands: {
|
|
186
|
+
convert: CONVERT,
|
|
187
|
+
list: LIST
|
|
188
|
+
},
|
|
189
|
+
name: "blockfs",
|
|
190
|
+
ctx: async (ctx) => {
|
|
191
|
+
if (ctx.opts.verbose) ctx.logger.level = LogLevel.DEBUG;
|
|
192
|
+
return ctx;
|
|
193
|
+
},
|
|
194
|
+
start: 3,
|
|
195
|
+
usage: {
|
|
196
|
+
prefix: `
|
|
197
|
+
\u2588 \u2588 \u2588 \u2502
|
|
198
|
+
\u2588\u2588 \u2588 \u2502
|
|
199
|
+
\u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2502 ${PKG.name} ${PKG.version}
|
|
200
|
+
\u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2502 Block-based storage & file system layer
|
|
201
|
+
\u2588 \u2502
|
|
202
|
+
\u2588 \u2588 \u2502
|
|
203
|
+
|
|
204
|
+
Usage: blockfs <cmd> [opts] input [...]
|
|
205
|
+
blockfs <cmd> --help
|
|
206
|
+
`,
|
|
207
|
+
showGroupNames: true,
|
|
208
|
+
paramWidth: 32
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
export {
|
|
212
|
+
CONVERT,
|
|
213
|
+
LIST
|
|
214
|
+
};
|
package/directory.js
CHANGED
|
@@ -65,7 +65,7 @@ class Directory {
|
|
|
65
65
|
const block = (await fs.allocateBlocks(1))[0];
|
|
66
66
|
const data = await fs.storage.loadBlock(block);
|
|
67
67
|
fs.setBlockMeta(data, fs.sentinelID, 0);
|
|
68
|
-
fs.setBlockLink(data, this.entry.start, fs.
|
|
68
|
+
fs.setBlockLink(data, this.entry.start, fs.blockDataOffset);
|
|
69
69
|
await fs.storage.saveBlock(block, data);
|
|
70
70
|
return this.addEntry(
|
|
71
71
|
{
|
package/fs.d.ts
CHANGED
|
@@ -155,6 +155,19 @@ export declare class BlockFS {
|
|
|
155
155
|
* @param path
|
|
156
156
|
*/
|
|
157
157
|
readJSON<T>(path: string | number): Promise<T>;
|
|
158
|
+
/**
|
|
159
|
+
* Fully reads given file into a single byte buffer and returns it as blob
|
|
160
|
+
* object URL, optionally typed with given MIME type.
|
|
161
|
+
*
|
|
162
|
+
* @remarks
|
|
163
|
+
* Reference:
|
|
164
|
+
*
|
|
165
|
+
* - https://developer.mozilla.org/en-US/docs/Web/API/Blob#creating_a_url_representing_the_contents_of_a_typed_array
|
|
166
|
+
*
|
|
167
|
+
* @param path
|
|
168
|
+
* @param type
|
|
169
|
+
*/
|
|
170
|
+
readAsObjectURL(path: string | number, type?: string): Promise<string>;
|
|
158
171
|
/**
|
|
159
172
|
* Takes an array of block IDs (or `null`) and a `data` byte array. Writes
|
|
160
173
|
* chunks of data into given blocks and connecting each block as linked
|
package/fs.js
CHANGED
|
@@ -41,7 +41,7 @@ class BlockFS {
|
|
|
41
41
|
this.rootDirBlockID = align(this.blockIndex.data.length, storage.blockSize) / storage.blockSize;
|
|
42
42
|
this.dataStartBlockID = this.rootDirBlockID + 1;
|
|
43
43
|
this.dirDataOffset = this.blockDataOffset + this.blockIDBytes;
|
|
44
|
-
this.sentinelID =
|
|
44
|
+
this.sentinelID = 2 ** (this.blockIDBytes * 8) - 1;
|
|
45
45
|
this.tmp = new Uint8Array(storage.blockSize);
|
|
46
46
|
}
|
|
47
47
|
blockIndex;
|
|
@@ -284,6 +284,23 @@ class BlockFS {
|
|
|
284
284
|
async readJSON(path) {
|
|
285
285
|
return JSON.parse(await this.readText(path));
|
|
286
286
|
}
|
|
287
|
+
/**
|
|
288
|
+
* Fully reads given file into a single byte buffer and returns it as blob
|
|
289
|
+
* object URL, optionally typed with given MIME type.
|
|
290
|
+
*
|
|
291
|
+
* @remarks
|
|
292
|
+
* Reference:
|
|
293
|
+
*
|
|
294
|
+
* - https://developer.mozilla.org/en-US/docs/Web/API/Blob#creating_a_url_representing_the_contents_of_a_typed_array
|
|
295
|
+
*
|
|
296
|
+
* @param path
|
|
297
|
+
* @param type
|
|
298
|
+
*/
|
|
299
|
+
async readAsObjectURL(path, type) {
|
|
300
|
+
return URL.createObjectURL(
|
|
301
|
+
new Blob([await this.readFile(path)], { type })
|
|
302
|
+
);
|
|
303
|
+
}
|
|
287
304
|
/**
|
|
288
305
|
* Takes an array of block IDs (or `null`) and a `data` byte array. Writes
|
|
289
306
|
* chunks of data into given blocks and connecting each block as linked
|
package/package.json
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thi.ng/block-fs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Customizable block-based storage, adapters & file system layer",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./index.js",
|
|
7
7
|
"typings": "./index.d.ts",
|
|
8
|
+
"bin": {
|
|
9
|
+
"blockfs": "bin/blockfs"
|
|
10
|
+
},
|
|
8
11
|
"sideEffects": false,
|
|
9
12
|
"repository": {
|
|
10
13
|
"type": "git",
|
|
@@ -40,6 +43,7 @@
|
|
|
40
43
|
},
|
|
41
44
|
"dependencies": {
|
|
42
45
|
"@thi.ng/api": "^8.11.25",
|
|
46
|
+
"@thi.ng/args": "^2.3.66",
|
|
43
47
|
"@thi.ng/binary": "^3.4.48",
|
|
44
48
|
"@thi.ng/bitfield": "^2.4.0",
|
|
45
49
|
"@thi.ng/checks": "^3.7.5",
|
|
@@ -58,10 +62,13 @@
|
|
|
58
62
|
"keywords": [
|
|
59
63
|
"binary",
|
|
60
64
|
"block",
|
|
65
|
+
"cli",
|
|
66
|
+
"conversion",
|
|
61
67
|
"file",
|
|
62
68
|
"file-system",
|
|
63
69
|
"memory",
|
|
64
70
|
"memory-mapped",
|
|
71
|
+
"nodejs",
|
|
65
72
|
"path",
|
|
66
73
|
"storage",
|
|
67
74
|
"typedarray",
|
|
@@ -80,6 +87,7 @@
|
|
|
80
87
|
"files": [
|
|
81
88
|
"./*.js",
|
|
82
89
|
"./*.d.ts",
|
|
90
|
+
"bin",
|
|
83
91
|
"storage"
|
|
84
92
|
],
|
|
85
93
|
"exports": {
|
|
@@ -89,6 +97,9 @@
|
|
|
89
97
|
"./api": {
|
|
90
98
|
"default": "./api.js"
|
|
91
99
|
},
|
|
100
|
+
"./cli": {
|
|
101
|
+
"default": "./cli.js"
|
|
102
|
+
},
|
|
92
103
|
"./directory": {
|
|
93
104
|
"default": "./directory.js"
|
|
94
105
|
},
|
|
@@ -118,5 +129,5 @@
|
|
|
118
129
|
"status": "alpha",
|
|
119
130
|
"year": 2024
|
|
120
131
|
},
|
|
121
|
-
"gitHead": "
|
|
132
|
+
"gitHead": "549e798e6a82254ee68727f12229c5252892b8f1\n"
|
|
122
133
|
}
|
package/storage/file.d.ts
CHANGED
|
@@ -8,7 +8,13 @@ export declare class FileBlock implements IBlock {
|
|
|
8
8
|
save(data: Uint8Array): Promise<void>;
|
|
9
9
|
delete(): Promise<void>;
|
|
10
10
|
}
|
|
11
|
+
/**
|
|
12
|
+
* Configuration options for {@link FileBlockStorage}.
|
|
13
|
+
*/
|
|
11
14
|
export interface FileBlockStorageOpts extends BlockStorageOpts {
|
|
15
|
+
/**
|
|
16
|
+
* Path to host filesystem base directory used for storing blocks.
|
|
17
|
+
*/
|
|
12
18
|
baseDir: string;
|
|
13
19
|
}
|
|
14
20
|
export declare class FileBlockStorage extends ABlockStorage<FileBlock> {
|
package/storage/file.js
CHANGED
|
@@ -11,18 +11,15 @@ class FileBlock {
|
|
|
11
11
|
this.id = id;
|
|
12
12
|
}
|
|
13
13
|
async load() {
|
|
14
|
-
const
|
|
15
|
-
|
|
14
|
+
const { storage } = this;
|
|
15
|
+
const path = storage.getPath(this.id);
|
|
16
|
+
return existsSync(path) ? readBinary(path, storage.logger) : new Uint8Array(storage.blockSize);
|
|
16
17
|
}
|
|
17
18
|
async save(data) {
|
|
18
|
-
|
|
19
|
+
const { storage } = this;
|
|
20
|
+
if (data.length !== storage.blockSize)
|
|
19
21
|
illegalArgs(`wrong block size: ${data.length}`);
|
|
20
|
-
writeFile(
|
|
21
|
-
this.storage.getPath(this.id),
|
|
22
|
-
data,
|
|
23
|
-
null,
|
|
24
|
-
this.storage.logger
|
|
25
|
-
);
|
|
22
|
+
writeFile(storage.getPath(this.id), data, null, storage.logger);
|
|
26
23
|
}
|
|
27
24
|
async delete() {
|
|
28
25
|
deleteFile(this.storage.getPath(this.id), this.storage.logger);
|
package/storage/memory.d.ts
CHANGED
|
@@ -8,9 +8,19 @@ export declare class MemoryBlock implements IBlock {
|
|
|
8
8
|
save(data: Uint8Array): Promise<void>;
|
|
9
9
|
delete(): Promise<void>;
|
|
10
10
|
}
|
|
11
|
+
/**
|
|
12
|
+
* Configuration options for {@link MemoryBlockStorage}.
|
|
13
|
+
*/
|
|
14
|
+
export interface MemoryBlockStorageOpts extends BlockStorageOpts {
|
|
15
|
+
/**
|
|
16
|
+
* Optional, pre-defined/loaded byte buffer. Must have at least `numBlocks *
|
|
17
|
+
* blockSize` capacity.
|
|
18
|
+
*/
|
|
19
|
+
buffer?: Uint8Array;
|
|
20
|
+
}
|
|
11
21
|
export declare class MemoryBlockStorage extends ABlockStorage<MemoryBlock> {
|
|
12
22
|
buffer: Uint8Array;
|
|
13
|
-
constructor(opts:
|
|
23
|
+
constructor(opts: MemoryBlockStorageOpts);
|
|
14
24
|
hasBlock(id: number): Promise<boolean>;
|
|
15
25
|
ensureBlock(id: number): MemoryBlock;
|
|
16
26
|
}
|
package/storage/memory.js
CHANGED
|
@@ -6,16 +6,16 @@ class MemoryBlock {
|
|
|
6
6
|
this.id = id;
|
|
7
7
|
}
|
|
8
8
|
async load() {
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
);
|
|
9
|
+
const { storage } = this;
|
|
10
|
+
storage.logger.debug("load block", this.id);
|
|
11
|
+
const size = storage.blockSize;
|
|
12
|
+
return storage.buffer.subarray(this.id * size, (this.id + 1) * size);
|
|
14
13
|
}
|
|
15
14
|
async save(data) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
const { storage } = this;
|
|
16
|
+
if (data.length !== storage.blockSize) illegalArgs(`wrong block size`);
|
|
17
|
+
storage.logger.debug("save block", this.id);
|
|
18
|
+
storage.buffer.set(data, this.id * storage.blockSize);
|
|
19
19
|
}
|
|
20
20
|
async delete() {
|
|
21
21
|
const size = this.storage.blockSize;
|
|
@@ -26,7 +26,13 @@ class MemoryBlockStorage extends ABlockStorage {
|
|
|
26
26
|
buffer;
|
|
27
27
|
constructor(opts) {
|
|
28
28
|
super(opts);
|
|
29
|
-
|
|
29
|
+
const size = this.numBlocks * this.blockSize;
|
|
30
|
+
if (opts.buffer && opts.buffer.length < size) {
|
|
31
|
+
illegalArgs(
|
|
32
|
+
`given buffer is too small, expected at least ${size} bytes`
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
this.buffer = opts.buffer ?? new Uint8Array(size);
|
|
30
36
|
}
|
|
31
37
|
async hasBlock(id) {
|
|
32
38
|
this.ensureValidID(id);
|