@thi.ng/block-fs 0.5.10 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/README.md +2 -1
- package/api.d.ts +2 -2
- package/cli/api.d.ts +21 -0
- package/cli/api.js +19 -0
- package/cli/convert.d.ts +11 -0
- package/cli/convert.js +124 -0
- package/cli/list.d.ts +12 -0
- package/cli/list.js +87 -0
- package/cli.d.ts +0 -23
- package/cli.js +5 -208
- package/directory.d.ts +2 -1
- package/directory.js +16 -4
- package/package.json +14 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
-
- **Last updated**: 2025-07-
|
|
3
|
+
- **Last updated**: 2025-07-12T12:28:22Z
|
|
4
4
|
- **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
|
|
5
5
|
|
|
6
6
|
All notable changes to this project will be documented in this file.
|
|
@@ -11,6 +11,23 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
|
|
|
11
11
|
**Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
|
|
12
12
|
and/or version bumps of transitive dependencies.
|
|
13
13
|
|
|
14
|
+
### [0.6.1](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.6.1) (2025-07-12)
|
|
15
|
+
|
|
16
|
+
#### 🩹 Bug fixes
|
|
17
|
+
|
|
18
|
+
- fix tree output in CLI list cmd, update deps ([8d33970](https://github.com/thi-ng/umbrella/commit/8d33970))
|
|
19
|
+
- update entry sorting for tree output to avoid order edge cases w/ prev approach
|
|
20
|
+
|
|
21
|
+
## [0.6.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.6.0) (2025-07-11)
|
|
22
|
+
|
|
23
|
+
#### 🚀 Features
|
|
24
|
+
|
|
25
|
+
- update Directory.tree(), add optional comparator ([a3c7f4c](https://github.com/thi-ng/umbrella/commit/a3c7f4c))
|
|
26
|
+
|
|
27
|
+
#### ♻️ Refactoring
|
|
28
|
+
|
|
29
|
+
- split out CLI cmds to own files, fix tree display ([8665244](https://github.com/thi-ng/umbrella/commit/8665244))
|
|
30
|
+
|
|
14
31
|
### [0.5.3](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.5.3) (2025-06-27)
|
|
15
32
|
|
|
16
33
|
#### ♻️ Refactoring
|
package/README.md
CHANGED
|
@@ -294,7 +294,7 @@ For Node.js REPL:
|
|
|
294
294
|
const bf = await import("@thi.ng/block-fs");
|
|
295
295
|
```
|
|
296
296
|
|
|
297
|
-
Package sizes (brotli'd, pre-treeshake): ESM: 4.
|
|
297
|
+
Package sizes (brotli'd, pre-treeshake): ESM: 4.33 KB
|
|
298
298
|
|
|
299
299
|
## Dependencies
|
|
300
300
|
|
|
@@ -303,6 +303,7 @@ Package sizes (brotli'd, pre-treeshake): ESM: 4.29 KB
|
|
|
303
303
|
- [@thi.ng/binary](https://github.com/thi-ng/umbrella/tree/develop/packages/binary)
|
|
304
304
|
- [@thi.ng/bitfield](https://github.com/thi-ng/umbrella/tree/develop/packages/bitfield)
|
|
305
305
|
- [@thi.ng/checks](https://github.com/thi-ng/umbrella/tree/develop/packages/checks)
|
|
306
|
+
- [@thi.ng/compare](https://github.com/thi-ng/umbrella/tree/develop/packages/compare)
|
|
306
307
|
- [@thi.ng/errors](https://github.com/thi-ng/umbrella/tree/develop/packages/errors)
|
|
307
308
|
- [@thi.ng/file-io](https://github.com/thi-ng/umbrella/tree/develop/packages/file-io)
|
|
308
309
|
- [@thi.ng/logger](https://github.com/thi-ng/umbrella/tree/develop/packages/logger)
|
package/api.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Maybe } from "@thi.ng/api";
|
|
1
|
+
import type { Comparator, Maybe } from "@thi.ng/api";
|
|
2
2
|
import type { Pow2 } from "@thi.ng/binary";
|
|
3
3
|
import type { ILogger } from "@thi.ng/logger";
|
|
4
4
|
import type { BlockFS } from "./fs.js";
|
|
@@ -66,7 +66,7 @@ export interface IEntry extends Required<EntrySpec> {
|
|
|
66
66
|
export interface IDirectory extends AsyncIterable<IEntry> {
|
|
67
67
|
readonly fs: BlockFS;
|
|
68
68
|
readonly entry: IEntry;
|
|
69
|
-
tree(): AsyncIterableIterator<IEntry>;
|
|
69
|
+
tree(cmp?: Comparator<IEntry>): AsyncIterableIterator<IEntry>;
|
|
70
70
|
traverse(): Promise<{
|
|
71
71
|
blocks: number[];
|
|
72
72
|
entries: IEntry[];
|
package/cli/api.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type CommandCtx } from "@thi.ng/args";
|
|
2
|
+
export interface CLIOpts {
|
|
3
|
+
verbose: boolean;
|
|
4
|
+
quiet: boolean;
|
|
5
|
+
}
|
|
6
|
+
export declare const ARG_BLOCKSIZE: {
|
|
7
|
+
blockSize: {
|
|
8
|
+
alias: string;
|
|
9
|
+
desc: string;
|
|
10
|
+
hint: string;
|
|
11
|
+
default: number;
|
|
12
|
+
coerce: (x: string) => import("@thi.ng/binary").Pow2;
|
|
13
|
+
} & {
|
|
14
|
+
coerce: import("@thi.ng/api").Fn<string, number>;
|
|
15
|
+
hint: string;
|
|
16
|
+
group: string;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
export interface AppCtx<T extends CLIOpts> extends CommandCtx<T, CLIOpts> {
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=api.d.ts.map
|
package/cli/api.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { int } from "@thi.ng/args";
|
|
2
|
+
import { isPow2 } from "@thi.ng/binary";
|
|
3
|
+
import { illegalArgs } from "@thi.ng/errors";
|
|
4
|
+
const ARG_BLOCKSIZE = {
|
|
5
|
+
blockSize: int({
|
|
6
|
+
alias: "bs",
|
|
7
|
+
desc: "Block size",
|
|
8
|
+
hint: "BYTES",
|
|
9
|
+
default: 1024,
|
|
10
|
+
coerce: (x) => {
|
|
11
|
+
const size = +x;
|
|
12
|
+
if (!isPow2(size)) illegalArgs("block size must be a power of 2");
|
|
13
|
+
return size;
|
|
14
|
+
}
|
|
15
|
+
})
|
|
16
|
+
};
|
|
17
|
+
export {
|
|
18
|
+
ARG_BLOCKSIZE
|
|
19
|
+
};
|
package/cli/convert.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type Command } from "@thi.ng/args";
|
|
2
|
+
import { type AppCtx, type CLIOpts } from "./api.js";
|
|
3
|
+
export interface ConvertOpts extends CLIOpts {
|
|
4
|
+
numBlocks?: number;
|
|
5
|
+
blockSize: number;
|
|
6
|
+
exclude?: string[];
|
|
7
|
+
include?: string[];
|
|
8
|
+
out: string;
|
|
9
|
+
}
|
|
10
|
+
export declare const CONVERT: Command<ConvertOpts, CLIOpts, AppCtx<ConvertOpts>>;
|
|
11
|
+
//# sourceMappingURL=convert.d.ts.map
|
package/cli/convert.js
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { int, string, strings } from "@thi.ng/args";
|
|
2
|
+
import { align } from "@thi.ng/binary";
|
|
3
|
+
import { files, readBinary, writeFile } from "@thi.ng/file-io";
|
|
4
|
+
import { statSync } from "node:fs";
|
|
5
|
+
import { relative, resolve } from "node:path";
|
|
6
|
+
import { Entry } from "../entry.js";
|
|
7
|
+
import { BlockFS } from "../fs.js";
|
|
8
|
+
import { MemoryBlockStorage } from "../storage/memory.js";
|
|
9
|
+
import { ARG_BLOCKSIZE } from "./api.js";
|
|
10
|
+
const CONVERT = {
|
|
11
|
+
opts: {
|
|
12
|
+
...ARG_BLOCKSIZE,
|
|
13
|
+
numBlocks: int({
|
|
14
|
+
alias: "n",
|
|
15
|
+
desc: "Number of blocks (multiple of 8)",
|
|
16
|
+
optional: true
|
|
17
|
+
}),
|
|
18
|
+
out: string({
|
|
19
|
+
alias: "o",
|
|
20
|
+
desc: "Output file path",
|
|
21
|
+
optional: false
|
|
22
|
+
}),
|
|
23
|
+
exclude: strings({
|
|
24
|
+
alias: "e",
|
|
25
|
+
desc: "File exclusion regexp",
|
|
26
|
+
hint: "EXT"
|
|
27
|
+
}),
|
|
28
|
+
include: strings({
|
|
29
|
+
alias: "i",
|
|
30
|
+
desc: "File inclusion regexp",
|
|
31
|
+
hint: "EXT"
|
|
32
|
+
})
|
|
33
|
+
},
|
|
34
|
+
desc: "Convert file tree into single BlockFS blob",
|
|
35
|
+
fn: async (ctx) => {
|
|
36
|
+
const collected = collectFiles(ctx);
|
|
37
|
+
const numBlocks = align(
|
|
38
|
+
ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize, ctx.logger),
|
|
39
|
+
8
|
|
40
|
+
);
|
|
41
|
+
ctx.logger.info("number of files:", collected.files.length);
|
|
42
|
+
ctx.logger.info("number of directories:", collected.dirs.length);
|
|
43
|
+
ctx.logger.info("total file size:", collected.size);
|
|
44
|
+
ctx.logger.info("number of blocks:", numBlocks);
|
|
45
|
+
const storage = new MemoryBlockStorage({
|
|
46
|
+
numBlocks,
|
|
47
|
+
blockSize: ctx.opts.blockSize,
|
|
48
|
+
logger: ctx.logger
|
|
49
|
+
});
|
|
50
|
+
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
51
|
+
await bfs.init();
|
|
52
|
+
ctx.logger.info("root dir block:", bfs.rootDirBlockID);
|
|
53
|
+
ctx.logger.info("first data block:", bfs.dataStartBlockID);
|
|
54
|
+
ctx.logger.info("block data size:", bfs.blockDataSize);
|
|
55
|
+
for (let f of collected.files) {
|
|
56
|
+
const data = readBinary(f.src, ctx.logger);
|
|
57
|
+
ctx.logger.info("writing file:", f.dest, "size:", data.length);
|
|
58
|
+
await bfs.writeFile(f.dest, data);
|
|
59
|
+
const entry = await bfs.entryForPath(f.dest);
|
|
60
|
+
entry.ctime = f.ctime;
|
|
61
|
+
entry.mtime = f.mtime;
|
|
62
|
+
await entry.save();
|
|
63
|
+
}
|
|
64
|
+
writeFile(ctx.opts.out, storage.buffer, void 0, ctx.logger);
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
const pathsForFile = (f) => {
|
|
68
|
+
const parts = f.split("/");
|
|
69
|
+
const dirs = [];
|
|
70
|
+
for (let i = 1; i < parts.length; i++)
|
|
71
|
+
dirs.push(parts.slice(0, i).join("/"));
|
|
72
|
+
return dirs;
|
|
73
|
+
};
|
|
74
|
+
const collectFiles = ({
|
|
75
|
+
opts: { include, exclude },
|
|
76
|
+
inputs
|
|
77
|
+
}) => {
|
|
78
|
+
const root = resolve(inputs[0]);
|
|
79
|
+
const filtered = [];
|
|
80
|
+
const dirs = /* @__PURE__ */ new Set();
|
|
81
|
+
const $include = include?.map((x) => new RegExp(x));
|
|
82
|
+
const $exclude = exclude?.map((x) => new RegExp(x));
|
|
83
|
+
let total = 0;
|
|
84
|
+
for (let f of files(root)) {
|
|
85
|
+
const stats = statSync(f);
|
|
86
|
+
if ($exclude && $exclude.some((x) => x.test(f))) continue;
|
|
87
|
+
if ($include && !$include.some((x) => x.test(f))) continue;
|
|
88
|
+
const dest = relative(root, f);
|
|
89
|
+
filtered.push({
|
|
90
|
+
src: f,
|
|
91
|
+
dest,
|
|
92
|
+
size: stats.size,
|
|
93
|
+
ctime: stats.ctimeMs,
|
|
94
|
+
mtime: stats.mtimeMs
|
|
95
|
+
});
|
|
96
|
+
for (let d of pathsForFile(dest)) dirs.add(d);
|
|
97
|
+
total += stats.size;
|
|
98
|
+
}
|
|
99
|
+
return { files: filtered, dirs: [...dirs], size: total };
|
|
100
|
+
};
|
|
101
|
+
const computeBlockCount = (collected, blockSize, logger, numBlocks) => {
|
|
102
|
+
let blocks = collected.dirs.length;
|
|
103
|
+
const blockIDBytes = requiredBytes(
|
|
104
|
+
numBlocks ?? collected.size / blockSize + blocks
|
|
105
|
+
);
|
|
106
|
+
const blockDataSizeBytes = requiredBytes(blockSize);
|
|
107
|
+
const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
|
|
108
|
+
const numEntries = collected.files.length + collected.dirs.length;
|
|
109
|
+
const numEntryBlocks = Math.ceil(numEntries * Entry.SIZE / blockDataSize);
|
|
110
|
+
logger.info("num entries:", numEntries);
|
|
111
|
+
logger.info("num entry blocks:", numEntryBlocks);
|
|
112
|
+
blocks += numEntryBlocks;
|
|
113
|
+
for (let f of collected.files) {
|
|
114
|
+
const size = Math.ceil(f.size / blockDataSize);
|
|
115
|
+
logger.debug("file:", f.src, "blocks:", size);
|
|
116
|
+
blocks += size;
|
|
117
|
+
}
|
|
118
|
+
const blockIDBytes2 = requiredBytes(blocks);
|
|
119
|
+
return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, logger, blocks) : blocks;
|
|
120
|
+
};
|
|
121
|
+
const requiredBytes = (x) => align(Math.ceil(Math.log2(x)), 8) >> 3;
|
|
122
|
+
export {
|
|
123
|
+
CONVERT
|
|
124
|
+
};
|
package/cli/list.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type Command } from "@thi.ng/args";
|
|
2
|
+
import { type AppCtx, type CLIOpts } from "./api.js";
|
|
3
|
+
interface ListOpts extends CLIOpts {
|
|
4
|
+
blockSize: number;
|
|
5
|
+
all: boolean;
|
|
6
|
+
tree: boolean;
|
|
7
|
+
withMtime: boolean;
|
|
8
|
+
withSize: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare const LIST: Command<ListOpts, CLIOpts, AppCtx<ListOpts>>;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=list.d.ts.map
|
package/cli/list.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { flag } from "@thi.ng/args";
|
|
2
|
+
import { compareByKey } from "@thi.ng/compare";
|
|
3
|
+
import { readBinary } from "@thi.ng/file-io";
|
|
4
|
+
import { BlockFS } from "../fs.js";
|
|
5
|
+
import { MemoryBlockStorage } from "../storage/memory.js";
|
|
6
|
+
import { ARG_BLOCKSIZE } from "./api.js";
|
|
7
|
+
const INDENT = ["\u2502 ", " "];
|
|
8
|
+
const PREFIX = ["\u251C\u2500\u2500 ", "\u2514\u2500\u2500 "];
|
|
9
|
+
const LIST = {
|
|
10
|
+
opts: {
|
|
11
|
+
...ARG_BLOCKSIZE,
|
|
12
|
+
all: flag({
|
|
13
|
+
alias: "a",
|
|
14
|
+
desc: "Display all attribs"
|
|
15
|
+
}),
|
|
16
|
+
tree: flag({
|
|
17
|
+
alias: "t",
|
|
18
|
+
desc: "List files as tree"
|
|
19
|
+
}),
|
|
20
|
+
withMtime: flag({
|
|
21
|
+
alias: "m",
|
|
22
|
+
desc: "Display modified times"
|
|
23
|
+
}),
|
|
24
|
+
withSize: flag({
|
|
25
|
+
alias: "s",
|
|
26
|
+
desc: "Display file sizes"
|
|
27
|
+
})
|
|
28
|
+
},
|
|
29
|
+
desc: "List file tree of a BlockFS blob",
|
|
30
|
+
fn: async (ctx) => {
|
|
31
|
+
if (ctx.opts.all) {
|
|
32
|
+
ctx.opts.withMtime = ctx.opts.withSize = true;
|
|
33
|
+
}
|
|
34
|
+
const buffer = readBinary(ctx.inputs[0]);
|
|
35
|
+
const storage = new MemoryBlockStorage({
|
|
36
|
+
numBlocks: buffer.length / ctx.opts.blockSize >>> 0,
|
|
37
|
+
blockSize: ctx.opts.blockSize,
|
|
38
|
+
logger: ctx.logger,
|
|
39
|
+
buffer
|
|
40
|
+
});
|
|
41
|
+
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
42
|
+
await bfs.init();
|
|
43
|
+
const tree = [];
|
|
44
|
+
for await (let entry of bfs.root.tree(compareByKey("name"))) {
|
|
45
|
+
const path = entry.path;
|
|
46
|
+
const depth = path.split("/").length - 2;
|
|
47
|
+
tree.push([path, depth, entry]);
|
|
48
|
+
}
|
|
49
|
+
const rows = [];
|
|
50
|
+
const maxWidths = [0, 0, 0];
|
|
51
|
+
const last = [];
|
|
52
|
+
for (let i = 0, num = tree.length - 1; i <= num; i++) {
|
|
53
|
+
const [path, depth, entry] = tree[i];
|
|
54
|
+
last[depth] = ~~(i === num);
|
|
55
|
+
for (let j = i + 1; j <= num; j++) {
|
|
56
|
+
if (tree[j][1] === depth) break;
|
|
57
|
+
if (tree[j][1] < depth || j === num) {
|
|
58
|
+
last[depth] = 1;
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const row = ctx.opts.tree ? [
|
|
63
|
+
[
|
|
64
|
+
...last.slice(0, depth).map((x) => INDENT[x]),
|
|
65
|
+
PREFIX[last[depth]],
|
|
66
|
+
entry.name
|
|
67
|
+
].join("")
|
|
68
|
+
] : [path];
|
|
69
|
+
if (ctx.opts.withSize) {
|
|
70
|
+
row.push(entry.isDirectory() ? "" : String(Number(entry.size)));
|
|
71
|
+
}
|
|
72
|
+
if (ctx.opts.withMtime) {
|
|
73
|
+
row.push(new Date(entry.mtime).toISOString());
|
|
74
|
+
}
|
|
75
|
+
rows.push(row);
|
|
76
|
+
for (let i2 = 0; i2 < row.length; i2++) {
|
|
77
|
+
maxWidths[i2] = Math.max(maxWidths[i2], row[i2].length);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (let row of rows) {
|
|
81
|
+
console.log(row.map((x, i) => x.padEnd(maxWidths[i])).join(" "));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
export {
|
|
86
|
+
LIST
|
|
87
|
+
};
|
package/cli.d.ts
CHANGED
|
@@ -1,25 +1,2 @@
|
|
|
1
|
-
import { type Command, type CommandCtx } from "@thi.ng/args";
|
|
2
|
-
interface CLIOpts {
|
|
3
|
-
verbose: boolean;
|
|
4
|
-
quiet: boolean;
|
|
5
|
-
}
|
|
6
|
-
interface ConvertOpts extends CLIOpts {
|
|
7
|
-
numBlocks?: number;
|
|
8
|
-
blockSize: number;
|
|
9
|
-
exclude?: string[];
|
|
10
|
-
include?: string[];
|
|
11
|
-
out: string;
|
|
12
|
-
}
|
|
13
|
-
interface ListOpts extends CLIOpts {
|
|
14
|
-
blockSize: number;
|
|
15
|
-
all: boolean;
|
|
16
|
-
tree: boolean;
|
|
17
|
-
withMtime: boolean;
|
|
18
|
-
withSize: boolean;
|
|
19
|
-
}
|
|
20
|
-
export interface AppCtx<T extends CLIOpts> extends CommandCtx<T, CLIOpts> {
|
|
21
|
-
}
|
|
22
|
-
export declare const CONVERT: Command<ConvertOpts, CLIOpts, AppCtx<ConvertOpts>>;
|
|
23
|
-
export declare const LIST: Command<ListOpts, CLIOpts, AppCtx<ListOpts>>;
|
|
24
1
|
export {};
|
|
25
2
|
//# sourceMappingURL=cli.d.ts.map
|
package/cli.js
CHANGED
|
@@ -1,209 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
flag,
|
|
4
|
-
int,
|
|
5
|
-
string,
|
|
6
|
-
strings,
|
|
7
|
-
THING_HEADER
|
|
8
|
-
} from "@thi.ng/args";
|
|
9
|
-
import { align, isPow2 } from "@thi.ng/binary";
|
|
10
|
-
import { illegalArgs } from "@thi.ng/errors";
|
|
11
|
-
import { files, readBinary, readJSON, writeFile } from "@thi.ng/file-io";
|
|
1
|
+
import { cliApp, flag, THING_HEADER } from "@thi.ng/args";
|
|
2
|
+
import { readJSON } from "@thi.ng/file-io";
|
|
12
3
|
import { LogLevel } from "@thi.ng/logger";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import { BlockFS } from "./fs.js";
|
|
17
|
-
import { MemoryBlockStorage } from "./storage/memory.js";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { CONVERT } from "./cli/convert.js";
|
|
6
|
+
import { LIST } from "./cli/list.js";
|
|
18
7
|
const PKG = readJSON(join(process.argv[2], "package.json"));
|
|
19
|
-
const ARG_BLOCKSIZE = {
|
|
20
|
-
blockSize: int({
|
|
21
|
-
alias: "bs",
|
|
22
|
-
desc: "Block size",
|
|
23
|
-
hint: "BYTES",
|
|
24
|
-
default: 1024,
|
|
25
|
-
coerce: (x) => {
|
|
26
|
-
const size = +x;
|
|
27
|
-
if (!isPow2(size)) illegalArgs("block size must be a power of 2");
|
|
28
|
-
return size;
|
|
29
|
-
}
|
|
30
|
-
})
|
|
31
|
-
};
|
|
32
|
-
const requiredBytes = (x) => align(Math.ceil(Math.log2(x)), 8) >> 3;
|
|
33
|
-
const pathsForFile = (f) => {
|
|
34
|
-
const parts = f.split("/");
|
|
35
|
-
const dirs = [];
|
|
36
|
-
for (let i = 1; i < parts.length; i++)
|
|
37
|
-
dirs.push(parts.slice(0, i).join("/"));
|
|
38
|
-
return dirs;
|
|
39
|
-
};
|
|
40
|
-
const collectFiles = ({
|
|
41
|
-
opts: { include, exclude },
|
|
42
|
-
inputs
|
|
43
|
-
}) => {
|
|
44
|
-
const root = resolve(inputs[0]);
|
|
45
|
-
const filtered = [];
|
|
46
|
-
const dirs = /* @__PURE__ */ new Set();
|
|
47
|
-
const $include = include?.map((x) => new RegExp(x));
|
|
48
|
-
const $exclude = exclude?.map((x) => new RegExp(x));
|
|
49
|
-
let total = 0;
|
|
50
|
-
for (let f of files(root)) {
|
|
51
|
-
const stats = statSync(f);
|
|
52
|
-
if ($exclude && $exclude.some((x) => x.test(f))) continue;
|
|
53
|
-
if ($include && !$include.some((x) => x.test(f))) continue;
|
|
54
|
-
const dest = relative(root, f);
|
|
55
|
-
filtered.push({
|
|
56
|
-
src: f,
|
|
57
|
-
dest,
|
|
58
|
-
size: stats.size,
|
|
59
|
-
ctime: stats.ctimeMs,
|
|
60
|
-
mtime: stats.mtimeMs
|
|
61
|
-
});
|
|
62
|
-
for (let d of pathsForFile(dest)) dirs.add(d);
|
|
63
|
-
total += stats.size;
|
|
64
|
-
}
|
|
65
|
-
return { files: filtered, dirs: [...dirs], size: total };
|
|
66
|
-
};
|
|
67
|
-
const computeBlockCount = (collected, blockSize, logger, numBlocks) => {
|
|
68
|
-
let blocks = collected.dirs.length;
|
|
69
|
-
const blockIDBytes = requiredBytes(
|
|
70
|
-
numBlocks ?? collected.size / blockSize + blocks
|
|
71
|
-
);
|
|
72
|
-
const blockDataSizeBytes = requiredBytes(blockSize);
|
|
73
|
-
const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
|
|
74
|
-
const numEntries = collected.files.length + collected.dirs.length;
|
|
75
|
-
const numEntryBlocks = Math.ceil(numEntries * Entry.SIZE / blockDataSize);
|
|
76
|
-
logger.info("num entries:", numEntries);
|
|
77
|
-
logger.info("num entry blocks:", numEntryBlocks);
|
|
78
|
-
blocks += numEntryBlocks;
|
|
79
|
-
for (let f of collected.files) {
|
|
80
|
-
const size = Math.ceil(f.size / blockDataSize);
|
|
81
|
-
logger.debug("file:", f.src, "blocks:", size);
|
|
82
|
-
blocks += size;
|
|
83
|
-
}
|
|
84
|
-
const blockIDBytes2 = requiredBytes(blocks);
|
|
85
|
-
return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, logger, blocks) : blocks;
|
|
86
|
-
};
|
|
87
|
-
const CONVERT = {
|
|
88
|
-
opts: {
|
|
89
|
-
...ARG_BLOCKSIZE,
|
|
90
|
-
numBlocks: int({
|
|
91
|
-
alias: "n",
|
|
92
|
-
desc: "Number of blocks (multiple of 8)",
|
|
93
|
-
optional: true
|
|
94
|
-
}),
|
|
95
|
-
out: string({
|
|
96
|
-
alias: "o",
|
|
97
|
-
desc: "Output file path",
|
|
98
|
-
optional: false
|
|
99
|
-
}),
|
|
100
|
-
exclude: strings({
|
|
101
|
-
alias: "e",
|
|
102
|
-
desc: "File exclusion regexp",
|
|
103
|
-
hint: "EXT"
|
|
104
|
-
}),
|
|
105
|
-
include: strings({
|
|
106
|
-
alias: "i",
|
|
107
|
-
desc: "File inclusion regexp",
|
|
108
|
-
hint: "EXT"
|
|
109
|
-
})
|
|
110
|
-
},
|
|
111
|
-
desc: "Convert file tree into single BlockFS blob",
|
|
112
|
-
fn: async (ctx) => {
|
|
113
|
-
const collected = collectFiles(ctx);
|
|
114
|
-
const numBlocks = align(
|
|
115
|
-
ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize, ctx.logger),
|
|
116
|
-
8
|
|
117
|
-
);
|
|
118
|
-
ctx.logger.info("number of files:", collected.files.length);
|
|
119
|
-
ctx.logger.info("number of directories:", collected.dirs.length);
|
|
120
|
-
ctx.logger.info("total file size:", collected.size);
|
|
121
|
-
ctx.logger.info("number of blocks:", numBlocks);
|
|
122
|
-
const storage = new MemoryBlockStorage({
|
|
123
|
-
numBlocks,
|
|
124
|
-
blockSize: ctx.opts.blockSize,
|
|
125
|
-
logger: ctx.logger
|
|
126
|
-
});
|
|
127
|
-
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
128
|
-
await bfs.init();
|
|
129
|
-
ctx.logger.info("root dir block:", bfs.rootDirBlockID);
|
|
130
|
-
ctx.logger.info("first data block:", bfs.dataStartBlockID);
|
|
131
|
-
ctx.logger.info("block data size:", bfs.blockDataSize);
|
|
132
|
-
for (let f of collected.files) {
|
|
133
|
-
const data = readBinary(f.src, ctx.logger);
|
|
134
|
-
ctx.logger.info("writing file:", f.dest, "size:", data.length);
|
|
135
|
-
await bfs.writeFile(f.dest, data);
|
|
136
|
-
const entry = await bfs.entryForPath(f.dest);
|
|
137
|
-
entry.ctime = f.ctime;
|
|
138
|
-
entry.mtime = f.mtime;
|
|
139
|
-
await entry.save();
|
|
140
|
-
}
|
|
141
|
-
writeFile(ctx.opts.out, storage.buffer, void 0, ctx.logger);
|
|
142
|
-
}
|
|
143
|
-
};
|
|
144
|
-
const LIST = {
|
|
145
|
-
opts: {
|
|
146
|
-
...ARG_BLOCKSIZE,
|
|
147
|
-
all: flag({
|
|
148
|
-
alias: "a",
|
|
149
|
-
desc: "Display all attribs"
|
|
150
|
-
}),
|
|
151
|
-
tree: flag({
|
|
152
|
-
alias: "t",
|
|
153
|
-
desc: "List files as tree"
|
|
154
|
-
}),
|
|
155
|
-
withMtime: flag({
|
|
156
|
-
alias: "m",
|
|
157
|
-
desc: "Display modified times"
|
|
158
|
-
}),
|
|
159
|
-
withSize: flag({
|
|
160
|
-
alias: "s",
|
|
161
|
-
desc: "Display file sizes"
|
|
162
|
-
})
|
|
163
|
-
},
|
|
164
|
-
desc: "List file tree of a BlockFS blob",
|
|
165
|
-
fn: async (ctx) => {
|
|
166
|
-
if (ctx.opts.all) {
|
|
167
|
-
ctx.opts.withMtime = ctx.opts.withSize = true;
|
|
168
|
-
}
|
|
169
|
-
const buffer = readBinary(ctx.inputs[0]);
|
|
170
|
-
const storage = new MemoryBlockStorage({
|
|
171
|
-
numBlocks: buffer.length / ctx.opts.blockSize >>> 0,
|
|
172
|
-
blockSize: ctx.opts.blockSize,
|
|
173
|
-
logger: ctx.logger,
|
|
174
|
-
buffer
|
|
175
|
-
});
|
|
176
|
-
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
177
|
-
await bfs.init();
|
|
178
|
-
const tree = [];
|
|
179
|
-
for await (let entry of bfs.root.tree()) {
|
|
180
|
-
const path = entry.path;
|
|
181
|
-
const depth = path.split("/").length - 2;
|
|
182
|
-
tree.push([path, depth, entry]);
|
|
183
|
-
}
|
|
184
|
-
tree.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
185
|
-
const rows = [];
|
|
186
|
-
const maxWidths = [0, 0, 0];
|
|
187
|
-
for (let i = 0, num = tree.length - 1; i <= num; i++) {
|
|
188
|
-
const f = tree[i];
|
|
189
|
-
const isLast = i === num || f[1] > tree[i + 1][1];
|
|
190
|
-
const row = ctx.opts.tree ? ["\u2502 ".repeat(f[1]) + (isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ") + f[2].name] : [f[0]];
|
|
191
|
-
if (ctx.opts.withSize) {
|
|
192
|
-
row.push(f[2].isDirectory() ? "" : String(Number(f[2].size)));
|
|
193
|
-
}
|
|
194
|
-
if (ctx.opts.withMtime) {
|
|
195
|
-
row.push(new Date(f[2].mtime).toISOString());
|
|
196
|
-
}
|
|
197
|
-
rows.push(row);
|
|
198
|
-
for (let i2 = 0; i2 < row.length; i2++) {
|
|
199
|
-
maxWidths[i2] = Math.max(maxWidths[i2], row[i2].length);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
for (let row of rows) {
|
|
203
|
-
console.log(row.map((x, i) => x.padEnd(maxWidths[i])).join(" "));
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
};
|
|
207
8
|
cliApp({
|
|
208
9
|
name: "blockfs",
|
|
209
10
|
start: 3,
|
|
@@ -240,7 +41,3 @@ Usage: blockfs <cmd> [opts] input [...]
|
|
|
240
41
|
paramWidth: 32
|
|
241
42
|
}
|
|
242
43
|
});
|
|
243
|
-
export {
|
|
244
|
-
CONVERT,
|
|
245
|
-
LIST
|
|
246
|
-
};
|
package/directory.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Comparator } from "@thi.ng/api";
|
|
1
2
|
import { type EntrySpec, type IDirectory, type IEntry } from "./api.js";
|
|
2
3
|
import type { BlockFS } from "./fs.js";
|
|
3
4
|
export declare class Directory implements IDirectory {
|
|
@@ -5,7 +6,7 @@ export declare class Directory implements IDirectory {
|
|
|
5
6
|
entry: IEntry;
|
|
6
7
|
constructor(fs: BlockFS, entry: IEntry);
|
|
7
8
|
[Symbol.asyncIterator](): AsyncIterableIterator<IEntry>;
|
|
8
|
-
tree(): AsyncIterableIterator<IEntry>;
|
|
9
|
+
tree(cmp?: Comparator<IEntry>): AsyncIterableIterator<IEntry>;
|
|
9
10
|
traverse(): Promise<{
|
|
10
11
|
blocks: number[];
|
|
11
12
|
entries: IEntry[];
|
package/directory.js
CHANGED
|
@@ -23,10 +23,22 @@ class Directory {
|
|
|
23
23
|
blockID = next;
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
-
async *tree() {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
26
|
+
async *tree(cmp) {
|
|
27
|
+
if (cmp) {
|
|
28
|
+
const entries = [];
|
|
29
|
+
for await (let entry of this) {
|
|
30
|
+
entries.push(entry);
|
|
31
|
+
}
|
|
32
|
+
entries.sort(cmp);
|
|
33
|
+
for (let entry of entries) {
|
|
34
|
+
yield entry;
|
|
35
|
+
if (entry.isDirectory()) yield* entry.directory.tree(cmp);
|
|
36
|
+
}
|
|
37
|
+
} else {
|
|
38
|
+
for await (let entry of this) {
|
|
39
|
+
yield entry;
|
|
40
|
+
if (entry.isDirectory()) yield* entry.directory.tree();
|
|
41
|
+
}
|
|
30
42
|
}
|
|
31
43
|
}
|
|
32
44
|
async traverse() {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thi.ng/block-fs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Customizable block-based storage, adapters & file system layer",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./index.js",
|
|
@@ -43,10 +43,11 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@thi.ng/api": "^8.11.30",
|
|
46
|
-
"@thi.ng/args": "^2.7.
|
|
46
|
+
"@thi.ng/args": "^2.7.2",
|
|
47
47
|
"@thi.ng/binary": "^3.4.53",
|
|
48
48
|
"@thi.ng/bitfield": "^2.4.6",
|
|
49
49
|
"@thi.ng/checks": "^3.7.10",
|
|
50
|
+
"@thi.ng/compare": "^2.4.22",
|
|
50
51
|
"@thi.ng/errors": "^2.5.36",
|
|
51
52
|
"@thi.ng/file-io": "^2.2.1",
|
|
52
53
|
"@thi.ng/logger": "^3.1.11",
|
|
@@ -91,6 +92,7 @@
|
|
|
91
92
|
"./*.js",
|
|
92
93
|
"./*.d.ts",
|
|
93
94
|
"bin",
|
|
95
|
+
"cli",
|
|
94
96
|
"storage"
|
|
95
97
|
],
|
|
96
98
|
"exports": {
|
|
@@ -100,6 +102,15 @@
|
|
|
100
102
|
"./api": {
|
|
101
103
|
"default": "./api.js"
|
|
102
104
|
},
|
|
105
|
+
"./cli/api": {
|
|
106
|
+
"default": "./cli/api.js"
|
|
107
|
+
},
|
|
108
|
+
"./cli/convert": {
|
|
109
|
+
"default": "./cli/convert.js"
|
|
110
|
+
},
|
|
111
|
+
"./cli/list": {
|
|
112
|
+
"default": "./cli/list.js"
|
|
113
|
+
},
|
|
103
114
|
"./cli": {
|
|
104
115
|
"default": "./cli.js"
|
|
105
116
|
},
|
|
@@ -132,5 +143,5 @@
|
|
|
132
143
|
"status": "alpha",
|
|
133
144
|
"year": 2024
|
|
134
145
|
},
|
|
135
|
-
"gitHead": "
|
|
146
|
+
"gitHead": "4d75e2d11a3d1a3c3769ca00338e32a820c69d2c\n"
|
|
136
147
|
}
|