@thi.ng/block-fs 0.5.10 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -1
- package/README.md +1 -1
- package/api.d.ts +2 -2
- package/cli/api.d.ts +21 -0
- package/cli/api.js +19 -0
- package/cli/convert.d.ts +11 -0
- package/cli/convert.js +124 -0
- package/cli/list.d.ts +12 -0
- package/cli/list.js +87 -0
- package/cli.d.ts +0 -23
- package/cli.js +5 -208
- package/directory.d.ts +2 -1
- package/directory.js +16 -4
- package/package.json +13 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
-
- **Last updated**: 2025-07-
|
|
3
|
+
- **Last updated**: 2025-07-11T22:07:07Z
|
|
4
4
|
- **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
|
|
5
5
|
|
|
6
6
|
All notable changes to this project will be documented in this file.
|
|
@@ -11,6 +11,16 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
|
|
|
11
11
|
**Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
|
|
12
12
|
and/or version bumps of transitive dependencies.
|
|
13
13
|
|
|
14
|
+
## [0.6.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.6.0) (2025-07-11)
|
|
15
|
+
|
|
16
|
+
#### 🚀 Features
|
|
17
|
+
|
|
18
|
+
- update Directory.tree(), add optional comparator ([a3c7f4c](https://github.com/thi-ng/umbrella/commit/a3c7f4c))
|
|
19
|
+
|
|
20
|
+
#### ♻️ Refactoring
|
|
21
|
+
|
|
22
|
+
- split out CLI cmds to own files, fix tree display ([8665244](https://github.com/thi-ng/umbrella/commit/8665244))
|
|
23
|
+
|
|
14
24
|
### [0.5.3](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.5.3) (2025-06-27)
|
|
15
25
|
|
|
16
26
|
#### ♻️ Refactoring
|
package/README.md
CHANGED
package/api.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Maybe } from "@thi.ng/api";
|
|
1
|
+
import type { Comparator, Maybe } from "@thi.ng/api";
|
|
2
2
|
import type { Pow2 } from "@thi.ng/binary";
|
|
3
3
|
import type { ILogger } from "@thi.ng/logger";
|
|
4
4
|
import type { BlockFS } from "./fs.js";
|
|
@@ -66,7 +66,7 @@ export interface IEntry extends Required<EntrySpec> {
|
|
|
66
66
|
export interface IDirectory extends AsyncIterable<IEntry> {
|
|
67
67
|
readonly fs: BlockFS;
|
|
68
68
|
readonly entry: IEntry;
|
|
69
|
-
tree(): AsyncIterableIterator<IEntry>;
|
|
69
|
+
tree(cmp?: Comparator<IEntry>): AsyncIterableIterator<IEntry>;
|
|
70
70
|
traverse(): Promise<{
|
|
71
71
|
blocks: number[];
|
|
72
72
|
entries: IEntry[];
|
package/cli/api.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type CommandCtx } from "@thi.ng/args";
|
|
2
|
+
export interface CLIOpts {
|
|
3
|
+
verbose: boolean;
|
|
4
|
+
quiet: boolean;
|
|
5
|
+
}
|
|
6
|
+
export declare const ARG_BLOCKSIZE: {
|
|
7
|
+
blockSize: {
|
|
8
|
+
alias: string;
|
|
9
|
+
desc: string;
|
|
10
|
+
hint: string;
|
|
11
|
+
default: number;
|
|
12
|
+
coerce: (x: string) => import("@thi.ng/binary").Pow2;
|
|
13
|
+
} & {
|
|
14
|
+
coerce: import("@thi.ng/api").Fn<string, number>;
|
|
15
|
+
hint: string;
|
|
16
|
+
group: string;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
export interface AppCtx<T extends CLIOpts> extends CommandCtx<T, CLIOpts> {
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=api.d.ts.map
|
package/cli/api.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { int } from "@thi.ng/args";
|
|
2
|
+
import { isPow2 } from "@thi.ng/binary";
|
|
3
|
+
import { illegalArgs } from "@thi.ng/errors";
|
|
4
|
+
const ARG_BLOCKSIZE = {
|
|
5
|
+
blockSize: int({
|
|
6
|
+
alias: "bs",
|
|
7
|
+
desc: "Block size",
|
|
8
|
+
hint: "BYTES",
|
|
9
|
+
default: 1024,
|
|
10
|
+
coerce: (x) => {
|
|
11
|
+
const size = +x;
|
|
12
|
+
if (!isPow2(size)) illegalArgs("block size must be a power of 2");
|
|
13
|
+
return size;
|
|
14
|
+
}
|
|
15
|
+
})
|
|
16
|
+
};
|
|
17
|
+
export {
|
|
18
|
+
ARG_BLOCKSIZE
|
|
19
|
+
};
|
package/cli/convert.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type Command } from "@thi.ng/args";
|
|
2
|
+
import { type AppCtx, type CLIOpts } from "./api.js";
|
|
3
|
+
export interface ConvertOpts extends CLIOpts {
|
|
4
|
+
numBlocks?: number;
|
|
5
|
+
blockSize: number;
|
|
6
|
+
exclude?: string[];
|
|
7
|
+
include?: string[];
|
|
8
|
+
out: string;
|
|
9
|
+
}
|
|
10
|
+
export declare const CONVERT: Command<ConvertOpts, CLIOpts, AppCtx<ConvertOpts>>;
|
|
11
|
+
//# sourceMappingURL=convert.d.ts.map
|
package/cli/convert.js
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { int, string, strings } from "@thi.ng/args";
|
|
2
|
+
import { align } from "@thi.ng/binary";
|
|
3
|
+
import { files, readBinary, writeFile } from "@thi.ng/file-io";
|
|
4
|
+
import { statSync } from "node:fs";
|
|
5
|
+
import { relative, resolve } from "node:path";
|
|
6
|
+
import { Entry } from "../entry.js";
|
|
7
|
+
import { BlockFS } from "../fs.js";
|
|
8
|
+
import { MemoryBlockStorage } from "../storage/memory.js";
|
|
9
|
+
import { ARG_BLOCKSIZE } from "./api.js";
|
|
10
|
+
const CONVERT = {
|
|
11
|
+
opts: {
|
|
12
|
+
...ARG_BLOCKSIZE,
|
|
13
|
+
numBlocks: int({
|
|
14
|
+
alias: "n",
|
|
15
|
+
desc: "Number of blocks (multiple of 8)",
|
|
16
|
+
optional: true
|
|
17
|
+
}),
|
|
18
|
+
out: string({
|
|
19
|
+
alias: "o",
|
|
20
|
+
desc: "Output file path",
|
|
21
|
+
optional: false
|
|
22
|
+
}),
|
|
23
|
+
exclude: strings({
|
|
24
|
+
alias: "e",
|
|
25
|
+
desc: "File exclusion regexp",
|
|
26
|
+
hint: "EXT"
|
|
27
|
+
}),
|
|
28
|
+
include: strings({
|
|
29
|
+
alias: "i",
|
|
30
|
+
desc: "File inclusion regexp",
|
|
31
|
+
hint: "EXT"
|
|
32
|
+
})
|
|
33
|
+
},
|
|
34
|
+
desc: "Convert file tree into single BlockFS blob",
|
|
35
|
+
fn: async (ctx) => {
|
|
36
|
+
const collected = collectFiles(ctx);
|
|
37
|
+
const numBlocks = align(
|
|
38
|
+
ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize, ctx.logger),
|
|
39
|
+
8
|
|
40
|
+
);
|
|
41
|
+
ctx.logger.info("number of files:", collected.files.length);
|
|
42
|
+
ctx.logger.info("number of directories:", collected.dirs.length);
|
|
43
|
+
ctx.logger.info("total file size:", collected.size);
|
|
44
|
+
ctx.logger.info("number of blocks:", numBlocks);
|
|
45
|
+
const storage = new MemoryBlockStorage({
|
|
46
|
+
numBlocks,
|
|
47
|
+
blockSize: ctx.opts.blockSize,
|
|
48
|
+
logger: ctx.logger
|
|
49
|
+
});
|
|
50
|
+
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
51
|
+
await bfs.init();
|
|
52
|
+
ctx.logger.info("root dir block:", bfs.rootDirBlockID);
|
|
53
|
+
ctx.logger.info("first data block:", bfs.dataStartBlockID);
|
|
54
|
+
ctx.logger.info("block data size:", bfs.blockDataSize);
|
|
55
|
+
for (let f of collected.files) {
|
|
56
|
+
const data = readBinary(f.src, ctx.logger);
|
|
57
|
+
ctx.logger.info("writing file:", f.dest, "size:", data.length);
|
|
58
|
+
await bfs.writeFile(f.dest, data);
|
|
59
|
+
const entry = await bfs.entryForPath(f.dest);
|
|
60
|
+
entry.ctime = f.ctime;
|
|
61
|
+
entry.mtime = f.mtime;
|
|
62
|
+
await entry.save();
|
|
63
|
+
}
|
|
64
|
+
writeFile(ctx.opts.out, storage.buffer, void 0, ctx.logger);
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
const pathsForFile = (f) => {
|
|
68
|
+
const parts = f.split("/");
|
|
69
|
+
const dirs = [];
|
|
70
|
+
for (let i = 1; i < parts.length; i++)
|
|
71
|
+
dirs.push(parts.slice(0, i).join("/"));
|
|
72
|
+
return dirs;
|
|
73
|
+
};
|
|
74
|
+
const collectFiles = ({
|
|
75
|
+
opts: { include, exclude },
|
|
76
|
+
inputs
|
|
77
|
+
}) => {
|
|
78
|
+
const root = resolve(inputs[0]);
|
|
79
|
+
const filtered = [];
|
|
80
|
+
const dirs = /* @__PURE__ */ new Set();
|
|
81
|
+
const $include = include?.map((x) => new RegExp(x));
|
|
82
|
+
const $exclude = exclude?.map((x) => new RegExp(x));
|
|
83
|
+
let total = 0;
|
|
84
|
+
for (let f of files(root)) {
|
|
85
|
+
const stats = statSync(f);
|
|
86
|
+
if ($exclude && $exclude.some((x) => x.test(f))) continue;
|
|
87
|
+
if ($include && !$include.some((x) => x.test(f))) continue;
|
|
88
|
+
const dest = relative(root, f);
|
|
89
|
+
filtered.push({
|
|
90
|
+
src: f,
|
|
91
|
+
dest,
|
|
92
|
+
size: stats.size,
|
|
93
|
+
ctime: stats.ctimeMs,
|
|
94
|
+
mtime: stats.mtimeMs
|
|
95
|
+
});
|
|
96
|
+
for (let d of pathsForFile(dest)) dirs.add(d);
|
|
97
|
+
total += stats.size;
|
|
98
|
+
}
|
|
99
|
+
return { files: filtered, dirs: [...dirs], size: total };
|
|
100
|
+
};
|
|
101
|
+
const computeBlockCount = (collected, blockSize, logger, numBlocks) => {
|
|
102
|
+
let blocks = collected.dirs.length;
|
|
103
|
+
const blockIDBytes = requiredBytes(
|
|
104
|
+
numBlocks ?? collected.size / blockSize + blocks
|
|
105
|
+
);
|
|
106
|
+
const blockDataSizeBytes = requiredBytes(blockSize);
|
|
107
|
+
const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
|
|
108
|
+
const numEntries = collected.files.length + collected.dirs.length;
|
|
109
|
+
const numEntryBlocks = Math.ceil(numEntries * Entry.SIZE / blockDataSize);
|
|
110
|
+
logger.info("num entries:", numEntries);
|
|
111
|
+
logger.info("num entry blocks:", numEntryBlocks);
|
|
112
|
+
blocks += numEntryBlocks;
|
|
113
|
+
for (let f of collected.files) {
|
|
114
|
+
const size = Math.ceil(f.size / blockDataSize);
|
|
115
|
+
logger.debug("file:", f.src, "blocks:", size);
|
|
116
|
+
blocks += size;
|
|
117
|
+
}
|
|
118
|
+
const blockIDBytes2 = requiredBytes(blocks);
|
|
119
|
+
return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, logger, blocks) : blocks;
|
|
120
|
+
};
|
|
121
|
+
const requiredBytes = (x) => align(Math.ceil(Math.log2(x)), 8) >> 3;
|
|
122
|
+
export {
|
|
123
|
+
CONVERT
|
|
124
|
+
};
|
package/cli/list.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type Command } from "@thi.ng/args";
|
|
2
|
+
import { type AppCtx, type CLIOpts } from "./api.js";
|
|
3
|
+
interface ListOpts extends CLIOpts {
|
|
4
|
+
blockSize: number;
|
|
5
|
+
all: boolean;
|
|
6
|
+
tree: boolean;
|
|
7
|
+
withMtime: boolean;
|
|
8
|
+
withSize: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare const LIST: Command<ListOpts, CLIOpts, AppCtx<ListOpts>>;
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=list.d.ts.map
|
package/cli/list.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { flag } from "@thi.ng/args";
|
|
2
|
+
import { readBinary } from "@thi.ng/file-io";
|
|
3
|
+
import { BlockFS } from "../fs.js";
|
|
4
|
+
import { MemoryBlockStorage } from "../storage/memory.js";
|
|
5
|
+
import { ARG_BLOCKSIZE } from "./api.js";
|
|
6
|
+
const INDENT = ["\u2502 ", " "];
|
|
7
|
+
const PREFIX = ["\u251C\u2500\u2500 ", "\u2514\u2500\u2500 "];
|
|
8
|
+
const LIST = {
|
|
9
|
+
opts: {
|
|
10
|
+
...ARG_BLOCKSIZE,
|
|
11
|
+
all: flag({
|
|
12
|
+
alias: "a",
|
|
13
|
+
desc: "Display all attribs"
|
|
14
|
+
}),
|
|
15
|
+
tree: flag({
|
|
16
|
+
alias: "t",
|
|
17
|
+
desc: "List files as tree"
|
|
18
|
+
}),
|
|
19
|
+
withMtime: flag({
|
|
20
|
+
alias: "m",
|
|
21
|
+
desc: "Display modified times"
|
|
22
|
+
}),
|
|
23
|
+
withSize: flag({
|
|
24
|
+
alias: "s",
|
|
25
|
+
desc: "Display file sizes"
|
|
26
|
+
})
|
|
27
|
+
},
|
|
28
|
+
desc: "List file tree of a BlockFS blob",
|
|
29
|
+
fn: async (ctx) => {
|
|
30
|
+
if (ctx.opts.all) {
|
|
31
|
+
ctx.opts.withMtime = ctx.opts.withSize = true;
|
|
32
|
+
}
|
|
33
|
+
const buffer = readBinary(ctx.inputs[0]);
|
|
34
|
+
const storage = new MemoryBlockStorage({
|
|
35
|
+
numBlocks: buffer.length / ctx.opts.blockSize >>> 0,
|
|
36
|
+
blockSize: ctx.opts.blockSize,
|
|
37
|
+
logger: ctx.logger,
|
|
38
|
+
buffer
|
|
39
|
+
});
|
|
40
|
+
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
41
|
+
await bfs.init();
|
|
42
|
+
const tree = [];
|
|
43
|
+
for await (let entry of bfs.root.tree()) {
|
|
44
|
+
const path = entry.path;
|
|
45
|
+
const depth = path.split("/").length - 2;
|
|
46
|
+
tree.push([path, depth, entry]);
|
|
47
|
+
}
|
|
48
|
+
tree.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
49
|
+
const rows = [];
|
|
50
|
+
const maxWidths = [0, 0, 0];
|
|
51
|
+
const last = [];
|
|
52
|
+
for (let i = 0, num = tree.length - 1; i <= num; i++) {
|
|
53
|
+
const [path, depth, entry] = tree[i];
|
|
54
|
+
last[depth] = ~~(i === num);
|
|
55
|
+
for (let j = i + 1; j <= num; j++) {
|
|
56
|
+
if (tree[j][1] === depth) break;
|
|
57
|
+
if (tree[j][1] < depth || j === num) {
|
|
58
|
+
last[depth] = 1;
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const row = ctx.opts.tree ? [
|
|
63
|
+
[
|
|
64
|
+
...last.slice(0, depth).map((x) => INDENT[x]),
|
|
65
|
+
PREFIX[last[depth]],
|
|
66
|
+
entry.name
|
|
67
|
+
].join("")
|
|
68
|
+
] : [path];
|
|
69
|
+
if (ctx.opts.withSize) {
|
|
70
|
+
row.push(entry.isDirectory() ? "" : String(Number(entry.size)));
|
|
71
|
+
}
|
|
72
|
+
if (ctx.opts.withMtime) {
|
|
73
|
+
row.push(new Date(entry.mtime).toISOString());
|
|
74
|
+
}
|
|
75
|
+
rows.push(row);
|
|
76
|
+
for (let i2 = 0; i2 < row.length; i2++) {
|
|
77
|
+
maxWidths[i2] = Math.max(maxWidths[i2], row[i2].length);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (let row of rows) {
|
|
81
|
+
console.log(row.map((x, i) => x.padEnd(maxWidths[i])).join(" "));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
export {
|
|
86
|
+
LIST
|
|
87
|
+
};
|
package/cli.d.ts
CHANGED
|
@@ -1,25 +1,2 @@
|
|
|
1
|
-
import { type Command, type CommandCtx } from "@thi.ng/args";
|
|
2
|
-
interface CLIOpts {
|
|
3
|
-
verbose: boolean;
|
|
4
|
-
quiet: boolean;
|
|
5
|
-
}
|
|
6
|
-
interface ConvertOpts extends CLIOpts {
|
|
7
|
-
numBlocks?: number;
|
|
8
|
-
blockSize: number;
|
|
9
|
-
exclude?: string[];
|
|
10
|
-
include?: string[];
|
|
11
|
-
out: string;
|
|
12
|
-
}
|
|
13
|
-
interface ListOpts extends CLIOpts {
|
|
14
|
-
blockSize: number;
|
|
15
|
-
all: boolean;
|
|
16
|
-
tree: boolean;
|
|
17
|
-
withMtime: boolean;
|
|
18
|
-
withSize: boolean;
|
|
19
|
-
}
|
|
20
|
-
export interface AppCtx<T extends CLIOpts> extends CommandCtx<T, CLIOpts> {
|
|
21
|
-
}
|
|
22
|
-
export declare const CONVERT: Command<ConvertOpts, CLIOpts, AppCtx<ConvertOpts>>;
|
|
23
|
-
export declare const LIST: Command<ListOpts, CLIOpts, AppCtx<ListOpts>>;
|
|
24
1
|
export {};
|
|
25
2
|
//# sourceMappingURL=cli.d.ts.map
|
package/cli.js
CHANGED
|
@@ -1,209 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
flag,
|
|
4
|
-
int,
|
|
5
|
-
string,
|
|
6
|
-
strings,
|
|
7
|
-
THING_HEADER
|
|
8
|
-
} from "@thi.ng/args";
|
|
9
|
-
import { align, isPow2 } from "@thi.ng/binary";
|
|
10
|
-
import { illegalArgs } from "@thi.ng/errors";
|
|
11
|
-
import { files, readBinary, readJSON, writeFile } from "@thi.ng/file-io";
|
|
1
|
+
import { cliApp, flag, THING_HEADER } from "@thi.ng/args";
|
|
2
|
+
import { readJSON } from "@thi.ng/file-io";
|
|
12
3
|
import { LogLevel } from "@thi.ng/logger";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import { BlockFS } from "./fs.js";
|
|
17
|
-
import { MemoryBlockStorage } from "./storage/memory.js";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { CONVERT } from "./cli/convert.js";
|
|
6
|
+
import { LIST } from "./cli/list.js";
|
|
18
7
|
const PKG = readJSON(join(process.argv[2], "package.json"));
|
|
19
|
-
const ARG_BLOCKSIZE = {
|
|
20
|
-
blockSize: int({
|
|
21
|
-
alias: "bs",
|
|
22
|
-
desc: "Block size",
|
|
23
|
-
hint: "BYTES",
|
|
24
|
-
default: 1024,
|
|
25
|
-
coerce: (x) => {
|
|
26
|
-
const size = +x;
|
|
27
|
-
if (!isPow2(size)) illegalArgs("block size must be a power of 2");
|
|
28
|
-
return size;
|
|
29
|
-
}
|
|
30
|
-
})
|
|
31
|
-
};
|
|
32
|
-
const requiredBytes = (x) => align(Math.ceil(Math.log2(x)), 8) >> 3;
|
|
33
|
-
const pathsForFile = (f) => {
|
|
34
|
-
const parts = f.split("/");
|
|
35
|
-
const dirs = [];
|
|
36
|
-
for (let i = 1; i < parts.length; i++)
|
|
37
|
-
dirs.push(parts.slice(0, i).join("/"));
|
|
38
|
-
return dirs;
|
|
39
|
-
};
|
|
40
|
-
const collectFiles = ({
|
|
41
|
-
opts: { include, exclude },
|
|
42
|
-
inputs
|
|
43
|
-
}) => {
|
|
44
|
-
const root = resolve(inputs[0]);
|
|
45
|
-
const filtered = [];
|
|
46
|
-
const dirs = /* @__PURE__ */ new Set();
|
|
47
|
-
const $include = include?.map((x) => new RegExp(x));
|
|
48
|
-
const $exclude = exclude?.map((x) => new RegExp(x));
|
|
49
|
-
let total = 0;
|
|
50
|
-
for (let f of files(root)) {
|
|
51
|
-
const stats = statSync(f);
|
|
52
|
-
if ($exclude && $exclude.some((x) => x.test(f))) continue;
|
|
53
|
-
if ($include && !$include.some((x) => x.test(f))) continue;
|
|
54
|
-
const dest = relative(root, f);
|
|
55
|
-
filtered.push({
|
|
56
|
-
src: f,
|
|
57
|
-
dest,
|
|
58
|
-
size: stats.size,
|
|
59
|
-
ctime: stats.ctimeMs,
|
|
60
|
-
mtime: stats.mtimeMs
|
|
61
|
-
});
|
|
62
|
-
for (let d of pathsForFile(dest)) dirs.add(d);
|
|
63
|
-
total += stats.size;
|
|
64
|
-
}
|
|
65
|
-
return { files: filtered, dirs: [...dirs], size: total };
|
|
66
|
-
};
|
|
67
|
-
const computeBlockCount = (collected, blockSize, logger, numBlocks) => {
|
|
68
|
-
let blocks = collected.dirs.length;
|
|
69
|
-
const blockIDBytes = requiredBytes(
|
|
70
|
-
numBlocks ?? collected.size / blockSize + blocks
|
|
71
|
-
);
|
|
72
|
-
const blockDataSizeBytes = requiredBytes(blockSize);
|
|
73
|
-
const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
|
|
74
|
-
const numEntries = collected.files.length + collected.dirs.length;
|
|
75
|
-
const numEntryBlocks = Math.ceil(numEntries * Entry.SIZE / blockDataSize);
|
|
76
|
-
logger.info("num entries:", numEntries);
|
|
77
|
-
logger.info("num entry blocks:", numEntryBlocks);
|
|
78
|
-
blocks += numEntryBlocks;
|
|
79
|
-
for (let f of collected.files) {
|
|
80
|
-
const size = Math.ceil(f.size / blockDataSize);
|
|
81
|
-
logger.debug("file:", f.src, "blocks:", size);
|
|
82
|
-
blocks += size;
|
|
83
|
-
}
|
|
84
|
-
const blockIDBytes2 = requiredBytes(blocks);
|
|
85
|
-
return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, logger, blocks) : blocks;
|
|
86
|
-
};
|
|
87
|
-
const CONVERT = {
|
|
88
|
-
opts: {
|
|
89
|
-
...ARG_BLOCKSIZE,
|
|
90
|
-
numBlocks: int({
|
|
91
|
-
alias: "n",
|
|
92
|
-
desc: "Number of blocks (multiple of 8)",
|
|
93
|
-
optional: true
|
|
94
|
-
}),
|
|
95
|
-
out: string({
|
|
96
|
-
alias: "o",
|
|
97
|
-
desc: "Output file path",
|
|
98
|
-
optional: false
|
|
99
|
-
}),
|
|
100
|
-
exclude: strings({
|
|
101
|
-
alias: "e",
|
|
102
|
-
desc: "File exclusion regexp",
|
|
103
|
-
hint: "EXT"
|
|
104
|
-
}),
|
|
105
|
-
include: strings({
|
|
106
|
-
alias: "i",
|
|
107
|
-
desc: "File inclusion regexp",
|
|
108
|
-
hint: "EXT"
|
|
109
|
-
})
|
|
110
|
-
},
|
|
111
|
-
desc: "Convert file tree into single BlockFS blob",
|
|
112
|
-
fn: async (ctx) => {
|
|
113
|
-
const collected = collectFiles(ctx);
|
|
114
|
-
const numBlocks = align(
|
|
115
|
-
ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize, ctx.logger),
|
|
116
|
-
8
|
|
117
|
-
);
|
|
118
|
-
ctx.logger.info("number of files:", collected.files.length);
|
|
119
|
-
ctx.logger.info("number of directories:", collected.dirs.length);
|
|
120
|
-
ctx.logger.info("total file size:", collected.size);
|
|
121
|
-
ctx.logger.info("number of blocks:", numBlocks);
|
|
122
|
-
const storage = new MemoryBlockStorage({
|
|
123
|
-
numBlocks,
|
|
124
|
-
blockSize: ctx.opts.blockSize,
|
|
125
|
-
logger: ctx.logger
|
|
126
|
-
});
|
|
127
|
-
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
128
|
-
await bfs.init();
|
|
129
|
-
ctx.logger.info("root dir block:", bfs.rootDirBlockID);
|
|
130
|
-
ctx.logger.info("first data block:", bfs.dataStartBlockID);
|
|
131
|
-
ctx.logger.info("block data size:", bfs.blockDataSize);
|
|
132
|
-
for (let f of collected.files) {
|
|
133
|
-
const data = readBinary(f.src, ctx.logger);
|
|
134
|
-
ctx.logger.info("writing file:", f.dest, "size:", data.length);
|
|
135
|
-
await bfs.writeFile(f.dest, data);
|
|
136
|
-
const entry = await bfs.entryForPath(f.dest);
|
|
137
|
-
entry.ctime = f.ctime;
|
|
138
|
-
entry.mtime = f.mtime;
|
|
139
|
-
await entry.save();
|
|
140
|
-
}
|
|
141
|
-
writeFile(ctx.opts.out, storage.buffer, void 0, ctx.logger);
|
|
142
|
-
}
|
|
143
|
-
};
|
|
144
|
-
const LIST = {
|
|
145
|
-
opts: {
|
|
146
|
-
...ARG_BLOCKSIZE,
|
|
147
|
-
all: flag({
|
|
148
|
-
alias: "a",
|
|
149
|
-
desc: "Display all attribs"
|
|
150
|
-
}),
|
|
151
|
-
tree: flag({
|
|
152
|
-
alias: "t",
|
|
153
|
-
desc: "List files as tree"
|
|
154
|
-
}),
|
|
155
|
-
withMtime: flag({
|
|
156
|
-
alias: "m",
|
|
157
|
-
desc: "Display modified times"
|
|
158
|
-
}),
|
|
159
|
-
withSize: flag({
|
|
160
|
-
alias: "s",
|
|
161
|
-
desc: "Display file sizes"
|
|
162
|
-
})
|
|
163
|
-
},
|
|
164
|
-
desc: "List file tree of a BlockFS blob",
|
|
165
|
-
fn: async (ctx) => {
|
|
166
|
-
if (ctx.opts.all) {
|
|
167
|
-
ctx.opts.withMtime = ctx.opts.withSize = true;
|
|
168
|
-
}
|
|
169
|
-
const buffer = readBinary(ctx.inputs[0]);
|
|
170
|
-
const storage = new MemoryBlockStorage({
|
|
171
|
-
numBlocks: buffer.length / ctx.opts.blockSize >>> 0,
|
|
172
|
-
blockSize: ctx.opts.blockSize,
|
|
173
|
-
logger: ctx.logger,
|
|
174
|
-
buffer
|
|
175
|
-
});
|
|
176
|
-
const bfs = new BlockFS(storage, { logger: ctx.logger });
|
|
177
|
-
await bfs.init();
|
|
178
|
-
const tree = [];
|
|
179
|
-
for await (let entry of bfs.root.tree()) {
|
|
180
|
-
const path = entry.path;
|
|
181
|
-
const depth = path.split("/").length - 2;
|
|
182
|
-
tree.push([path, depth, entry]);
|
|
183
|
-
}
|
|
184
|
-
tree.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
185
|
-
const rows = [];
|
|
186
|
-
const maxWidths = [0, 0, 0];
|
|
187
|
-
for (let i = 0, num = tree.length - 1; i <= num; i++) {
|
|
188
|
-
const f = tree[i];
|
|
189
|
-
const isLast = i === num || f[1] > tree[i + 1][1];
|
|
190
|
-
const row = ctx.opts.tree ? ["\u2502 ".repeat(f[1]) + (isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ") + f[2].name] : [f[0]];
|
|
191
|
-
if (ctx.opts.withSize) {
|
|
192
|
-
row.push(f[2].isDirectory() ? "" : String(Number(f[2].size)));
|
|
193
|
-
}
|
|
194
|
-
if (ctx.opts.withMtime) {
|
|
195
|
-
row.push(new Date(f[2].mtime).toISOString());
|
|
196
|
-
}
|
|
197
|
-
rows.push(row);
|
|
198
|
-
for (let i2 = 0; i2 < row.length; i2++) {
|
|
199
|
-
maxWidths[i2] = Math.max(maxWidths[i2], row[i2].length);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
for (let row of rows) {
|
|
203
|
-
console.log(row.map((x, i) => x.padEnd(maxWidths[i])).join(" "));
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
};
|
|
207
8
|
cliApp({
|
|
208
9
|
name: "blockfs",
|
|
209
10
|
start: 3,
|
|
@@ -240,7 +41,3 @@ Usage: blockfs <cmd> [opts] input [...]
|
|
|
240
41
|
paramWidth: 32
|
|
241
42
|
}
|
|
242
43
|
});
|
|
243
|
-
export {
|
|
244
|
-
CONVERT,
|
|
245
|
-
LIST
|
|
246
|
-
};
|
package/directory.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Comparator } from "@thi.ng/api";
|
|
1
2
|
import { type EntrySpec, type IDirectory, type IEntry } from "./api.js";
|
|
2
3
|
import type { BlockFS } from "./fs.js";
|
|
3
4
|
export declare class Directory implements IDirectory {
|
|
@@ -5,7 +6,7 @@ export declare class Directory implements IDirectory {
|
|
|
5
6
|
entry: IEntry;
|
|
6
7
|
constructor(fs: BlockFS, entry: IEntry);
|
|
7
8
|
[Symbol.asyncIterator](): AsyncIterableIterator<IEntry>;
|
|
8
|
-
tree(): AsyncIterableIterator<IEntry>;
|
|
9
|
+
tree(cmp?: Comparator<IEntry>): AsyncIterableIterator<IEntry>;
|
|
9
10
|
traverse(): Promise<{
|
|
10
11
|
blocks: number[];
|
|
11
12
|
entries: IEntry[];
|
package/directory.js
CHANGED
|
@@ -23,10 +23,22 @@ class Directory {
|
|
|
23
23
|
blockID = next;
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
-
async *tree() {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
26
|
+
async *tree(cmp) {
|
|
27
|
+
if (cmp) {
|
|
28
|
+
const entries = [];
|
|
29
|
+
for await (let entry of this) {
|
|
30
|
+
entries.push(entry);
|
|
31
|
+
}
|
|
32
|
+
entries.sort(cmp);
|
|
33
|
+
for (let entry of entries) {
|
|
34
|
+
yield entry;
|
|
35
|
+
if (entry.isDirectory()) yield* entry.directory.tree(cmp);
|
|
36
|
+
}
|
|
37
|
+
} else {
|
|
38
|
+
for await (let entry of this) {
|
|
39
|
+
yield entry;
|
|
40
|
+
if (entry.isDirectory()) yield* entry.directory.tree();
|
|
41
|
+
}
|
|
30
42
|
}
|
|
31
43
|
}
|
|
32
44
|
async traverse() {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thi.ng/block-fs",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Customizable block-based storage, adapters & file system layer",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./index.js",
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
45
|
"@thi.ng/api": "^8.11.30",
|
|
46
|
-
"@thi.ng/args": "^2.7.
|
|
46
|
+
"@thi.ng/args": "^2.7.2",
|
|
47
47
|
"@thi.ng/binary": "^3.4.53",
|
|
48
48
|
"@thi.ng/bitfield": "^2.4.6",
|
|
49
49
|
"@thi.ng/checks": "^3.7.10",
|
|
@@ -91,6 +91,7 @@
|
|
|
91
91
|
"./*.js",
|
|
92
92
|
"./*.d.ts",
|
|
93
93
|
"bin",
|
|
94
|
+
"cli",
|
|
94
95
|
"storage"
|
|
95
96
|
],
|
|
96
97
|
"exports": {
|
|
@@ -100,6 +101,15 @@
|
|
|
100
101
|
"./api": {
|
|
101
102
|
"default": "./api.js"
|
|
102
103
|
},
|
|
104
|
+
"./cli/api": {
|
|
105
|
+
"default": "./cli/api.js"
|
|
106
|
+
},
|
|
107
|
+
"./cli/convert": {
|
|
108
|
+
"default": "./cli/convert.js"
|
|
109
|
+
},
|
|
110
|
+
"./cli/list": {
|
|
111
|
+
"default": "./cli/list.js"
|
|
112
|
+
},
|
|
103
113
|
"./cli": {
|
|
104
114
|
"default": "./cli.js"
|
|
105
115
|
},
|
|
@@ -132,5 +142,5 @@
|
|
|
132
142
|
"status": "alpha",
|
|
133
143
|
"year": 2024
|
|
134
144
|
},
|
|
135
|
-
"gitHead": "
|
|
145
|
+
"gitHead": "186d38b07f4ba940a8127cbe2379a38036eff387\n"
|
|
136
146
|
}
|