@thi.ng/block-fs 0.1.0 → 0.3.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-04-01T21:42:04Z
+- **Last updated**: 2025-04-02T18:47:59Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,42 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.3.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.3.0) (2025-04-02)
+
+#### 🚀 Features
+
+- add `BlockFS.readAsObjectURL()` ([551327c](https://github.com/thi-ng/umbrella/commit/551327c))
+- add mem storage opts, add logging ([27016d6](https://github.com/thi-ng/umbrella/commit/27016d6))
+  - add `MemoryBlockStorage` support for pre-loaded buffers
+  - add logging
+  - add docs
+- add CLI app wrapper ([68abe74](https://github.com/thi-ng/umbrella/commit/68abe74))
+  - add CLI app wrapper with these commands:
+    - `convert`: convert file tree into single BlockFS blob
+    - `list`: list file tree of a BlockFS blob
+  - update deps
+- improve tree display (`list` cmd) ([a23866c](https://github.com/thi-ng/umbrella/commit/a23866c))
+
+#### 🩹 Bug fixes
+
+- fix parent dir linkage ([a121e76](https://github.com/thi-ng/umbrella/commit/a121e76))
+
+#### ♻️ Refactoring
+
+- update sentinel block ID ([51a1e44](https://github.com/thi-ng/umbrella/commit/51a1e44))
+
+## [0.2.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.2.0) (2025-04-02)
+
+#### 🚀 Features
+
+- add path separator option, various refactoring ([875465e](https://github.com/thi-ng/umbrella/commit/875465e))
+  - add `BlockFSOpts.separator`
+  - rename `.readFileRaw()` => `.readBlocks()`
+  - rename `.writeFileRaw()` => `.writeBlocks()`
+  - add additional internal safety checks
+  - internal refactoring (`this` destructuring)
+  - add docs
+
 ## [0.1.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.1.0) (2025-04-01)
 
 #### 🚀 Features

package/README.md

@@ -21,6 +21,9 @@
     - [Root directory](#root-directory)
     - [Directory entries](#directory-entries)
     - [File blocks](#file-blocks)
+  - [Command line app](#command-line-app)
+    - [Convert file tree into single BlockFS blob](#convert-file-tree-into-single-blockfs-blob)
+    - [List file tree of a BlockFS blob](#list-file-tree-of-a-blockfs-blob)
 - [Status](#status)
 - [Installation](#installation)
 - [Dependencies](#dependencies)
@@ -53,6 +56,7 @@ The package also provides an hierarchical filesystem layer with pluggable
 storage providers and other customizable aspects. The default implementation
 supports:
 
+- 8 - 32bit block IDs
 - arbitrarily nested directories
 - filenames of max. 31 bytes (UTF-8) per directory level
 - max. 32 owner IDs
@@ -73,6 +77,9 @@ The filesystem stores a [bitfield](https://thi.ng/bitfield) of block allocations
 in the first N blocks. The number of blocks used depends on configured block
 size and the max. number of blocks in the storage backend.
 
+Blocks can be reserved for custom purposes by calling
+[`.allocateBlocks()`](https://docs.thi.ng/umbrella/block-fs/classes/BlockFS.html#allocateblocks).
+
 #### Root directory
 
 The root directory starts in block N, directly after the block allocation table.
@@ -98,6 +105,121 @@ the max. number of blocks in the storage backend.
 
 TODO diagram
 
+### Command line app
+
+The package includes a mult-command CLI app with the following operations:
+
+#### Convert file tree into single BlockFS blob
+
+The `convert` command is used to bundle an entire file tree from the host system
+into a single binary blob based on `BlockFS` with configured block size. The
+file tree MUST fit into the RAM available to `bun` (or `node`).
+
+Once bundled, the binary blob can then be used together with
+[`MemoryBlockStorage`](https://docs.thi.ng/umbrella/block-fs/classes/MemoryBlockStorage.html)
+and [`BlockFS`](https://docs.thi.ng/umbrella/block-fs/classes/BlockFS.html) for
+other purposes (e.g. distributed with your web app to provide a virtual
+filesystem).
+
+Example usage to bundle the source directory of this package:
+
+```bash
+npx @thi.ng/block-fs convert -o dummy.dat packages/block-fs/src/
+
+# [INFO] blockfs: number of files: 11
+# [INFO] blockfs: number of directories: 2
+# [INFO] blockfs: total file size: 40341
+# [INFO] blockfs: number of blocks: 56
+# [INFO] blockfs: writing file: dummy.dat
+```
+
+General usage:
+
+```text
+npx @thi.ng/block-fs convert --help
+
+Usage: blockfs <cmd> [opts] input
+
+Flags:
+
+-v, --verbose                   Display extra process information
+
+Main:
+
+-bs BYTES, --block-size BYTES   Block size (default: 1024)
+-n INT, --num-blocks INT        Number of blocks (multiple of 8)
+-o STR, --out STR               [required] Output file path
+```
+
+#### List file tree of a BlockFS blob
+
+The `list` command is used to list the files & directories stored in a binary blob created via the `convert` command. Several output options (e.g. `tree`-like output) are supported.
+
+```bash
+npx @thi.ng/block-fs list dummy.dat
+# /api.ts
+# /cli.ts
+# /directory.ts
+# /entry.ts
+# /fs.ts
+# /index.ts
+# /lock.ts
+# /storage
+# /storage/astorage.ts
+# /storage/file.ts
+# /storage/memory.ts
+# /utils.ts
+
+npx @thi.ng/block-fs list --tree dummy.dat
+# ├── api.ts
+# ├── cli.ts
+# ├── directory.ts
+# ├── entry.ts
+# ├── fs.ts
+# ├── index.ts
+# ├── lock.ts
+# ├── storage
+# │   ├── astorage.ts
+# │   ├── file.ts
+# │   └── memory.ts
+# └── utils.ts
+
+# display file sizes & modification times
+npx @thi.ng/block-fs list --tree --all dummy.dat
+# ├── api.ts           2204   2025-04-02T10:22:55.573Z
+# ├── cli.ts           6799   2025-04-02T18:07:58.895Z
+# ├── directory.ts     3994   2025-04-02T13:47:00.108Z
+# ├── entry.ts         4130   2025-04-02T10:22:55.574Z
+# ├── fs.ts            16377  2025-04-02T13:46:36.608Z
+# ├── index.ts         317    2025-04-01T21:38:08.232Z
+# ├── lock.ts          1501   2025-04-01T21:38:08.232Z
+# ├── storage                 2025-04-02T18:33:47.389Z
+# │   ├── astorage.ts  1205   2025-04-02T10:22:55.574Z
+# │   ├── file.ts      1780   2025-04-02T14:25:12.461Z
+# │   └── memory.ts    1802   2025-04-02T14:26:02.163Z
+# └── utils.ts         418    2025-04-02T10:22:55.574Z
+```
+
+General usage:
+
+```text
+npx @thi.ng/block-fs list --help
+
+Usage: blockfs <cmd> [opts] input
+
+Flags:
+
+-a, --all                       Display all attribs
+-t, --tree                      List files as tree
+-v, --verbose                   Display extra process information
+-m, --with-mtime                Display modified times
+-s, --with-size                 Display file sizes
+
+Main:
+
+-bs BYTES, --block-size BYTES   Block size (default: 1024)
+```
+
 ## Status
 
 **ALPHA** - bleeding edge / work-in-progress
@@ -130,11 +252,12 @@ For Node.js REPL:
 const bf = await import("@thi.ng/block-fs");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.11 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.43 KB
 
 ## Dependencies
 
 - [@thi.ng/api](https://github.com/thi-ng/umbrella/tree/develop/packages/api)
+- [@thi.ng/args](https://github.com/thi-ng/umbrella/tree/develop/packages/args)
 - [@thi.ng/binary](https://github.com/thi-ng/umbrella/tree/develop/packages/binary)
 - [@thi.ng/bitfield](https://github.com/thi-ng/umbrella/tree/develop/packages/bitfield)
 - [@thi.ng/checks](https://github.com/thi-ng/umbrella/tree/develop/packages/checks)
@@ -198,7 +321,10 @@ console.log(await fs.readFile("/deeply/nested/paths/are-ok"));
 
 // iterate all files & directory entries in root dir
 for await (let entry of fs.root.tree()) {
-    console.log(entry.type, entry.path, entry.size, new Date(entry.ctime));
+    // entry.path is absolute path
+    // entry.size is always a bigint
+    // entry.ctime/mtime is UNIX epoch
+    console.log(entry.path, entry.size, new Date(entry.ctime));
 }
 
 // /hello 0n 2025-04-01T20:18:55.916Z

package/api.d.ts

@@ -18,18 +18,35 @@ export interface IBlockStorage {
 }
 export interface BlockStorageOpts {
     logger?: ILogger;
+    /**
+     * Max. number of blocks. Will be rounded up to a multiple of 8.
+     */
     numBlocks: number;
+    /**
+     * Block size in bytes. Must be a power of 2. For usage with
+     * {@link BlockFS}, requires a minimum size of 128 bytes (though in practice
+     * should be larger to minimize overhead per block).
+     */
     blockSize: Pow2;
 }
 export interface EntrySpec {
+    /** Entry type (file or directory) */
     type: EntryType;
+    /** TODO currently still unused */
     locked?: boolean;
+    /** Owner ID */
     owner?: number;
+    /** Entry name */
     name: string;
+    /** Only used for files. File size as bigint */
     size?: bigint;
+    /** Entry creation timestamp */
     ctime?: number;
+    /** Entry modification timestamp */
     mtime?: number;
+    /** Data start block ID */
     start: number;
+    /** Data end block ID */
     end?: number;
 }
 export declare enum EntryType {

package/bin/blockfs

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# https://stackoverflow.com/a/246128/294515
+SOURCE="${BASH_SOURCE[0]}"
+while [ -L "$SOURCE" ]; do
+  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+DIR="$(dirname $DIR)"
+
+# prefer using bun
+if [ -x "$(command -v bun)" ]; then
+  CMD=bun
+else
+  CMD=node
+fi
+
+/usr/bin/env $CMD "$DIR/cli.js" "$DIR" "$@"

package/cli.d.ts

@@ -0,0 +1,23 @@
+import { type Command, type CommandCtx } from "@thi.ng/args";
+interface CLIOpts {
+    verbose: boolean;
+}
+interface ConvertOpts extends CLIOpts {
+    numBlocks?: number;
+    blockSize: number;
+    include?: string[];
+    out: string;
+}
+interface ListOpts extends CLIOpts {
+    blockSize: number;
+    all: boolean;
+    tree: boolean;
+    withMtime: boolean;
+    withSize: boolean;
+}
+export interface AppCtx<T extends CLIOpts> extends CommandCtx<T, CLIOpts> {
+}
+export declare const CONVERT: Command<ConvertOpts, CLIOpts, AppCtx<ConvertOpts>>;
+export declare const LIST: Command<ListOpts, CLIOpts, AppCtx<ListOpts>>;
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/cli.js

@@ -0,0 +1,214 @@
+import {
+  cliApp,
+  flag,
+  int,
+  string,
+  strings
+} from "@thi.ng/args";
+import { align, isPow2 } from "@thi.ng/binary";
+import { illegalArgs } from "@thi.ng/errors";
+import { files, readBinary, readJSON, writeFile } from "@thi.ng/file-io";
+import { LogLevel } from "@thi.ng/logger";
+import { statSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { Entry } from "./entry.js";
+import { BlockFS } from "./fs.js";
+import { MemoryBlockStorage } from "./storage/memory.js";
+const PKG = readJSON(join(process.argv[2], "package.json"));
+const ARG_BLOCKSIZE = {
+  blockSize: int({
+    alias: "bs",
+    desc: "Block size",
+    hint: "BYTES",
+    default: 1024,
+    coerce: (x) => {
+      const size = +x;
+      if (!isPow2(size)) illegalArgs("block size must be a power of 2");
+      return size;
+    }
+  })
+};
+const collectFiles = (ctx) => {
+  const root = resolve(ctx.inputs[0]);
+  const filtered = [];
+  const dirs = /* @__PURE__ */ new Set();
+  let total = 0;
+  for (let f of files(root)) {
+    const stats = statSync(f);
+    if (stats.isFile()) {
+      const dest = relative(root, f);
+      filtered.push({
+        src: f,
+        dest,
+        size: stats.size,
+        ctime: stats.ctimeMs,
+        mtime: stats.mtimeMs
+      });
+      dirs.add(dirname(dest));
+      total += stats.size;
+    }
+  }
+  return { files: filtered, dirs: [...dirs], size: total };
+};
+const computeBlockCount = (collected, blockSize, numBlocks) => {
+  let blocks = collected.dirs.length;
+  const blockIDBytes = numBlocks ? align(Math.ceil(Math.log2(numBlocks)), 8) >> 3 : align(Math.ceil(Math.log2(collected.size / blockSize + blocks)), 8) >> 3;
+  const blockDataSizeBytes = align(Math.ceil(Math.log2(blockSize)), 8) >> 3;
+  const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
+  blocks += Math.ceil(
+    (collected.files.length + collected.dirs.length) * Entry.SIZE / blockDataSize
+  );
+  for (let f of collected.files) {
+    blocks += Math.ceil(f.size / blockDataSize);
+  }
+  const blockIDBytes2 = align(Math.ceil(Math.log2(blocks)), 8) >> 3;
+  return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, blocks) : blocks;
+};
+const CONVERT = {
+  opts: {
+    ...ARG_BLOCKSIZE,
+    numBlocks: int({
+      alias: "n",
+      desc: "Number of blocks (multiple of 8)",
+      optional: true
+    }),
+    out: string({
+      alias: "o",
+      desc: "Output file path",
+      optional: false
+    }),
+    include: strings({
+      alias: "i",
+      desc: "Only include file extensions",
+      hint: "EXT"
+    })
+  },
+  desc: "Convert file tree into single BlockFS blob",
+  fn: async (ctx) => {
+    const collected = collectFiles(ctx);
+    const numBlocks = align(
+      ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize),
+      8
+    );
+    ctx.logger.info("number of files:", collected.files.length);
+    ctx.logger.info("number of directories:", collected.dirs.length);
+    ctx.logger.info("total file size:", collected.size);
+    ctx.logger.info("number of blocks:", numBlocks);
+    const storage = new MemoryBlockStorage({
+      numBlocks,
+      blockSize: ctx.opts.blockSize,
+      logger: ctx.logger
+    });
+    const bfs = new BlockFS(storage, { logger: ctx.logger });
+    await bfs.init();
+    for (let f of collected.files) {
+      ctx.logger.debug("writing file:", f.dest);
+      await bfs.writeFile(f.dest, readBinary(f.src));
+      const entry = await bfs.entryForPath(f.dest);
+      entry.ctime = f.ctime;
+      entry.mtime = f.mtime;
+      await entry.save();
+    }
+    writeFile(ctx.opts.out, storage.buffer, void 0, ctx.logger);
+  }
+};
+const LIST = {
+  opts: {
+    ...ARG_BLOCKSIZE,
+    all: flag({
+      alias: "a",
+      desc: "Display all attribs"
+    }),
+    tree: flag({
+      alias: "t",
+      desc: "List files as tree"
+    }),
+    withMtime: flag({
+      alias: "m",
+      desc: "Display modified times"
+    }),
+    withSize: flag({
+      alias: "s",
+      desc: "Display file sizes"
+    })
+  },
+  desc: "List file tree of a BlockFS blob",
+  fn: async (ctx) => {
+    if (ctx.opts.all) {
+      ctx.opts.withMtime = ctx.opts.withSize = true;
+    }
+    const buffer = readBinary(ctx.inputs[0]);
+    const storage = new MemoryBlockStorage({
+      numBlocks: buffer.length / ctx.opts.blockSize >>> 0,
+      blockSize: ctx.opts.blockSize,
+      logger: ctx.logger,
+      buffer
+    });
+    const bfs = new BlockFS(storage, { logger: ctx.logger });
+    await bfs.init();
+    const tree = [];
+    for await (let entry of bfs.root.tree()) {
+      const path = entry.path;
+      const depth = path.split("/").length - 2;
+      tree.push([path, depth, entry]);
+    }
+    tree.sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+    const rows = [];
+    const maxWidths = [0, 0, 0];
+    for (let i = 0, num = tree.length - 1; i <= num; i++) {
+      const f = tree[i];
+      const isLast = i === num || f[1] > tree[i + 1][1];
+      const row = ctx.opts.tree ? ["\u2502   ".repeat(f[1]) + (isLast ? "\u2514\u2500\u2500 " : "\u251C\u2500\u2500 ") + f[2].name] : [f[0]];
+      if (ctx.opts.withSize) {
+        row.push(f[2].isDirectory() ? "" : String(Number(f[2].size)));
+      }
+      if (ctx.opts.withMtime) {
+        row.push(new Date(f[2].mtime).toISOString());
+      }
+      rows.push(row);
+      for (let i2 = 0; i2 < row.length; i2++) {
+        maxWidths[i2] = Math.max(maxWidths[i2], row[i2].length);
+      }
+    }
+    for (let row of rows) {
+      console.log(row.map((x, i) => x.padEnd(maxWidths[i])).join("  "));
+    }
+  }
+};
+cliApp({
+  opts: {
+    verbose: flag({
+      alias: "v",
+      desc: "Display extra process information"
+    })
+  },
+  commands: {
+    convert: CONVERT,
+    list: LIST
+  },
+  name: "blockfs",
+  ctx: async (ctx) => {
+    if (ctx.opts.verbose) ctx.logger.level = LogLevel.DEBUG;
+    return ctx;
+  },
+  start: 3,
+  usage: {
+    prefix: `
+ \u2588 \u2588   \u2588           \u2502
+\u2588\u2588 \u2588               \u2502
+ \u2588 \u2588 \u2588 \u2588   \u2588 \u2588 \u2588 \u2588 \u2502 ${PKG.name} ${PKG.version}
+ \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2588 \u2502 Block-based storage & file system layer
+                 \u2588 \u2502
+               \u2588 \u2588 \u2502
+
+Usage: blockfs <cmd> [opts] input [...]
+       blockfs <cmd> --help
+`,
+    showGroupNames: true,
+    paramWidth: 32
+  }
+});
+export {
+  CONVERT,
+  LIST
+};

package/directory.js

@@ -30,18 +30,22 @@ class Directory {
     }
   }
   async traverse() {
+    const {
+      fs,
+      fs: { sentinelID, storage }
+    } = this;
     const blocks = [];
     const entries = [];
     let blockID = this.entry.start;
     while (true) {
       blocks.push(blockID);
-      const block = await this.fs.storage.loadBlock(blockID);
-      let { next, size } = this.fs.getBlockMeta(block);
-      if (!next) next = this.fs.sentinelID;
+      const block = await storage.loadBlock(blockID);
+      let { next, size } = fs.getBlockMeta(block);
+      if (!next) next = sentinelID;
       for (let i = 0; i < size; i++) {
         entries.push(this.defEntry(i, blockID, block));
       }
-      if (next === this.fs.sentinelID) break;
+      if (next === sentinelID) break;
       blockID = next;
     }
     return { blocks, entries };
@@ -52,16 +56,17 @@ class Directory {
     }
   }
   async mkdir(name) {
+    const fs = this.fs;
     const length = utf8Length(name);
-    if (!length || length > this.fs.opts.entry.maxLength)
+    if (!length || length > fs.opts.entry.maxLength)
       illegalArgs(`invalid name: '${name}'`);
     const traversed = await this.traverse();
     this.ensureUniqueName(name, traversed.entries);
-    const block = (await this.fs.allocateBlocks(1))[0];
-    const data = await this.fs.storage.loadBlock(block);
-    this.fs.setBlockMeta(data, this.fs.sentinelID, 0);
-    this.fs.setBlockLink(data, this.entry.start, this.fs.dataStartBlockID);
-    await this.fs.storage.saveBlock(block, data);
+    const block = (await fs.allocateBlocks(1))[0];
+    const data = await fs.storage.loadBlock(block);
+    fs.setBlockMeta(data, fs.sentinelID, 0);
+    fs.setBlockLink(data, this.entry.start, fs.blockDataOffset);
+    await fs.storage.saveBlock(block, data);
     return this.addEntry(
       {
         name,

package/entry.js

@@ -124,7 +124,7 @@ class Entry {
       path.unshift(entry.name);
       entry = entry.parent?.entry;
     }
-    return path.join("/");
+    return path.join(this.fs.opts.separator);
   }
   /**
    * Returns {@link IDirectory} wrapper for this entry (only if a directory,

package/fs.d.ts

@@ -3,6 +3,15 @@ import type { ILogger } from "@thi.ng/logger";
 import { EntryType, type IBlockStorage, type IDirectory, type IEntry } from "./api.js";
 import { Lock } from "./lock.js";
 export interface BlockFSOpts {
+    /**
+     * Path separator.
+     *
+     * @defaultValue `/`
+     */
+    separator: string;
+    /**
+     * Logger instance to use.
+     */
     logger: ILogger;
     /**
      * Customizable {@link IDirectory} factory function. By default creates
@@ -66,16 +75,112 @@ export declare class BlockFS {
     root: IDirectory;
     constructor(storage: IBlockStorage, opts?: Partial<BlockFSOpts>);
     init(): Promise<this>;
+    /**
+     * Attempts to find the {@link IEntry} for given `path` and returns it if
+     * successful. Throws error if no such path exists.
+     *
+     * @remarks
+     * Also see {@link BlockFS.ensureEntryForPath}.
+     *
+     * @param path
+     */
     entryForPath(path: string): Promise<IEntry>;
+    /**
+     * Attempts to find or to create the {@link IEntry} for given `path` and
+     * entry type (file or directory) and then returns it if successful. Throws
+     * error if the path exists, but does not match the expected type or if the
+     * entry could not be created for other reasons.
+     *
+     * @remarks
+     * Also see {@link BlockFS.entryForPath}.
+     *
+     * @param path
+     * @param type
+     */
     ensureEntryForPath(path: string, type: EntryType): Promise<IEntry>;
+    /**
+     * Attempts to allocate a number of free blocks required for storing the
+     * given number of `bytes`. If successful, marks the blocks as used in the
+     * allocation table and then returns list of their IDs, otherwise throws an
+     * error if there're insufficient blocks available.
+     *
+     * @param bytes
+     */
     allocateBlocks(bytes: number): Promise<number[]>;
+    /**
+     * Marks the given block IDs as free/unused in the block allocation table
+     * and deletes/clears the blocks.
+     *
+     * @param ids
+     */
     freeBlocks(ids: number[]): Promise<void>;
+    /**
+     * Same as POSIX `mkdirp`. Attempts to create given directory, including any
+     * missing intermediate ones defined by `path`.
+     *
+     * @param path
+     */
     mkdir(path: string): Promise<IEntry>;
-    readFileRaw(path: string | number): AsyncGenerator<Uint8Array<ArrayBufferLike>, void, unknown>;
+    /**
+     * Async iterator. Reads block list for given `path` (file path or start
+     * block ID) and yields data contents (byte arrays) of each block.
+     *
+     * @remarks
+     * Also see other read methods:
+     *
+     * - {@link BlockFS.readFile}
+     * - {@link BlockFS.readText}
+     * - {@link BlockFS.readJSON}
+     *
+     * @param path
+     */
+    readBlocks(path: string | number): AsyncGenerator<Uint8Array<ArrayBufferLike>, void, unknown>;
+    /**
+     * Fully reads given file into a single byte buffer and returns it.
+     *
+     * @param path
+     */
     readFile(path: string | number): Promise<Uint8Array<ArrayBuffer>>;
+    /**
+     * Fully reads given file into a single UTF-8 byte buffer, then decodes and
+     * returns it as string.
+     *
+     * @param path
+     */
     readText(path: string | number): Promise<string>;
-    readJSON(path: string | number): Promise<any>;
-    writeFileRaw(blocks: number[] | null, data: Uint8Array): Promise<{
+    /**
+     * Fully reads given file into a single UTF-8 byte buffer, then decodes it
+     * as JSON and returns result.
+     *
+     * @param path
+     */
+    readJSON<T>(path: string | number): Promise<T>;
+    /**
+     * Fully reads given file into a single byte buffer and returns it as blob
+     * object URL, optionally typed with given MIME type.
+     *
+     * @remarks
+     * Reference:
+     *
+     * - https://developer.mozilla.org/en-US/docs/Web/API/Blob#creating_a_url_representing_the_contents_of_a_typed_array
+     *
+     * @param path
+     * @param type
+     */
+    readAsObjectURL(path: string | number, type?: string): Promise<string>;
+    /**
+     * Takes an array of block IDs (or `null`) and a `data` byte array. Writes
+     * chunks of data into given blocks and connecting each block as linked
+     * list. Returns object of start/end block IDs and data size.
+     *
+     * @remarks
+     * If `blocks` is null, blocks are automatically allocated via
+     * {@link BlockFS.allocateBlocks}.
+     *
+     * @param blocks
+     * @param data
+     */
+    writeBlocks(blocks: number[] | null, data: Uint8Array): Promise<{
         start: number;
         end: number;
         size: number;
@@ -85,7 +190,7 @@ export declare class BlockFS {
         end: number;
         size: number;
     }>;
-    appendFileRaw(blockID: number, data: Uint8Array): Promise<{
+    appendBlocks(blockID: number, data: Uint8Array): Promise<{
         start: number;
         end: number;
         size: number;

package/fs.js

@@ -3,6 +3,7 @@ import { defBitField } from "@thi.ng/bitfield/bitfield";
 import { isString } from "@thi.ng/checks/is-string";
 import { assert } from "@thi.ng/errors/assert";
 import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
+import { illegalState } from "@thi.ng/errors/illegal-state";
 import { NULL_LOGGER } from "@thi.ng/logger/null";
 import {
   EntryType
@@ -22,9 +23,10 @@ class BlockFS {
   constructor(storage, opts) {
     this.storage = storage;
     this.opts = {
-      logger: NULL_LOGGER,
       directory: (fs, entry) => new Directory(fs, entry),
       entry: DEFAULT_ENTRY,
+      logger: NULL_LOGGER,
+      separator: "/",
       ...opts
     };
     assert(
@@ -39,7 +41,7 @@ class BlockFS {
     this.rootDirBlockID = align(this.blockIndex.data.length, storage.blockSize) / storage.blockSize;
     this.dataStartBlockID = this.rootDirBlockID + 1;
     this.dirDataOffset = this.blockDataOffset + this.blockIDBytes;
-    this.sentinelID = storage.numBlocks - 1;
+    this.sentinelID = 2 ** (this.blockIDBytes * 8) - 1;
     this.tmp = new Uint8Array(storage.blockSize);
   }
   blockIndex;
@@ -67,60 +69,84 @@ class BlockFS {
   /** Root directory */
   root;
   async init() {
-    const indexSize = this.blockIndex.data.length;
-    const blockSize = this.storage.blockSize;
-    for (let i = 0; i < this.rootDirBlockID; i++) {
-      const data = await this.storage.loadBlock(i);
+    const { blockIndex, storage, opts, rootDirBlockID } = this;
+    const indexSize = blockIndex.data.length;
+    const blockSize = storage.blockSize;
+    for (let i = 0; i < rootDirBlockID; i++) {
+      const data = await storage.loadBlock(i);
       const offset = i * blockSize;
       if (offset + data.length > indexSize) {
-        this.blockIndex.data.set(
+        blockIndex.data.set(
           data.subarray(0, indexSize - offset),
           offset
         );
       } else {
-        this.blockIndex.data.set(data, offset);
+        blockIndex.data.set(data, offset);
       }
     }
-    this.blockIndex.fill(1, 0, this.dataStartBlockID);
-    const rootEntry = this.opts.entry.factory(
+    blockIndex.fill(1, 0, this.dataStartBlockID);
+    const rootEntry = opts.entry.factory(
       this,
       null,
-      this.rootDirBlockID,
-      new Uint8Array(this.opts.entry.size),
+      rootDirBlockID,
+      new Uint8Array(opts.entry.size),
       0
     );
     rootEntry.set({
       name: "",
       type: EntryType.DIR,
-      start: this.rootDirBlockID,
+      start: rootDirBlockID,
       owner: 0
     });
-    this.root = this.opts.directory(this, rootEntry);
+    this.root = opts.directory(this, rootEntry);
     return this;
   }
+  /**
+   * Attempts to find the {@link IEntry} for given `path` and returns it if
+   * successful. Throws error if no such path exists.
+   *
+   * @remarks
+   * Also see {@link BlockFS.ensureEntryForPath}.
+   *
+   * @param path
+   */
   async entryForPath(path) {
     let dir = this.root;
-    if (path[0] === "/") path = path.substring(1);
+    const { directory, separator } = this.opts;
+    if (path[0] === separator) path = path.substring(1);
     if (path === "") return dir.entry;
-    const $path = path.split("/");
-    for (let i = 0; i < $path.length; i++) {
+    const $path = path.split(separator);
+    for (let i = 0, len = $path.length - 1; i <= len; i++) {
       let entry = await dir.findName($path[i]);
       if (!entry) break;
-      if (i === $path.length - 1) return entry;
+      if (i === len) return entry;
       if (!entry.isDirectory()) illegalArgs(path);
-      dir = this.opts.directory(this, entry);
+      dir = directory(this, entry);
     }
     illegalArgs(path);
   }
+  /**
+   * Attempts to find or to create the {@link IEntry} for given `path` and
+   * entry type (file or directory) and then returns it if successful. Throws
+   * error if the path exists, but does not match the expected type or if the
+   * entry could not be created for other reasons.
+   *
+   * @remarks
+   * Also see {@link BlockFS.entryForPath}.
+   *
+   * @param path
+   * @param type
+   */
   async ensureEntryForPath(path, type) {
     let dir = this.root;
-    if (path[0] === "/") path = path.substring(1);
+    const { directory, separator } = this.opts;
+    if (path[0] === separator) path = path.substring(1);
     if (path === "") return dir.entry;
-    const $path = path.split("/");
-    for (let i = 0; i < $path.length; i++) {
+    const $path = path.split(separator);
+    for (let i = 0, len = $path.length - 1; i <= len; i++) {
       let entry = await dir.findName($path[i]);
       if (!entry) {
-        if (i < $path.length - 1) {
+        if (i < len) {
           entry = await dir.mkdir($path[i]);
         } else {
           return await dir.addEntry({
@@ -130,7 +156,7 @@ class BlockFS {
           });
         }
       }
-      if (i === $path.length - 1) {
+      if (i === len) {
         if (entry.type !== type)
           illegalArgs(
             `path exists, but is not a ${EntryType[type]}: ${path}`
@@ -138,32 +164,53 @@ class BlockFS {
         return entry;
       }
       if (!entry.isDirectory()) illegalArgs(path);
-      dir = this.opts.directory(this, entry);
+      dir = directory(this, entry);
     }
     illegalArgs(path);
   }
+  /**
+   * Attempts to allocate a number of free blocks required for storing the
+   * given number of `bytes`. If successful, marks the blocks as used in the
+   * allocation table and then returns list of their IDs, otherwise throws an
+   * error if there're insufficient blocks available.
+   *
+   * @param bytes
+   */
   async allocateBlocks(bytes) {
-    const lockID = await this.lock.acquire();
+    const {
+      blockDataSize,
+      blockIndex,
+      dataStartBlockID,
+      lock,
+      sentinelID
+    } = this;
+    const lockID = await lock.acquire();
     try {
       const ids = [];
-      let last = this.dataStartBlockID;
+      let last = dataStartBlockID;
       while (bytes > 0) {
-        const next = this.blockIndex.firstZero(last);
-        if (next < 0 || next >= this.sentinelID) {
+        const next = blockIndex.firstZero(last);
+        if (next < 0 || next >= sentinelID) {
           throw new Error(
             `insufficient free blocks for storing ${bytes} bytes`
           );
         }
         ids.push(next);
         last = next + 1;
-        bytes -= this.blockDataSize;
+        bytes -= blockDataSize;
       }
       await this.updateBlockIndex(ids, 1);
       return ids;
     } finally {
-      await this.lock.release(lockID);
+      await lock.release(lockID);
     }
   }
+  /**
+   * Marks the given block IDs as free/unused in the block allocation table
+   * and deletes/clears the blocks.
+   *
+   * @param ids
+   */
   async freeBlocks(ids) {
     const lockID = await this.lock.acquire();
     try {
@@ -173,54 +220,116 @@ class BlockFS {
       await this.lock.release(lockID);
     }
   }
+  /**
+   * Same as POSIX `mkdirp`. Attempts to create given directory, including any
+   * missing intermediate ones defined by `path`.
+   *
+   * @param path
+   */
   async mkdir(path) {
     return this.ensureEntryForPath(path, EntryType.DIR);
   }
-  async *readFileRaw(path) {
+  /**
+   * Async iterator. Reads block list for given `path` (file path or start
+   * block ID) and yields data contents (byte arrays) of each block.
+   *
+   * @remarks
+   * Also see other read methods:
+   *
+   * - {@link BlockFS.readFile}
+   * - {@link BlockFS.readText}
+   * - {@link BlockFS.readJSON}
+   *
+   * @param path
+   */
+  async *readBlocks(path) {
     let blockID = isString(path) ? (await this.entryForPath(path)).start : path;
+    const { blockDataOffset, blockIndex, sentinelID, storage } = this;
     while (true) {
-      if (!this.blockIndex.at(blockID)) {
+      if (!blockIndex.at(blockID)) {
         throw new Error(`invalid block ref: ${blockID}`);
       }
-      const bytes = await this.storage.loadBlock(blockID);
+      const bytes = await storage.loadBlock(blockID);
       const { next, size } = this.getBlockMeta(bytes);
-      yield bytes.subarray(
-        this.blockDataOffset,
-        this.blockDataOffset + size
-      );
-      if (next === this.sentinelID) return;
+      yield bytes.subarray(blockDataOffset, blockDataOffset + size);
+      if (next === sentinelID) return;
       blockID = next;
     }
   }
+  /**
+   * Fully reads given file into a single byte buffer and returns it.
+   *
+   * @param path
+   */
   async readFile(path) {
     const buffer = [];
-    for await (let block of this.readFileRaw(path)) buffer.push(...block);
+    for await (let block of this.readBlocks(path)) buffer.push(...block);
     return new Uint8Array(buffer);
   }
+  /**
+   * Fully reads given file into a single UTF-8 byte buffer, then decodes and
+   * returns it as string.
+   *
+   * @param path
+   */
   async readText(path) {
     return new TextDecoder().decode(await this.readFile(path));
   }
+  /**
+   * Fully reads given file into a single UTF-8 byte buffer, then decodes it
+   * as JSON and returns result.
+   *
+   * @param path
+   */
   async readJSON(path) {
     return JSON.parse(await this.readText(path));
   }
-  async writeFileRaw(blocks, data) {
+  /**
+   * Fully reads given file into a single byte buffer and returns it as blob
+   * object URL, optionally typed with given MIME type.
+   *
+   * @remarks
+   * Reference:
+   *
+   * - https://developer.mozilla.org/en-US/docs/Web/API/Blob#creating_a_url_representing_the_contents_of_a_typed_array
+   *
+   * @param path
+   * @param type
+   */
+  async readAsObjectURL(path, type) {
+    return URL.createObjectURL(
+      new Blob([await this.readFile(path)], { type })
+    );
+  }
+  /**
+   * Takes an array of block IDs (or `null`) and a `data` byte array. Writes
+   * chunks of data into given blocks and connecting each block as linked
+   * list. Returns object of start/end block IDs and data size.
+   *
+   * @remarks
+   * If `blocks` is null, blocks are automatically allocated via
+   * {@link BlockFS.allocateBlocks}.
+   *
+   * @param blocks
+   * @param data
+   */
+  async writeBlocks(blocks, data) {
+    const { blockDataOffset, blockDataSize, sentinelID, storage } = this;
     if (!blocks) blocks = await this.allocateBlocks(data.length);
     let offset = 0;
     for (let i = 0, numBlocks = blocks.length - 1; i <= numBlocks; i++) {
+      if (offset >= data.length)
+        illegalState(`too many blocks, EOF already reached`);
       const id = blocks[i];
-      const block = await this.storage.loadBlock(id);
-      i < numBlocks ? this.setBlockMeta(block, blocks[i + 1], this.blockDataSize) : this.setBlockMeta(
-        block,
-        this.sentinelID,
-        data.length - offset
-      );
-      const chunk = data.subarray(offset, offset + this.blockDataSize);
-      block.set(chunk, this.blockDataOffset);
-      if (chunk.length < this.blockDataSize) {
-        block.fill(0, this.blockDataOffset + chunk.length);
+      const block = await storage.loadBlock(id);
+      i < numBlocks ? this.setBlockMeta(block, blocks[i + 1], blockDataSize) : this.setBlockMeta(block, sentinelID, data.length - offset);
+      const chunk = data.subarray(offset, offset + blockDataSize);
+      block.set(chunk, blockDataOffset);
+      if (chunk.length < blockDataSize) {
+        block.fill(0, blockDataOffset + chunk.length);
       }
-      await this.storage.saveBlock(id, block);
-      offset += this.blockDataSize;
+      await storage.saveBlock(id, block);
+      offset += blockDataSize;
     }
     return {
       start: blocks[0],
@@ -230,7 +339,7 @@ class BlockFS {
   }
   async writeFile(path, data) {
     if (isString(data)) data = new TextEncoder().encode(data);
-    if (!path) return this.writeFileRaw(null, data);
+    if (!path) return this.writeBlocks(null, data);
     const entry = await this.ensureEntryForPath(path, EntryType.FILE);
     let blocks = await this.blockList(entry.start);
     const overflow = data.length - blocks.length * this.blockDataSize;
@@ -241,37 +350,38 @@ class BlockFS {
       await this.freeBlocks(blocks.slice(needed));
       blocks = blocks.slice(0, needed);
     }
-    blocks.sort();
+    blocks.sort((a, b) => a - b);
     entry.start = blocks[0];
     entry.end = blocks[blocks.length - 1];
     entry.size = BigInt(data.length);
     entry.mtime = Date.now();
     await entry.save();
-    return this.writeFileRaw(blocks, data);
+    return this.writeBlocks(blocks, data);
   }
-  async appendFileRaw(blockID, data) {
-    if (blockID === this.sentinelID) return this.writeFileRaw(null, data);
+  async appendBlocks(blockID, data) {
+    if (blockID === this.sentinelID) return this.writeBlocks(null, data);
+    const { blockDataOffset, blockDataSize, storage } = this;
     const blocks = await this.blockList(blockID);
     const lastBlockID = blocks[blocks.length - 1];
-    const lastBlock = await this.storage.loadBlock(lastBlockID);
+    const lastBlock = await storage.loadBlock(lastBlockID);
     const currLength = decodeBytes(
       lastBlock,
       this.blockIDBytes,
       this.blockDataSizeBytes
     );
-    const remaining = this.blockDataSize - currLength;
-    lastBlock.fill(0, 0, this.blockDataOffset);
+    const remaining = blockDataSize - currLength;
+    lastBlock.fill(0, 0, blockDataOffset);
     lastBlock.set(
       data.subarray(0, remaining),
-      this.blockDataOffset + currLength
+      blockDataOffset + currLength
     );
     let newEndBlockID;
     if (data.length > remaining) {
-      const { start, end } = await this.writeFileRaw(
+      const { start, end } = await this.writeBlocks(
         null,
         data.subarray(remaining)
       );
-      this.setBlockMeta(lastBlock, start, this.blockDataSize);
+      this.setBlockMeta(lastBlock, start, blockDataSize);
       newEndBlockID = end;
     } else {
       this.setBlockMeta(
@@ -281,14 +391,14 @@ class BlockFS {
       );
       newEndBlockID = lastBlockID;
     }
-    await this.storage.saveBlock(lastBlockID, lastBlock);
+    await storage.saveBlock(lastBlockID, lastBlock);
     return { start: blockID, end: newEndBlockID };
   }
   async appendFile(path, data) {
     if (isString(data)) data = new TextEncoder().encode(data);
-    if (!isString(path)) return this.appendFileRaw(path, data);
+    if (!isString(path)) return this.appendBlocks(path, data);
     const entry = await this.ensureEntryForPath(path, EntryType.FILE);
-    const { start, end } = await this.appendFileRaw(entry.end, data);
+    const { start, end } = await this.appendBlocks(entry.end, data);
     if (entry.start === this.sentinelID) entry.start = start;
     entry.end = end;
     entry.size += BigInt(data.length);
@@ -308,33 +418,35 @@ class BlockFS {
     }
   }
   async blockList(blockID) {
+    const { blockIDBytes, sentinelID, storage } = this;
     const blocks = [];
-    if (blockID === this.sentinelID) return blocks;
+    if (blockID === sentinelID) return blocks;
     while (true) {
       blocks.push(blockID);
-      const block = await this.storage.loadBlock(blockID);
-      const nextID = decodeBytes(block, 0, this.blockIDBytes);
-      if (nextID === this.sentinelID) break;
+      const block = await storage.loadBlock(blockID);
+      const nextID = decodeBytes(block, 0, blockIDBytes);
+      if (nextID === sentinelID) break;
       blockID = nextID;
     }
     return blocks;
   }
   async updateBlockIndex(ids, state) {
-    const blockSize = this.storage.blockSize;
+    const { blockIndex, storage, tmp } = this;
+    const blockSize = storage.blockSize;
     const updatedBlocks = /* @__PURE__ */ new Set();
     for (let id of ids) {
-      this.blockIndex.setAt(id, state);
+      blockIndex.setAt(id, state);
       updatedBlocks.add((id >>> 3) / blockSize | 0);
     }
     for (let id of updatedBlocks) {
       this.opts.logger.debug("update block index", id);
-      const chunk = this.blockIndex.data.subarray(
+      const chunk = blockIndex.data.subarray(
         id * blockSize,
         (id + 1) * blockSize
       );
-      this.tmp.set(chunk);
-      if (chunk.length < blockSize) this.tmp.fill(0, chunk.length);
-      await this.storage.saveBlock(id, this.tmp);
+      tmp.set(chunk);
+      if (chunk.length < blockSize) tmp.fill(0, chunk.length);
+      await storage.saveBlock(id, tmp);
     }
   }
   /** @internal */

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "@thi.ng/block-fs",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Customizable block-based storage, adapters & file system layer",
   "type": "module",
   "module": "./index.js",
   "typings": "./index.d.ts",
+  "bin": {
+    "blockfs": "bin/blockfs"
+  },
   "sideEffects": false,
   "repository": {
     "type": "git",
@@ -40,6 +43,7 @@
   },
   "dependencies": {
     "@thi.ng/api": "^8.11.25",
+    "@thi.ng/args": "^2.3.66",
     "@thi.ng/binary": "^3.4.48",
     "@thi.ng/bitfield": "^2.4.0",
     "@thi.ng/checks": "^3.7.5",
@@ -58,10 +62,13 @@
   "keywords": [
     "binary",
     "block",
+    "cli",
+    "conversion",
     "file",
     "file-system",
     "memory",
     "memory-mapped",
+    "nodejs",
     "path",
     "storage",
     "typedarray",
@@ -80,6 +87,7 @@
   "files": [
     "./*.js",
     "./*.d.ts",
+    "bin",
     "storage"
   ],
   "exports": {
@@ -89,6 +97,9 @@
     "./api": {
       "default": "./api.js"
     },
+    "./cli": {
+      "default": "./cli.js"
+    },
     "./directory": {
       "default": "./directory.js"
     },
@@ -118,5 +129,5 @@
     "status": "alpha",
     "year": 2024
   },
-  "gitHead": "87aa2d0e64a357476c10fd57aabdfded13c79f7d\n"
+  "gitHead": "549e798e6a82254ee68727f12229c5252892b8f1\n"
 }

package/storage/file.d.ts

@@ -8,7 +8,13 @@ export declare class FileBlock implements IBlock {
     save(data: Uint8Array): Promise<void>;
     delete(): Promise<void>;
 }
+/**
+ * Configuration options for {@link FileBlockStorage}.
+ */
 export interface FileBlockStorageOpts extends BlockStorageOpts {
+    /**
+     * Path to host filesystem base directory used for storing blocks.
+     */
     baseDir: string;
 }
 export declare class FileBlockStorage extends ABlockStorage<FileBlock> {

package/storage/file.js

@@ -11,18 +11,15 @@ class FileBlock {
     this.id = id;
   }
   async load() {
-    const path = this.storage.getPath(this.id);
-    return existsSync(path) ? readBinary(path, this.storage.logger) : new Uint8Array(this.storage.blockSize);
+    const { storage } = this;
+    const path = storage.getPath(this.id);
+    return existsSync(path) ? readBinary(path, storage.logger) : new Uint8Array(storage.blockSize);
   }
   async save(data) {
-    if (data.length !== this.storage.blockSize)
+    const { storage } = this;
+    if (data.length !== storage.blockSize)
       illegalArgs(`wrong block size: ${data.length}`);
-    writeFile(
-      this.storage.getPath(this.id),
-      data,
-      null,
-      this.storage.logger
-    );
+    writeFile(storage.getPath(this.id), data, null, storage.logger);
   }
   async delete() {
     deleteFile(this.storage.getPath(this.id), this.storage.logger);

package/storage/memory.d.ts

@@ -8,9 +8,19 @@ export declare class MemoryBlock implements IBlock {
     save(data: Uint8Array): Promise<void>;
     delete(): Promise<void>;
 }
+/**
+ * Configuration options for {@link MemoryBlockStorage}.
+ */
+export interface MemoryBlockStorageOpts extends BlockStorageOpts {
+    /**
+     * Optional, pre-defined/loaded byte buffer. Must have at least `numBlocks *
+     * blockSize` capacity.
+     */
+    buffer?: Uint8Array;
+}
 export declare class MemoryBlockStorage extends ABlockStorage<MemoryBlock> {
     buffer: Uint8Array;
-    constructor(opts: BlockStorageOpts);
+    constructor(opts: MemoryBlockStorageOpts);
     hasBlock(id: number): Promise<boolean>;
     ensureBlock(id: number): MemoryBlock;
 }

package/storage/memory.js

@@ -6,16 +6,16 @@ class MemoryBlock {
     this.id = id;
   }
   async load() {
-    const size = this.storage.blockSize;
-    return this.storage.buffer.subarray(
-      this.id * size,
-      (this.id + 1) * size
-    );
+    const { storage } = this;
+    storage.logger.debug("load block", this.id);
+    const size = storage.blockSize;
+    return storage.buffer.subarray(this.id * size, (this.id + 1) * size);
   }
   async save(data) {
-    if (data.length !== this.storage.blockSize)
-      illegalArgs(`wrong block size`);
-    this.storage.buffer.set(data, this.id * this.storage.blockSize);
+    const { storage } = this;
+    if (data.length !== storage.blockSize) illegalArgs(`wrong block size`);
+    storage.logger.debug("save block", this.id);
+    storage.buffer.set(data, this.id * storage.blockSize);
   }
   async delete() {
     const size = this.storage.blockSize;
@@ -26,7 +26,13 @@ class MemoryBlockStorage extends ABlockStorage {
   buffer;
   constructor(opts) {
     super(opts);
-    this.buffer = new Uint8Array(this.numBlocks * this.blockSize);
+    const size = this.numBlocks * this.blockSize;
+    if (opts.buffer && opts.buffer.length < size) {
+      illegalArgs(
+        `given buffer is too small, expected at least ${size} bytes`
+      );
+    }
+    this.buffer = opts.buffer ?? new Uint8Array(size);
   }
   async hasBlock(id) {
     this.ensureValidID(id);