@thi.ng/block-fs 0.4.4 → 0.5.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-06-09T17:24:08Z
+- **Last updated**: 2025-06-18T12:01:21Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,12 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.5.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.5.0) (2025-06-18)
+
+#### 🚀 Features
+
+- update block count estimation, add more logging ([02c8037](https://github.com/thi-ng/umbrella/commit/02c8037))
+
 ### [0.4.1](https://github.com/thi-ng/umbrella/tree/@thi.ng/block-fs@0.4.1) (2025-04-16)
 
 #### ♻️ Refactoring

package/README.md

@@ -48,6 +48,12 @@ storage providers are included:
 - [FileBlockStorage](https://docs.thi.ng/umbrella/block-fs/classes/FileBlockStorage.html):
   Host-filesystem based blocks (one block per file)
 
+> [IMPORTANT]
+> For browser-compatibility reasons, the `FileBlockStorage` is NOT exposed when
+> using package-level imports. Use the following import to use this class:
+>
+> `import { FileBlockStorage } from "@thi.ng/block-fs/storage/file";
+
 As the name indicates, block storage providers only support block-based
 read/write/delete access to arbitrary binary data (all ops are async). For
 file-based storage, blocks are created lazily/dynamically.
@@ -288,7 +294,7 @@ For Node.js REPL:
 const bf = await import("@thi.ng/block-fs");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 4.50 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 4.51 KB
 
 ## Dependencies
 

package/cli.js

@@ -10,7 +10,7 @@ import { illegalArgs } from "@thi.ng/errors";
 import { files, readBinary, readJSON, writeFile } from "@thi.ng/file-io";
 import { LogLevel } from "@thi.ng/logger";
 import { statSync } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
+import { join, relative, resolve } from "node:path";
 import { Entry } from "./entry.js";
 import { BlockFS } from "./fs.js";
 import { MemoryBlockStorage } from "./storage/memory.js";
@@ -28,6 +28,14 @@ const ARG_BLOCKSIZE = {
     }
   })
 };
+const requiredBytes = (x) => align(Math.ceil(Math.log2(x)), 8) >> 3;
+const pathsForFile = (f) => {
+  const parts = f.split("/");
+  const dirs = [];
+  for (let i = 1; i < parts.length; i++)
+    dirs.push(parts.slice(0, i).join("/"));
+  return dirs;
+};
 const collectFiles = ({
   opts: { include, exclude },
   inputs
@@ -50,24 +58,30 @@ const collectFiles = ({
       ctime: stats.ctimeMs,
       mtime: stats.mtimeMs
     });
-    dirs.add(dirname(dest));
+    for (let d of pathsForFile(dest)) dirs.add(d);
     total += stats.size;
   }
   return { files: filtered, dirs: [...dirs], size: total };
 };
-const computeBlockCount = (collected, blockSize, numBlocks) => {
+const computeBlockCount = (collected, blockSize, logger, numBlocks) => {
   let blocks = collected.dirs.length;
-  const blockIDBytes = numBlocks ? align(Math.ceil(Math.log2(numBlocks)), 8) >> 3 : align(Math.ceil(Math.log2(collected.size / blockSize + blocks)), 8) >> 3;
-  const blockDataSizeBytes = align(Math.ceil(Math.log2(blockSize)), 8) >> 3;
-  const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
-  blocks += Math.ceil(
-    (collected.files.length + collected.dirs.length) * Entry.SIZE / blockDataSize
+  const blockIDBytes = requiredBytes(
+    numBlocks ?? collected.size / blockSize + blocks
   );
+  const blockDataSizeBytes = requiredBytes(blockSize);
+  const blockDataSize = blockSize - blockIDBytes - blockDataSizeBytes;
+  const numEntries = collected.files.length + collected.dirs.length;
+  const numEntryBlocks = Math.ceil(numEntries * Entry.SIZE / blockDataSize);
+  logger.info("num entries:", numEntries);
+  logger.info("num entry blocks:", numEntryBlocks);
+  blocks += numEntryBlocks;
   for (let f of collected.files) {
-    blocks += Math.ceil(f.size / blockDataSize);
+    const size = Math.ceil(f.size / blockDataSize);
+    logger.debug("file:", f.src, "blocks:", size);
+    blocks += size;
   }
-  const blockIDBytes2 = align(Math.ceil(Math.log2(blocks)), 8) >> 3;
-  return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, blocks) : blocks;
+  const blockIDBytes2 = requiredBytes(blocks);
+  return blockIDBytes2 > blockIDBytes ? computeBlockCount(collected, blockSize, logger, blocks) : blocks;
 };
 const CONVERT = {
   opts: {
@@ -97,7 +111,7 @@ const CONVERT = {
   fn: async (ctx) => {
     const collected = collectFiles(ctx);
     const numBlocks = align(
-      ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize),
+      ctx.opts.numBlocks ?? computeBlockCount(collected, ctx.opts.blockSize, ctx.logger),
       8
     );
     ctx.logger.info("number of files:", collected.files.length);
@@ -111,9 +125,13 @@ const CONVERT = {
     });
     const bfs = new BlockFS(storage, { logger: ctx.logger });
     await bfs.init();
+    ctx.logger.info("root dir block:", bfs.rootDirBlockID);
+    ctx.logger.info("first data block:", bfs.dataStartBlockID);
+    ctx.logger.info("block data size:", bfs.blockDataSize);
     for (let f of collected.files) {
-      ctx.logger.info("writing file:", f.dest);
-      await bfs.writeFile(f.dest, readBinary(f.src));
+      const data = readBinary(f.src, ctx.logger);
+      ctx.logger.info("writing file:", f.dest, "size:", data.length);
+      await bfs.writeFile(f.dest, data);
       const entry = await bfs.entryForPath(f.dest);
       entry.ctime = f.ctime;
       entry.mtime = f.mtime;

package/fs.js

@@ -323,6 +323,7 @@ class BlockFS {
   async writeBlocks(blocks, data) {
     const { blockDataOffset, blockDataSize, sentinelID, storage } = this;
     if (!blocks) blocks = await this.allocateBlocks(data.length);
+    this.opts.logger.debug("allocated blocks", blocks.length, blocks);
     let offset = 0;
     for (let i = 0, numBlocks = blocks.length - 1; i <= numBlocks; i++) {
       if (offset >= data.length)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/block-fs",
-  "version": "0.4.4",
+  "version": "0.5.0",
   "description": "Customizable block-based storage, adapters & file system layer",
   "type": "module",
   "module": "./index.js",
@@ -43,16 +43,16 @@
   },
   "dependencies": {
     "@thi.ng/api": "^8.11.29",
-    "@thi.ng/args": "^2.3.70",
+    "@thi.ng/args": "^2.3.71",
     "@thi.ng/binary": "^3.4.52",
-    "@thi.ng/bitfield": "^2.4.4",
+    "@thi.ng/bitfield": "^2.4.5",
     "@thi.ng/checks": "^3.7.9",
     "@thi.ng/errors": "^2.5.35",
     "@thi.ng/file-io": "^2.1.38",
     "@thi.ng/logger": "^3.1.10",
     "@thi.ng/mime": "^2.7.11",
     "@thi.ng/random": "^4.1.20",
-    "@thi.ng/strings": "^3.9.14"
+    "@thi.ng/strings": "^3.9.15"
   },
   "devDependencies": {
     "@types/node": "^22.15.21",
@@ -132,5 +132,5 @@
     "status": "alpha",
     "year": 2024
   },
-  "gitHead": "93cdcd8db4d4669561a7f0ebc47697bdbfd04214\n"
+  "gitHead": "b076434a497b291ad33e81b1a15f6a71e2c82cc2\n"
 }