@zenfs/core 2.7.6 → 2.8.0

package/README.md

@@ -93,3 +93,8 @@ A huge thank you to [deco.cx](https://github.com/deco-cx) for sponsoring ZenFS a
 ## Contact and Support
 
 You can reach out [on Discord](https://zenfs.dev/discord) or by emailing jp@zenfs.dev
+
+## License
+
+[ZenFS](https://github.com/zen-fs/core) is licensed under the [LGPL 3.0 or later](https://www.gnu.org/licenses/lgpl-3.0.html) and [COPYING.md](https://github.com/zen-fs/core/blob/main/COPYING.md).
+Copyright © James Prevett and other ZenFS contributors.

package/dist/backends/index.d.ts

@@ -4,7 +4,9 @@ export * from './fetch.js';
 export * from './memory.js';
 export * from './passthrough.js';
 export * from './port.js';
-export * from './single_buffer.js';
+export * from './single_buffer/backend.js';
+export { SingleBufferStore } from './single_buffer/v1.js';
+export { SingleBufferStore as SingleBufferStoreV2 } from './single_buffer/v2.js';
 export * from './store/fs.js';
 export * from './store/map.js';
 export * from './store/store.js';

package/dist/backends/index.js

@@ -5,7 +5,9 @@ export * from './fetch.js';
 export * from './memory.js';
 export * from './passthrough.js';
 export * from './port.js';
-export * from './single_buffer.js';
+export * from './single_buffer/backend.js';
+export { SingleBufferStore } from './single_buffer/v1.js';
+export { SingleBufferStore as SingleBufferStoreV2 } from './single_buffer/v2.js';
 export * from './store/fs.js';
 export * from './store/map.js';
 export * from './store/store.js';

package/dist/backends/single_buffer/backend.d.ts

@@ -0,0 +1,42 @@
+import { StoreFS } from '../store/fs.js';
+import * as v1 from './v1.js';
+import * as v2 from './v2.js';
+/**
+ * Options for the `SingleBuffer` backend
+ * @category Backends and Configuration
+ */
+export interface SingleBufferOptions {
+    buffer: ArrayBufferLike | ArrayBufferView;
+    /**
+     * Create new file systems with version 2 of the format, and upgrade existing ones to it.
+     * File systems that already use version 2 always do.
+     */
+    v2?: boolean;
+}
+declare const _SingleBuffer: {
+    readonly name: "SingleBuffer";
+    readonly options: {
+        readonly buffer: {
+            readonly type: "object";
+            readonly required: true;
+        };
+        readonly v2: {
+            readonly type: "boolean";
+            readonly required: false;
+        };
+    };
+    readonly create: (opt: SingleBufferOptions) => StoreFS<v1.SingleBufferStore | v2.SingleBufferStore>;
+};
+type _SingleBuffer = typeof _SingleBuffer;
+/**
+ * A backend that uses a single buffer for storing data
+ * @category Backends and Configuration
+ */
+export interface SingleBuffer extends _SingleBuffer {
+}
+/**
+ * A backend that uses a single buffer for storing data
+ * @category Backends and Configuration
+ */
+export declare const SingleBuffer: SingleBuffer;
+export {};

package/dist/backends/single_buffer/backend.js

@@ -0,0 +1,204 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+// SPDX-License-Identifier: LGPL-3.0-or-later
+import { withErrno } from 'kerium';
+import { crit, err, info, warn } from 'kerium/log';
+import { offsetof, sizeof } from 'memium';
+import { Mutex } from 'memium/locks';
+import { crc32c } from 'utilium/checksum';
+import { _inode_version } from '../../internal/inode.js';
+import { StoreFS } from '../store/fs.js';
+import * as v1 from './v1.js';
+import * as v2 from './v2.js';
+const hex = (value) => '0x' + value.toString(16).padStart(8, '0');
+/** The lock held while a file system is created or upgraded, which is in the padding of a version 1 super block */
+const lockOffset = offsetof(v2.SuperBlock, 'lock');
+/** Checksum of a version 1 super block, treating the lock as zero */
+function v1Checksum(bytes) {
+    const data = bytes.slice(4, sizeof(v1.SuperBlock));
+    data.fill(0, lockOffset - 4, lockOffset + 4);
+    return crc32c(data);
+}
+/** Locks a file system while it is created or upgraded. Only shared buffers are locked. */
+function lockSetup(bytes, view) {
+    const word = new BigInt64Array(bytes.buffer, bytes.byteOffset + lockOffset, 1);
+    const garbage = Atomics.load(word, 0);
+    const isV1 = view.getUint32(4, true) == v1.sb_magic && view.getUint16(8, true) < 2;
+    // The padding of a version 1 super block was never cleared
+    if (garbage && isV1 && crc32c(bytes.subarray(4, sizeof(v1.SuperBlock))) == view.getUint32(0, true)) {
+        if (Atomics.compareExchange(word, 0, garbage, 0n) === garbage)
+            view.setUint32(0, v1Checksum(bytes), true);
+    }
+    if (typeof SharedArrayBuffer != 'undefined' && bytes.buffer instanceof SharedArrayBuffer)
+        return new Mutex(bytes.buffer, bytes.byteOffset + lockOffset).lock();
+    Atomics.store(word, 0, 0n);
+}
+/**
+ * Upgrades a version 1 file system in place.
+ * Existing data is not moved, and the version 1 metadata stays intact until the new super block is written.
+ */
+function upgrade(bytes, view) {
+    info('sbfs: upgrading from version 1');
+    const field = (name) => offsetof(v1.SuperBlock, name);
+    if (v1Checksum(bytes) != view.getUint32(0, true))
+        throw crit(withErrno('EIO', 'sbfs: checksum mismatch for version 1 super block'));
+    if (view.getUint16(field('inode_format'), true) != _inode_version)
+        throw crit(withErrno('EIO', 'sbfs: inode format mismatch'));
+    if (view.getUint32(field('metadata_block_size'), true) != sizeof(v1.MetadataBlock))
+        throw crit(withErrno('EIO', 'sbfs: metadata block size mismatch'));
+    const total = Number(view.getBigUint64(field('total_bytes'), true));
+    if (total > bytes.byteLength)
+        throw crit(withErrno('EIO', 'sbfs: invalid size in version 1 super block'));
+    const used = [];
+    const entries = new Map();
+    const blocks = [];
+    // Version 1 metadata block offsets are relative to the buffer, but data offsets are relative to the file system
+    for (let absolute = view.getUint32(field('metadata_offset'), true); absolute;) {
+        const offset = absolute - bytes.byteOffset;
+        if (offset < sizeof(v1.SuperBlock) || offset + sizeof(v1.MetadataBlock) > total || blocks.includes(offset))
+            throw crit(withErrno('EIO', `sbfs: invalid metadata block at ${hex(absolute)}`));
+        const block = new v1.MetadataBlock(bytes.buffer, absolute);
+        if (block.checksum != v1.checksum(block)) {
+            if (!blocks.length)
+                throw crit(withErrno('EIO', 'sbfs: checksum mismatch for metadata block'));
+            warn(`sbfs: checksum mismatch for metadata block at ${hex(absolute)}`);
+        }
+        blocks.push(offset);
+        used.push([offset, offset + sizeof(v1.MetadataBlock)]);
+        for (const entry of block.items) {
+            if (!entry.offset || entries.has(entry.id))
+                continue;
+            if (entry.offset + entry.size > total)
+                throw crit(withErrno('EIO', `sbfs: data for ${hex(entry.id)} is out of bounds`));
+            entries.set(entry.id, [entry.size ? entry.offset : 0, entry.size]);
+            if (entry.size)
+                used.push([entry.offset, entry.offset + entry.size]);
+        }
+        absolute = block.previous_offset;
+    }
+    let maxId = 0;
+    for (const id of entries.keys())
+        maxId = Math.max(maxId, id);
+    const store = new v2.SingleBufferStore(bytes.buffer, bytes.byteOffset, bytes.byteLength, {
+        format: {
+            used,
+            uuid: bytes.slice(field('uuid'), field('uuid') + 16),
+            label: bytes.slice(field('label'), field('label') + 64),
+            nextId: BigInt(Math.max(2, maxId + 1)),
+            populate(tx) {
+                for (const [id, [offset, size]] of entries)
+                    tx.index.set(BigInt(id), Object.assign(new v2.DataExtent(), { offset, size, capacity: size }));
+                for (const block of blocks)
+                    tx.retire(block, sizeof(v1.MetadataBlock));
+            },
+        },
+    });
+    info(`sbfs: upgraded ${entries.size} entries from version 1`);
+    return store;
+}
+const _SingleBuffer = {
+    name: 'SingleBuffer',
+    options: {
+        buffer: { type: 'object', required: true },
+        v2: { type: 'boolean', required: false },
+    },
+    create(opt) {
+        const env_1 = { stack: [], error: void 0, hasError: false };
+        try {
+            const [buffer, byteOffset, byteLength] = ArrayBuffer.isView(opt.buffer)
+                ? [opt.buffer.buffer, opt.buffer.byteOffset, opt.buffer.byteLength]
+                : [opt.buffer, 0, opt.buffer.byteLength];
+            if (byteLength < sizeof(v1.SuperBlock))
+                throw crit(withErrno('EINVAL', 'sbfs: buffer is too small for a file system'));
+            const bytes = new Uint8Array(buffer, byteOffset, byteLength);
+            const view = new DataView(buffer, byteOffset, byteLength);
+            const formatted = view.getUint32(4, true) == v1.sb_magic;
+            const _ = __addDisposableResource(env_1, opt.v2 || (!formatted && opt.v2 !== false) ? lockSetup(bytes, view) : undefined, false);
+            let store;
+            switch (view.getUint32(4, true) == v1.sb_magic ? view.getUint16(8, true) : undefined) {
+                case undefined:
+                    store =
+                        opt.v2 === false
+                            ? new v1.SingleBufferStore(buffer, byteOffset, byteLength)
+                            : new v2.SingleBufferStore(buffer, byteOffset, byteLength, { format: true });
+                    break;
+                case 0:
+                    err('sbfs: got unexpected super block version 0');
+                // falls through
+                case 1:
+                    store = opt.v2 ? upgrade(bytes, view) : new v1.SingleBufferStore(buffer, byteOffset, byteLength);
+                    break;
+                case 2:
+                    store = new v2.SingleBufferStore(buffer, byteOffset, byteLength);
+                    break;
+                default:
+                    throw crit(withErrno('EIO', 'sbfs: unsupported super block version'));
+            }
+            const fs = new StoreFS(store);
+            fs.checkRootSync();
+            return fs;
+        }
+        catch (e_1) {
+            env_1.error = e_1;
+            env_1.hasError = true;
+        }
+        finally {
+            __disposeResources(env_1);
+        }
+    },
+};
+/**
+ * A backend that uses a single buffer for storing data
+ * @category Backends and Configuration
+ */
+export const SingleBuffer = _SingleBuffer;

package/dist/backends/{single_buffer.d.ts → single_buffer/v1.d.ts}

@@ -1,10 +1,10 @@
 import type { ArrayOf } from 'memium';
 import type { UUID } from 'node:crypto';
 import { BufferView } from 'utilium/buffer';
-import type { UsageInfo } from '../internal/filesystem.js';
-import { StoreFS } from './store/fs.js';
-import { SyncMapTransaction, type SyncMapStore } from './store/map.js';
-import type { Store } from './store/store.js';
+import type { UsageInfo } from '../../internal/filesystem.js';
+import type { StoreFS } from '../store/fs.js';
+import { SyncMapTransaction, type SyncMapStore } from '../store/map.js';
+import type { Store } from '../store/store.js';
 type SBLock = Disposable & (() => void);
 declare const MetadataEntry_base: import("memium").StructConstructor<BufferView<any>>;
 declare class MetadataEntry extends MetadataEntry_base {
@@ -54,6 +54,12 @@ export declare class MetadataBlock extends MetadataBlock_base<ArrayBufferLike> {
     waitUnlocked(depth?: number): void;
     lock(): SBLock;
 }
+export declare const sb_magic = 1651715706;
+/**
+ * Shortcut for minor perf. bump
+ * @internal
+ */
+export declare const kUsedBytes = 2;
 declare const SuperBlock_base: import("memium/decorators").StructFromTypedArray<BigUint64Array<ArrayBuffer>>;
 /**
  * The super block structure for a single-buffer file system
@@ -69,7 +75,10 @@ export declare class SuperBlock extends SuperBlock_base<ArrayBufferLike> {
     accessor checksum: number;
     /** Signature for the superblock. */
     accessor magic: number;
-    /** The version of the on-disk format */
+    /**
+     * The version of the on-disk format
+     * 1: v1.10.0+
+     */
     accessor version: number;
     /** Which format of `Inode` is used */
     accessor inode_format: number;
@@ -108,6 +117,16 @@ export declare class SuperBlock extends SuperBlock_base<ArrayBufferLike> {
      */
     isUnused(offset: number, length: number): boolean;
 }
+/**
+ * Compute the checksum for a super block or metadata block.
+ * Note we don't include the checksum when computing a new one.
+ */
+export declare function checksum(value: SuperBlock | MetadataBlock): number;
+/**
+ * Update a block's checksum and timestamp.
+ * @internal @hidden
+ */
+export declare function _update(value: SuperBlock | MetadataBlock): void;
 /**
  *
  * @category Stores and Transactions
@@ -135,33 +154,4 @@ export declare class SingleBufferStore extends BufferView implements SyncMapStor
     usage(): UsageInfo;
     transaction(): SyncMapTransaction;
 }
-/**
- * Options for the `SingleBuffer` backend
- * @category Backends and Configuration
- */
-export interface SingleBufferOptions {
-    buffer: ArrayBufferLike | ArrayBufferView;
-}
-declare const _SingleBuffer: {
-    readonly name: "SingleBuffer";
-    readonly options: {
-        readonly buffer: {
-            readonly type: "object";
-            readonly required: true;
-        };
-    };
-    readonly create: (opt: SingleBufferOptions) => StoreFS<SingleBufferStore>;
-};
-type _SingleBuffer = typeof _SingleBuffer;
-/**
- * A backend that uses a single buffer for storing data
- * @category Backends and Configuration
- */
-export interface SingleBuffer extends _SingleBuffer {
-}
-/**
- * A backend that uses a single buffer for storing data
- * @category Backends and Configuration
- */
-export declare const SingleBuffer: SingleBuffer;
 export {};

package/dist/backends/{single_buffer.js → single_buffer/v1.js}

@@ -92,9 +92,8 @@ import { $from, field, struct, types as t } from 'memium/decorators';
 import { BufferView } from 'utilium/buffer';
 import { crc32c } from 'utilium/checksum';
 import { decodeUUID, encodeUUID } from 'utilium/string';
-import { _inode_version, Inode } from '../internal/inode.js';
-import { StoreFS } from './store/fs.js';
-import { SyncMapTransaction } from './store/map.js';
+import { _inode_version, Inode } from '../../internal/inode.js';
+import { SyncMapTransaction } from '../store/map.js';
 const hex = (value) => '0x' + value.toString(16).padStart(8, '0');
 // eslint-disable-next-line @typescript-eslint/unbound-method
 const { format } = new Intl.NumberFormat('en-US', {
@@ -315,12 +314,12 @@ let MetadataBlock = (() => {
 export { MetadataBlock };
 // workaround for https://github.com/microsoft/TypeScript/issues/61862
 Object.assign(MetadataBlock, { lockIndex: offsetof(MetadataBlock, 'locked') / Int32Array.BYTES_PER_ELEMENT });
-const sb_magic = 0x62732e7a; // 'z.sb'
+export const sb_magic = 0x62732e7a; // 'z.sb'
 /**
  * Shortcut for minor perf. bump
  * @internal
  */
-const kUsedBytes = 2;
+export const kUsedBytes = 2;
 /**
  * The super block structure for a single-buffer file system
  */
@@ -448,7 +447,10 @@ let SuperBlock = (() => {
         get magic() { return this.#magic_accessor_storage; }
         set magic(value) { this.#magic_accessor_storage = value; }
         #version_accessor_storage = (__runInitializers(this, _magic_extraInitializers), __runInitializers(this, _version_initializers, void 0));
-        /** The version of the on-disk format */
+        /**
+         * The version of the on-disk format
+         * 1: v1.10.0+
+         */
         get version() { return this.#version_accessor_storage; }
         set version(value) { this.#version_accessor_storage = value; }
         #inode_format_accessor_storage = (__runInitializers(this, _version_extraInitializers), __runInitializers(this, _inode_format_initializers, void 0));
@@ -572,7 +574,7 @@ const kMetadataOffset = offsetof(SuperBlock, 'metadata_offset') / Uint32Array.BY
  * Compute the checksum for a super block or metadata block.
  * Note we don't include the checksum when computing a new one.
  */
-function checksum(value) {
+export function checksum(value) {
     let length = sizeof(value) - 4;
     if (value instanceof MetadataBlock)
         length -= Int32Array.BYTES_PER_ELEMENT;
@@ -582,7 +584,7 @@ function checksum(value) {
  * Update a block's checksum and timestamp.
  * @internal @hidden
  */
-function _update(value) {
+export function _update(value) {
     if (value instanceof MetadataBlock)
         value.timestamp = BigInt(Date.now());
     value.checksum = checksum(value);
@@ -732,21 +734,3 @@ export class SingleBufferStore extends BufferView {
         return new SyncMapTransaction(this);
     }
 }
-const _SingleBuffer = {
-    name: 'SingleBuffer',
-    options: {
-        buffer: { type: 'object', required: true },
-    },
-    create(opt) {
-        const fs = new StoreFS(ArrayBuffer.isView(opt.buffer)
-            ? new SingleBufferStore(opt.buffer.buffer, opt.buffer.byteOffset, opt.buffer.byteLength)
-            : new SingleBufferStore(opt.buffer));
-        fs.checkRootSync();
-        return fs;
-    },
-};
-/**
- * A backend that uses a single buffer for storing data
- * @category Backends and Configuration
- */
-export const SingleBuffer = _SingleBuffer;