@zenfs/core 1.10.1 → 1.10.3

package/dist/backends/{overlay.d.ts → cow.d.ts}

@@ -1,42 +1,75 @@
 import type { File } from '../internal/file.js';
 import type { CreationOptions, UsageInfo } from '../internal/filesystem.js';
-import type { Stats } from '../stats.js';
 import type { InodeLike } from '../internal/inode.js';
+import type { Stats } from '../stats.js';
 import { FileSystem } from '../internal/filesystem.js';
+import { EventEmitter } from 'eventemitter3';
 /**
- * Configuration options for OverlayFS instances.
+ * Configuration options for CoW.
  * @category Backends and Configuration
  */
-export interface OverlayOptions {
-    /**
-     * The file system to write modified files to.
-     */
+export interface CopyOnWriteOptions {
+    /** The file system that initially populates this file system. */
+    readable: FileSystem;
+    /** The file system to write modified files to. */
     writable: FileSystem;
+    /** @see {@link Journal} */
+    journal?: Journal;
+}
+/**
+ *  @hidden @deprecated use `CopyOnWriteOptions`
+ */
+export type OverlayOptions = CopyOnWriteOptions;
+declare const journalOperations: readonly ["delete"];
+/**
+ * @internal
+ */
+export type JournalOperation = (typeof journalOperations)[number];
+/**
+ * @internal
+ */
+export interface JournalEntry {
+    path: string;
+    op: JournalOperation;
+}
+/**
+ * Tracks various operations for the CoW backend
+ * @internal
+ */
+export declare class Journal extends EventEmitter<{
+    update: [op: JournalOperation, path: string];
+    delete: [path: string];
+}> {
+    protected entries: JournalEntry[];
+    toString(): string;
     /**
-     * The file system that initially populates this file system.
+     * Parse a journal from a string
      */
-    readable: FileSystem;
+    fromString(value: string): this;
+    add(op: JournalOperation, path: string): void;
+    has(op: JournalOperation, path: string): boolean;
+    isDeleted(path: string): boolean;
 }
 /**
- * OverlayFS makes a read-only filesystem writable by storing writes on a second, writable file system.
- * Deletes are persisted via metadata stored on the writable file system.
- *
- * This class contains no locking whatsoever. It is mutexed to prevent races.
- *
+ * Using a readable file system as a base, writes are done to a writable file system.
  * @internal
  */
-export declare class OverlayFS extends FileSystem {
-    ready(): Promise<void>;
-    readonly writable: FileSystem;
+export declare class CopyOnWriteFS extends FileSystem {
+    /** The file system that initially populates this file system. */
     readonly readable: FileSystem;
-    private _isInitialized;
-    private _deletedFiles;
-    private _deleteLog;
-    private _deleteLogUpdatePending;
-    private _deleteLogUpdateNeeded;
-    private _deleteLogError?;
-    private _ready;
-    constructor({ writable, readable }: OverlayOptions);
+    /** The file system to write modified files to. */
+    readonly writable: FileSystem;
+    /** The journal to use for persisting deletions */
+    readonly journal: Journal;
+    ready(): Promise<void>;
+    constructor(
+    /** The file system that initially populates this file system. */
+    readable: FileSystem, 
+    /** The file system to write modified files to. */
+    writable: FileSystem, 
+    /** The journal to use for persisting deletions */
+    journal?: Journal);
+    isDeleted(path: string): boolean;
     /**
      * @todo Consider trying to track information on the writable as well
      */
@@ -47,13 +80,6 @@ export declare class OverlayFS extends FileSystem {
     readSync(path: string, buffer: Uint8Array, offset: number, end: number): void;
     write(path: string, buffer: Uint8Array, offset: number): Promise<void>;
     writeSync(path: string, buffer: Uint8Array, offset: number): void;
-    /**
-     * Called once to load up metadata stored on the writable file system.
-     * @internal
-     */
-    _initialize(): Promise<void>;
-    getDeletionLog(): string;
-    restoreDeletionLog(log: string): Promise<void>;
     rename(oldPath: string, newPath: string): Promise<void>;
     renameSync(oldPath: string, newPath: string): void;
     stat(path: string): Promise<Stats>;
@@ -72,11 +98,6 @@ export declare class OverlayFS extends FileSystem {
     mkdirSync(path: string, mode: number, options: CreationOptions): void;
     readdir(path: string): Promise<string[]>;
     readdirSync(path: string): string[];
-    private deletePath;
-    private updateLog;
-    private _reparseDeletionLog;
-    private checkInitialized;
-    private checkPath;
     /**
      * Create the needed parent directories on the writable storage should they not exist.
      * Use modes from the read-only storage.
@@ -101,8 +122,13 @@ export declare class OverlayFS extends FileSystem {
     private copyToWritableSync;
     private copyToWritable;
 }
-declare const _Overlay: {
-    readonly name: "Overlay";
+/**
+ * @hidden @deprecated use `CopyOnWriteFS`
+ */
+export declare class OverlayFS extends CopyOnWriteFS {
+}
+declare const _CopyOnWrite: {
+    readonly name: "CopyOnWrite";
     readonly options: {
         readonly writable: {
             readonly type: "object";
@@ -112,11 +138,15 @@ declare const _Overlay: {
             readonly type: "object";
             readonly required: true;
         };
+        readonly journal: {
+            readonly type: "object";
+            readonly required: false;
+        };
     };
-    readonly create: (options: OverlayOptions) => OverlayFS;
+    readonly create: (options: CopyOnWriteOptions) => CopyOnWriteFS;
 };
-type _Overlay = typeof _Overlay;
-export interface Overlay extends _Overlay {
+type _CopyOnWrite = typeof _CopyOnWrite;
+export interface CopyOnWrite extends _CopyOnWrite {
 }
 /**
  * Overlay makes a read-only filesystem writable by storing writes on a second, writable file system.
@@ -124,5 +154,13 @@ export interface Overlay extends _Overlay {
  * @category Backends and Configuration
  * @internal
  */
+export declare const CopyOnWrite: CopyOnWrite;
+export interface Overlay extends _CopyOnWrite {
+}
+/**
+ * @deprecated Use `CopyOnWrite`
+ * @category Backends and Configuration
+ * @internal @hidden
+ */
 export declare const Overlay: Overlay;
 export {};

package/dist/backends/{overlay.js → cow.js}

@@ -52,43 +52,100 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
 });
 import { canary } from 'utilium';
 import { Errno, ErrnoError } from '../internal/error.js';
-import { LazyFile, parseFlag } from '../internal/file.js';
+import { LazyFile } from '../internal/file.js';
 import { FileSystem } from '../internal/filesystem.js';
-import { crit, err, info } from '../internal/log.js';
-import { decodeUTF8, encodeUTF8 } from '../utils.js';
+import { debug, err, warn } from '../internal/log.js';
 import { dirname, join } from '../vfs/path.js';
-/** @internal */
-const deletionLogPath = '/.deleted';
+import { EventEmitter } from 'eventemitter3';
+const journalOperations = ['delete'];
+/** Because TS doesn't work right w/o it */
+function isJournalOp(op) {
+    return journalOperations.includes(op);
+}
+const maxOpLength = Math.max(...journalOperations.map(op => op.length));
+const journalMagicString = '#journal@v0\n';
 /**
- * OverlayFS makes a read-only filesystem writable by storing writes on a second, writable file system.
- * Deletes are persisted via metadata stored on the writable file system.
- *
- * This class contains no locking whatsoever. It is mutexed to prevent races.
- *
+ * Tracks various operations for the CoW backend
  * @internal
  */
-export class OverlayFS extends FileSystem {
+export class Journal extends EventEmitter {
+    constructor() {
+        super(...arguments);
+        this.entries = [];
+    }
+    toString() {
+        return journalMagicString + this.entries.map(entry => `${entry.op.padEnd(maxOpLength)} ${entry.path}`).join('\n');
+    }
+    /**
+     * Parse a journal from a string
+     */
+    fromString(value) {
+        if (!value.startsWith(journalMagicString))
+            throw err(new ErrnoError(Errno.EINVAL, 'Invalid journal contents, refusing to parse'));
+        for (const line of value.split('\n')) {
+            if (line.startsWith('#'))
+                continue; // ignore comments
+            const [op, path] = line.split(/\s+/);
+            if (!isJournalOp(op)) {
+                warn('Unknown operation in journal (skipping): ' + op);
+                continue;
+            }
+            this.entries.push({ op, path });
+        }
+        return this;
+    }
+    add(op, path) {
+        this.entries.push({ op, path });
+        this.emit('update', op, path);
+        this.emit(op, path);
+    }
+    has(op, path) {
+        const test = JSON.stringify({ op, path });
+        for (const entry of this.entries)
+            if (JSON.stringify(entry) === test)
+                return true;
+        return false;
+    }
+    isDeleted(path) {
+        let deleted = false;
+        for (const entry of this.entries) {
+            if (entry.path != path)
+                continue;
+            switch (entry.op) {
+                case 'delete':
+                    deleted = true;
+            }
+        }
+        return deleted;
+    }
+}
+/**
+ * Using a readable file system as a base, writes are done to a writable file system.
+ * @internal
+ */
+export class CopyOnWriteFS extends FileSystem {
     async ready() {
         await this.readable.ready();
         await this.writable.ready();
-        await this._ready;
     }
-    constructor({ writable, readable }) {
+    constructor(
+    /** The file system that initially populates this file system. */
+    readable, 
+    /** The file system to write modified files to. */
+    writable, 
+    /** The journal to use for persisting deletions */
+    journal = new Journal()) {
         super(0x62756c6c, readable.name);
-        this._isInitialized = false;
-        this._deletedFiles = new Set();
-        this._deleteLog = '';
-        // If 'true', we have scheduled a delete log update.
-        this._deleteLogUpdatePending = false;
-        // If 'true', a delete log update is needed after the scheduled delete log
-        // update finishes.
-        this._deleteLogUpdateNeeded = false;
-        this.writable = writable;
         this.readable = readable;
-        if (this.writable.attributes.has('no_write')) {
+        this.writable = writable;
+        this.journal = journal;
+        if (writable.attributes.has('no_write')) {
             throw err(new ErrnoError(Errno.EINVAL, 'Writable file system can not be written to'));
         }
-        this._ready = this._initialize();
+        readable.attributes.set('no_write');
+    }
+    isDeleted(path) {
+        return this.journal.isDeleted(path);
     }
     /**
      * @todo Consider trying to track information on the writable as well
@@ -120,74 +177,35 @@ export class OverlayFS extends FileSystem {
         this.copyForWriteSync(path);
         return this.writable.writeSync(path, buffer, offset);
     }
-    /**
-     * Called once to load up metadata stored on the writable file system.
-     * @internal
-     */
-    async _initialize() {
-        if (this._isInitialized) {
-            return;
-        }
-        // Read deletion log, process into metadata.
-        try {
-            const file = await this.writable.openFile(deletionLogPath, parseFlag('r'));
-            const { size } = await file.stat();
-            const { buffer } = await file.read(new Uint8Array(size));
-            this._deleteLog = decodeUTF8(buffer);
-        }
-        catch (error) {
-            if (error.errno !== Errno.ENOENT)
-                throw err(error);
-            info('Overlay does not have a deletion log');
-        }
-        this._isInitialized = true;
-        this._reparseDeletionLog();
-    }
-    getDeletionLog() {
-        return this._deleteLog;
-    }
-    async restoreDeletionLog(log) {
-        this._deleteLog = log;
-        this._reparseDeletionLog();
-        await this.updateLog('');
-    }
     async rename(oldPath, newPath) {
-        this.checkInitialized();
-        this.checkPath(oldPath);
-        this.checkPath(newPath);
         await this.copyForWrite(oldPath);
         try {
             await this.writable.rename(oldPath, newPath);
         }
         catch {
-            if (this._deletedFiles.has(oldPath)) {
+            if (this.isDeleted(oldPath)) {
                 throw ErrnoError.With('ENOENT', oldPath, 'rename');
             }
         }
     }
     renameSync(oldPath, newPath) {
-        this.checkInitialized();
-        this.checkPath(oldPath);
-        this.checkPath(newPath);
         this.copyForWriteSync(oldPath);
         try {
             this.writable.renameSync(oldPath, newPath);
         }
         catch {
-            if (this._deletedFiles.has(oldPath)) {
+            if (this.isDeleted(oldPath)) {
                 throw ErrnoError.With('ENOENT', oldPath, 'rename');
             }
         }
     }
     async stat(path) {
-        this.checkInitialized();
         try {
             return await this.writable.stat(path);
         }
         catch {
-            if (this._deletedFiles.has(path)) {
+            if (this.isDeleted(path))
                 throw ErrnoError.With('ENOENT', path, 'stat');
-            }
             const oldStat = await this.readable.stat(path);
             // Make the oldStat's mode writable.
             oldStat.mode |= 0o222;
@@ -195,14 +213,12 @@ export class OverlayFS extends FileSystem {
         }
     }
     statSync(path) {
-        this.checkInitialized();
         try {
             return this.writable.statSync(path);
         }
         catch {
-            if (this._deletedFiles.has(path)) {
+            if (this.isDeleted(path))
                 throw ErrnoError.With('ENOENT', path, 'stat');
-            }
             const oldStat = this.readable.statSync(path);
             // Make the oldStat's mode writable.
             oldStat.mode |= 0o222;
@@ -224,28 +240,22 @@ export class OverlayFS extends FileSystem {
         return new LazyFile(this, path, flag, stats);
     }
     async createFile(path, flag, mode, options) {
-        this.checkInitialized();
         await this.writable.createFile(path, flag, mode, options);
         return this.openFile(path, flag);
     }
     createFileSync(path, flag, mode, options) {
-        this.checkInitialized();
         this.writable.createFileSync(path, flag, mode, options);
         return this.openFileSync(path, flag);
     }
     async link(srcpath, dstpath) {
-        this.checkInitialized();
         await this.copyForWrite(srcpath);
         await this.writable.link(srcpath, dstpath);
     }
     linkSync(srcpath, dstpath) {
-        this.checkInitialized();
         this.copyForWriteSync(srcpath);
         this.writable.linkSync(srcpath, dstpath);
     }
     async unlink(path) {
-        this.checkInitialized();
-        this.checkPath(path);
         if (!(await this.exists(path))) {
             throw ErrnoError.With('ENOENT', path, 'unlink');
         }
@@ -254,25 +264,21 @@ export class OverlayFS extends FileSystem {
         }
         // if it still exists add to the delete log
         if (await this.exists(path)) {
-            await this.deletePath(path);
+            this.journal.add('delete', path);
         }
     }
     unlinkSync(path) {
-        this.checkInitialized();
-        this.checkPath(path);
-        if (!this.existsSync(path)) {
+        if (!this.existsSync(path))
             throw ErrnoError.With('ENOENT', path, 'unlink');
-        }
         if (this.writable.existsSync(path)) {
             this.writable.unlinkSync(path);
         }
         // if it still exists add to the delete log
         if (this.existsSync(path)) {
-            void this.deletePath(path);
+            this.journal.add('delete', path);
         }
     }
     async rmdir(path) {
-        this.checkInitialized();
         if (!(await this.exists(path))) {
             throw ErrnoError.With('ENOENT', path, 'rmdir');
         }
@@ -286,10 +292,9 @@ export class OverlayFS extends FileSystem {
         if ((await this.readdir(path)).length) {
             throw ErrnoError.With('ENOTEMPTY', path, 'rmdir');
         }
-        await this.deletePath(path);
+        this.journal.add('delete', path);
     }
     rmdirSync(path) {
-        this.checkInitialized();
         if (!this.existsSync(path)) {
             throw ErrnoError.With('ENOENT', path, 'rmdir');
         }
@@ -303,137 +308,55 @@ export class OverlayFS extends FileSystem {
         if (this.readdirSync(path).length) {
             throw ErrnoError.With('ENOTEMPTY', path, 'rmdir');
         }
-        void this.deletePath(path);
+        this.journal.add('delete', path);
     }
     async mkdir(path, mode, options) {
-        this.checkInitialized();
-        if (await this.exists(path)) {
+        if (await this.exists(path))
             throw ErrnoError.With('EEXIST', path, 'mkdir');
-        }
-        // The below will throw should any of the parent directories fail to exist on _writable.
         await this.createParentDirectories(path);
         await this.writable.mkdir(path, mode, options);
     }
     mkdirSync(path, mode, options) {
-        this.checkInitialized();
-        if (this.existsSync(path)) {
+        if (this.existsSync(path))
             throw ErrnoError.With('EEXIST', path, 'mkdir');
-        }
-        // The below will throw should any of the parent directories fail to exist on _writable.
         this.createParentDirectoriesSync(path);
         this.writable.mkdirSync(path, mode, options);
     }
     async readdir(path) {
-        this.checkInitialized();
-        // Readdir in both, check delete log on RO file system's listing, merge, return.
-        const contents = [];
-        try {
-            contents.push(...(await this.writable.readdir(path)));
-        }
-        catch {
-            // NOP.
-        }
-        try {
-            contents.push(...(await this.readable.readdir(path)).filter((fPath) => !this._deletedFiles.has(`${path}/${fPath}`)));
-        }
-        catch {
-            // NOP.
-        }
-        const seenMap = {};
-        return contents.filter((path) => {
-            const result = !seenMap[path];
-            seenMap[path] = true;
-            return result;
-        });
-    }
-    readdirSync(path) {
-        this.checkInitialized();
-        // Readdir in both, check delete log on RO file system's listing, merge, return.
-        let contents = [];
-        try {
-            contents = contents.concat(this.writable.readdirSync(path));
-        }
-        catch {
-            // NOP.
-        }
-        try {
-            contents = contents.concat(this.readable.readdirSync(path).filter((fPath) => !this._deletedFiles.has(`${path}/${fPath}`)));
-        }
-        catch {
-            // NOP.
-        }
-        const seenMap = {};
-        return contents.filter((path) => {
-            const result = !seenMap[path];
-            seenMap[path] = true;
-            return result;
-        });
-    }
-    async deletePath(path) {
-        this._deletedFiles.add(path);
-        await this.updateLog(`d${path}\n`);
-    }
-    async updateLog(addition) {
-        this._deleteLog += addition;
-        if (this._deleteLogUpdatePending) {
-            this._deleteLogUpdateNeeded = true;
-            return;
-        }
-        this._deleteLogUpdatePending = true;
-        const log = await this.writable.openFile(deletionLogPath, parseFlag('w'));
-        try {
-            await log.write(encodeUTF8(this._deleteLog));
-            if (this._deleteLogUpdateNeeded) {
-                this._deleteLogUpdateNeeded = false;
-                await this.updateLog('');
+        if (this.isDeleted(path))
+            throw ErrnoError.With('ENOENT', path, 'readdir');
+        const entries = await this.writable.readdir(path);
+        if (await this.writable.exists(path))
+            for (const entry of await this.writable.readdir(path)) {
+                if (!entries.includes(entry))
+                    entries.push(entry);
             }
-        }
-        catch (e) {
-            this._deleteLogError = e;
-        }
-        finally {
-            this._deleteLogUpdatePending = false;
-        }
+        return entries.filter(entry => !this.isDeleted(join(path, entry)));
     }
-    _reparseDeletionLog() {
-        this._deletedFiles.clear();
-        for (const entry of this._deleteLog.split('\n')) {
-            if (!entry.startsWith('d')) {
-                continue;
+    readdirSync(path) {
+        if (this.isDeleted(path))
+            throw ErrnoError.With('ENOENT', path, 'readdir');
+        const entries = this.writable.readdirSync(path);
+        if (this.writable.existsSync(path))
+            for (const entry of this.writable.readdirSync(path)) {
+                if (!entries.includes(entry))
+                    entries.push(entry);
             }
-            // If the log entry begins w/ 'd', it's a deletion.
-            this._deletedFiles.add(entry.slice(1));
-        }
-    }
-    checkInitialized() {
-        if (!this._isInitialized) {
-            throw crit(new ErrnoError(Errno.EPERM, 'Overlay is not initialized'), { fs: this });
-        }
-        if (!this._deleteLogError) {
-            return;
-        }
-        const error = this._deleteLogError;
-        delete this._deleteLogError;
-        throw error;
-    }
-    checkPath(path) {
-        if (path == deletionLogPath) {
-            throw ErrnoError.With('EPERM', path, 'checkPath');
-        }
+        return entries.filter(entry => !this.isDeleted(join(path, entry)));
     }
     /**
      * Create the needed parent directories on the writable storage should they not exist.
      * Use modes from the read-only storage.
      */
     createParentDirectoriesSync(path) {
-        let parent = dirname(path);
         const toCreate = [];
         const silence = canary(ErrnoError.With('EDEADLK', path));
-        while (!this.writable.existsSync(parent)) {
+        for (let parent = dirname(path); !this.writable.existsSync(parent); parent = dirname(parent)) {
             toCreate.push(parent);
-            parent = dirname(parent);
         }
         silence();
+        if (toCreate.length)
+            debug('COW: Creating parent directories: ' + toCreate.join(', '));
         for (const path of toCreate.reverse()) {
             const { uid, gid, mode } = this.statSync(path);
             this.writable.mkdirSync(path, mode, { uid, gid });
@@ -444,14 +367,14 @@ export class OverlayFS extends FileSystem {
      * Use modes from the read-only storage.
      */
     async createParentDirectories(path) {
-        let parent = dirname(path);
         const toCreate = [];
         const silence = canary(ErrnoError.With('EDEADLK', path));
-        while (!(await this.writable.exists(parent))) {
+        for (let parent = dirname(path); !(await this.writable.exists(parent)); parent = dirname(parent)) {
             toCreate.push(parent);
-            parent = dirname(parent);
         }
         silence();
+        if (toCreate.length)
+            debug('COW: Creating parent directories: ' + toCreate.join(', '));
         for (const path of toCreate.reverse()) {
             const { uid, gid, mode } = await this.stat(path);
             await this.writable.mkdir(path, mode, { uid, gid });
@@ -543,14 +466,20 @@ export class OverlayFS extends FileSystem {
         }
     }
 }
-const _Overlay = {
-    name: 'Overlay',
+/**
+ * @hidden @deprecated use `CopyOnWriteFS`
+ */
+export class OverlayFS extends CopyOnWriteFS {
+}
+const _CopyOnWrite = {
+    name: 'CopyOnWrite',
     options: {
         writable: { type: 'object', required: true },
         readable: { type: 'object', required: true },
+        journal: { type: 'object', required: false },
     },
     create(options) {
-        return new OverlayFS(options);
+        return new CopyOnWriteFS(options.readable, options.writable, options.journal);
     },
 };
 /**
@@ -559,4 +488,10 @@ const _Overlay = {
  * @category Backends and Configuration
  * @internal
  */
-export const Overlay = _Overlay;
+export const CopyOnWrite = _CopyOnWrite;
+/**
+ * @deprecated Use `CopyOnWrite`
+ * @category Backends and Configuration
+ * @internal @hidden
+ */
+export const Overlay = _CopyOnWrite;

package/dist/backends/index.d.ts

@@ -1,7 +1,7 @@
 export * from './backend.js';
+export * from './cow.js';
 export * from './fetch.js';
 export * from './memory.js';
-export * from './overlay.js';
 export * from './passthrough.js';
 export * from './port/fs.js';
 export * from './single_buffer.js';