@zenfs/core 0.0.1

package/README.md

@@ -0,0 +1,293 @@
+# ZenFS
+
+ZenFS is an in-browser file system that emulates the [Node JS file system API](http://nodejs.org/api/fs.html) and supports storing and retrieving files from various backends. ZenFS also integrates nicely into the Emscripten file system.
+
+## Backends
+
+ZenFS is highly extensible, and includes many builtin filesystem backends:
+
+-   `InMemory`: Stores files in-memory. It is a temporary file store that clears when the user navigates away.
+-   `OverlayFS`: Mount a read-only file system as read-write by overlaying a writable file system on top of it. Like Docker's overlayfs, it will only write changed files to the writable file system.
+-   `AsyncMirror`: Use an asynchronous backend synchronously. Invaluable for Emscripten; let your Emscripten applications write to larger file stores with no additional effort!
+    -   `AsyncMirror` loads the entire contents of the async file system into a synchronous backend during construction. It performs operations synchronous file system and then queues them to be mirrored onto the asynchronous backend.
+-   `FolderAdapter`: Wraps a file system, and scopes all interactions to a subfolder of that file system.
+
+More backends can be defined by separate libraries, so long as they extend they implement `ZenFS.FileSystem`. Multiple backends can be active at once at different locations in the directory hierarchy.
+
+ZenFS supports a number of other backends (as `@zenfs/fs-[name]`).
+
+For more information, see the [API documentation for ZenFS](https://zen-fs.github.io/core).
+
+## Installing
+
+```sh
+npm install @zenfs/core
+```
+
+## Building
+
+-   Make sure you have Node and NPM installed. You must have Node v18 or newer.
+-   Install dependencies with `npm install`
+-   Build using `npm run build`
+-   You can find the built code in `dist`.
+
+## Usage
+
+> 🛈 The examples are written in ESM. If you are using CJS, you can `require` the package. If running in a borwser you can add a script tag to your HTML pointing to the `browser.min.js` and use ZenFS via the global `ZenFS` object.
+
+```js
+import { fs } from '@zenfs/core';
+
+fs.writeFileSync('/test.txt', 'Cool, I can do this in the browser!');
+
+const contents = fs.readFileSync('/test.txt', 'utf-8');
+console.log(contents);
+```
+
+#### Using different backends
+
+A `InMemory` backend is created by default. If you would like to use a different one, you must configure ZenFS. It is recommended to do so using the `configure` function. Here is an example using the `LocalStorage` backend from `@zenfs/fs-dom`:
+
+```js
+import { configure, fs } from '@zenfs/core';
+import '@zenfs/fs-dom'; // size effects are needed
+
+// you can also add a callback as the last parameter instead of using promises
+await configure({ fs: 'LocalStorage' });
+
+if (!fs.existsSync('/test.txt')) {
+	fs.writeFileSync('/test.txt', 'This will persist across reloads!');
+}
+
+const contents = fs.readFileSync('/test.txt', 'utf-8');
+console.log(contents);
+```
+
+#### Using multiple backends
+
+You can use multiple backends by passing an object to `configure` which maps paths to file systems. The following example mounts a zip file to `/zip`, in-memory storage to `/tmp`, and IndexedDB browser-local storage to `/home` (note that `/` has the default in-memory backend):
+
+```js
+import { configure } from '@zenfs/core';
+import '@zenfs/fs-dom';
+import '@zenfs/fs-zip';
+import Buffer from 'buffer';
+
+const zipData = await (await fetch('mydata.zip')).arrayBuffer();
+
+await configure({
+	'/mnt/zip': {
+		fs: 'ZipFS',
+		options: {
+			zipData: Buffer.from(zipData)
+		}
+	},
+	'/tmp': 'InMemory',
+	'/home': 'IndexedDB',
+};
+```
+
+#### FS Promises API
+
+The FS promises API is exposed as `promises`.
+
+```js
+import { configure, promises } from '@zenfs/core';
+import '@zenfs/fs-dom';
+
+await configure({ '/': 'IndexedDB' });
+
+const exists = await promises.exists('/myfile.txt');
+if (!exists) {
+	await promises.write('/myfile.txt', 'Lots of persistant data');
+}
+```
+
+ZenFS does _not_ provide a seperate method for importing promises in its built form. If you are using Typescript, you can import the promises API from source code (perhaps to reduce you bundle size). Doing so it not recommended as the files may be moved without notice.
+
+#### Using asynchronous backends synchronously
+
+You may have noticed that attempting to use a synchronous method on an asynchronous backend (e.g. IndexedDB) results in a "not supplied" error (`ENOTSUP`). If you wish to use an asynchronous backend synchronously you need to wrap it in an `AsyncMirror`:
+
+```js
+import { configure, fs } from '@zenfs/core';
+import '@zenfs/fs-dom';
+
+await configure({
+	'/': { fs: 'AsyncMirror', options: { sync: { fs: 'InMemory' }, async: { fs: 'IndexedDB' } } }
+});
+
+fs.writeFileSync('/persistant.txt', 'My persistant data'); // This fails if you configure the FS as IndexedDB
+```
+
+### Advanced usage
+
+#### Creating backends
+
+If you would like to create backends without configure, you may do so by importing the backend's class and calling its `Create` method. You can import the backend directly or with `backends`:
+
+```js
+import { configure, backends, InMemory } from '@zenfs/core';
+
+console.log(backends.InMemory === InMemory) // they are the same
+
+const inMemoryFS = await InMemory.Create();
+```
+
+> ⚠ Instances of backends follow the ***internal*** ZenFS API. You should never use a backend's method unless you are extending a backend.
+
+Coming soon:
+```js
+import { configure, InMemory } from '@zenfs/core';
+
+const inMemoryFS = new InMemory();
+await inMemoryFS.whenReady();
+```
+
+#### Mounting
+
+If you would like to mount and unmount backends, you can do so using the `mount` and `umount` functions:
+
+```js
+import { fs, InMemory } from '@zenfs/core';
+
+const inMemoryFS = await InMemory.Create(); // create an FS instance
+
+fs.mount('/tmp', inMemoryFS); // mount
+
+fs.umount('/tmp'); // unmount /tmp
+```
+
+This could be used in the "multiple backends" example like so:
+
+```js
+import { configure, fs, ZipFS } from '@zenfs/core';
+import '@zenfs/fs-dom';
+import '@zenfs/fs-zip';
+import Buffer from 'buffer';
+
+await configure({
+	'/tmp': 'InMemory',
+	'/home': 'IndexedDB',
+};
+
+fs.mkdirSync('/mnt');
+
+const res = await fetch('mydata.zip');
+const zipData = Buffer.from(await res.arrayBuffer());
+const zipFs = await ZipFS.Create({ zipData });
+fs.mount('/mnt/zip', zipFs);
+
+// do stuff with the mounted zip
+
+fs.umount('/mnt/zip'); // finished using the zip
+```
+
+## Using with bundlers
+
+ZenFS exports a drop-in for Node's `fs` module (up to the version of `@types/node` in package.json), so you can use it for your bundler of preference using the default export.
+
+#### ESBuild
+
+tsconfig.json
+
+```json
+{
+	...
+	"paths": {
+		"fs": ["node_modules/zenfs/dist/index.js"]
+	}
+	...
+}
+```
+
+[Why tsconfig.json?](https://stackoverflow.com/a/71935037/17637456)
+
+Webpack:
+
+```js
+module.exports = {
+	// ...
+	resolve: {
+		alias: {
+			fs: require.resolve('zenfs'),
+		},
+	},
+	// ...
+};
+```
+
+Rollup:
+
+```js
+import alias from '@rollup/plugin-alias';
+
+export default {
+	// ...
+	plugins: [
+		alias({
+			entries: [{ find: 'fs', replacement: 'zenfs' }],
+		}),
+	],
+	// ...
+};
+```
+
+## Using with Emscripten
+
+You can use any _synchronous_ ZenFS file systems with Emscripten.
+
+```js
+import { EmscriptenFS } from '@zenfs/fs-emscripten';
+const BFS = new EmscriptenFS(); // Create a ZenFS Emscripten FS plugin.
+FS.createFolder(FS.root, 'data', true, true); // Create the folder that we'll turn into a mount point.
+FS.mount(BFS, { root: '/' }, '/data'); // Mount BFS's root folder into the '/data' folder.
+```
+
+If you want to use an asynchronous backend, you must wrap it in an `AsyncMirror`.
+
+### Testing
+
+Run unit tests with `npm test`.
+
+### Citing
+
+ZenFS is a component of the [Doppio](http://doppiojvm.org/) and [Browsix](https://browsix.org/) research projects from the PLASMA lab at the University of Massachusetts Amherst. If you decide to use ZenFS in a project that leads to a publication, please cite the academic papers on [Doppio](https://dl.acm.org/citation.cfm?doid=2594291.2594293) and [Browsix](https://dl.acm.org/citation.cfm?id=3037727):
+
+> John Vilk and Emery D. Berger. Doppio: Breaking the Browser Language Barrier. In
+> _Proceedings of the 35th ACM SIGPLAN Conference on Programming Language Design and Implementation_
+> (2014), pp. 508–518.
+
+```bibtex
+@inproceedings{VilkDoppio,
+	author		= {John Vilk and
+							 Emery D. Berger},
+	title		 = {{Doppio: Breaking the Browser Language Barrier}},
+	booktitle = {Proceedings of the 35th {ACM} {SIGPLAN} Conference on Programming Language Design and Implementation},
+	pages		 = {508--518},
+	year			= {2014},
+	url			 = {http://doi.acm.org/10.1145/2594291.2594293},
+	doi			 = {10.1145/2594291.2594293}
+}
+```
+
+> Bobby Powers, John Vilk, and Emery D. Berger. Browsix: Bridging the Gap Between Unix and the Browser. In _Proceedings of the Twenty-Second International Conference on Architectural Support for Programming Languages and Operating Systems_ (2017), pp. 253–266.
+
+```bibtex
+@inproceedings{PowersBrowsix,
+	author		= {Bobby Powers and
+							 John Vilk and
+							 Emery D. Berger},
+	title		 = {{Browsix: Bridging the Gap Between Unix and the Browser}},
+	booktitle = {Proceedings of the Twenty-Second International Conference on Architectural
+							 Support for Programming Languages and Operating Systems},
+	pages		 = {253--266},
+	year			= {2017},
+	url			 = {http://doi.acm.org/10.1145/3037697.3037727},
+	doi			 = {10.1145/3037697.3037727}
+}
+```
+
+### License
+
+ZenFS is licensed under the MIT License. See `LICENSE` for details.

package/dist/ApiError.d.ts

@@ -0,0 +1,86 @@
+/// <reference types="node" />
+/**
+ * Standard libc error codes. More will be added to this enum and ErrorStrings as they are
+ * needed.
+ * @url http://www.gnu.org/software/libc/manual/html_node/Error-Codes.html
+ */
+export declare enum ErrorCode {
+    EPERM = 1,
+    ENOENT = 2,
+    EIO = 5,
+    EBADF = 9,
+    EACCES = 13,
+    EBUSY = 16,
+    EEXIST = 17,
+    ENOTDIR = 20,
+    EISDIR = 21,
+    EINVAL = 22,
+    EFBIG = 27,
+    ENOSPC = 28,
+    EROFS = 30,
+    ENOTEMPTY = 39,
+    ENOTSUP = 95
+}
+/**
+ * Strings associated with each error code.
+ * @internal
+ */
+export declare const ErrorStrings: {
+    [code: string | number]: string;
+};
+interface ApiErrorJSON {
+    errno: ErrorCode;
+    message: string;
+    path: string;
+    code: string;
+    stack: string;
+}
+/**
+ * Represents a ZenFS error. Passed back to applications after a failed
+ * call to the ZenFS API.
+ */
+export declare class ApiError extends Error implements NodeJS.ErrnoException {
+    static fromJSON(json: ApiErrorJSON): ApiError;
+    /**
+     * Creates an ApiError object from a buffer.
+     */
+    static fromBuffer(buffer: Buffer, i?: number): ApiError;
+    static FileError(code: ErrorCode, p: string): ApiError;
+    static EACCES(path: string): ApiError;
+    static ENOENT(path: string): ApiError;
+    static EEXIST(path: string): ApiError;
+    static EISDIR(path: string): ApiError;
+    static ENOTDIR(path: string): ApiError;
+    static EPERM(path: string): ApiError;
+    static ENOTEMPTY(path: string): ApiError;
+    errno: ErrorCode;
+    code: string;
+    path?: string;
+    syscall: string;
+    stack?: string;
+    /**
+     * Represents a ZenFS error. Passed back to applications after a failed
+     * call to the ZenFS API.
+     *
+     * Error codes mirror those returned by regular Unix file operations, which is
+     * what Node returns.
+     * @constructor ApiError
+     * @param type The type of the error.
+     * @param [message] A descriptive error message.
+     */
+    constructor(type: ErrorCode, message?: string, path?: string);
+    /**
+     * @return A friendly error message.
+     */
+    toString(): string;
+    toJSON(): any;
+    /**
+     * Writes the API error into a buffer.
+     */
+    writeToBuffer(buffer?: Buffer, i?: number): Buffer;
+    /**
+     * The size of the API error in buffer-form in bytes.
+     */
+    bufferSize(): number;
+}
+export {};

package/dist/ApiError.js

@@ -0,0 +1,135 @@
+import { Buffer } from 'buffer';
+/**
+ * Standard libc error codes. More will be added to this enum and ErrorStrings as they are
+ * needed.
+ * @url http://www.gnu.org/software/libc/manual/html_node/Error-Codes.html
+ */
+export var ErrorCode;
+(function (ErrorCode) {
+    ErrorCode[ErrorCode["EPERM"] = 1] = "EPERM";
+    ErrorCode[ErrorCode["ENOENT"] = 2] = "ENOENT";
+    ErrorCode[ErrorCode["EIO"] = 5] = "EIO";
+    ErrorCode[ErrorCode["EBADF"] = 9] = "EBADF";
+    ErrorCode[ErrorCode["EACCES"] = 13] = "EACCES";
+    ErrorCode[ErrorCode["EBUSY"] = 16] = "EBUSY";
+    ErrorCode[ErrorCode["EEXIST"] = 17] = "EEXIST";
+    ErrorCode[ErrorCode["ENOTDIR"] = 20] = "ENOTDIR";
+    ErrorCode[ErrorCode["EISDIR"] = 21] = "EISDIR";
+    ErrorCode[ErrorCode["EINVAL"] = 22] = "EINVAL";
+    ErrorCode[ErrorCode["EFBIG"] = 27] = "EFBIG";
+    ErrorCode[ErrorCode["ENOSPC"] = 28] = "ENOSPC";
+    ErrorCode[ErrorCode["EROFS"] = 30] = "EROFS";
+    ErrorCode[ErrorCode["ENOTEMPTY"] = 39] = "ENOTEMPTY";
+    ErrorCode[ErrorCode["ENOTSUP"] = 95] = "ENOTSUP";
+})(ErrorCode || (ErrorCode = {}));
+/**
+ * Strings associated with each error code.
+ * @internal
+ */
+export const ErrorStrings = {};
+ErrorStrings[ErrorCode.EPERM] = 'Operation not permitted.';
+ErrorStrings[ErrorCode.ENOENT] = 'No such file or directory.';
+ErrorStrings[ErrorCode.EIO] = 'Input/output error.';
+ErrorStrings[ErrorCode.EBADF] = 'Bad file descriptor.';
+ErrorStrings[ErrorCode.EACCES] = 'Permission denied.';
+ErrorStrings[ErrorCode.EBUSY] = 'Resource busy or locked.';
+ErrorStrings[ErrorCode.EEXIST] = 'File exists.';
+ErrorStrings[ErrorCode.ENOTDIR] = 'File is not a directory.';
+ErrorStrings[ErrorCode.EISDIR] = 'File is a directory.';
+ErrorStrings[ErrorCode.EINVAL] = 'Invalid argument.';
+ErrorStrings[ErrorCode.EFBIG] = 'File is too big.';
+ErrorStrings[ErrorCode.ENOSPC] = 'No space left on disk.';
+ErrorStrings[ErrorCode.EROFS] = 'Cannot modify a read-only file system.';
+ErrorStrings[ErrorCode.ENOTEMPTY] = 'Directory is not empty.';
+ErrorStrings[ErrorCode.ENOTSUP] = 'Operation is not supported.';
+/**
+ * Represents a ZenFS error. Passed back to applications after a failed
+ * call to the ZenFS API.
+ */
+export class ApiError extends Error {
+    static fromJSON(json) {
+        const err = new ApiError(json.errno, json.message, json.path);
+        err.code = json.code;
+        err.stack = json.stack;
+        return err;
+    }
+    /**
+     * Creates an ApiError object from a buffer.
+     */
+    static fromBuffer(buffer, i = 0) {
+        return ApiError.fromJSON(JSON.parse(buffer.toString('utf8', i + 4, i + 4 + buffer.readUInt32LE(i))));
+    }
+    static FileError(code, p) {
+        return new ApiError(code, ErrorStrings[code], p);
+    }
+    static EACCES(path) {
+        return this.FileError(ErrorCode.EACCES, path);
+    }
+    static ENOENT(path) {
+        return this.FileError(ErrorCode.ENOENT, path);
+    }
+    static EEXIST(path) {
+        return this.FileError(ErrorCode.EEXIST, path);
+    }
+    static EISDIR(path) {
+        return this.FileError(ErrorCode.EISDIR, path);
+    }
+    static ENOTDIR(path) {
+        return this.FileError(ErrorCode.ENOTDIR, path);
+    }
+    static EPERM(path) {
+        return this.FileError(ErrorCode.EPERM, path);
+    }
+    static ENOTEMPTY(path) {
+        return this.FileError(ErrorCode.ENOTEMPTY, path);
+    }
+    /**
+     * Represents a ZenFS error. Passed back to applications after a failed
+     * call to the ZenFS API.
+     *
+     * Error codes mirror those returned by regular Unix file operations, which is
+     * what Node returns.
+     * @constructor ApiError
+     * @param type The type of the error.
+     * @param [message] A descriptive error message.
+     */
+    constructor(type, message = ErrorStrings[type], path) {
+        super(message);
+        // Unsupported.
+        this.syscall = '';
+        this.errno = type;
+        this.code = ErrorCode[type];
+        this.path = path;
+        this.message = `Error: ${this.code}: ${message}${this.path ? `, '${this.path}'` : ''}`;
+    }
+    /**
+     * @return A friendly error message.
+     */
+    toString() {
+        return this.message;
+    }
+    toJSON() {
+        return {
+            errno: this.errno,
+            code: this.code,
+            path: this.path,
+            stack: this.stack,
+            message: this.message,
+        };
+    }
+    /**
+     * Writes the API error into a buffer.
+     */
+    writeToBuffer(buffer = Buffer.alloc(this.bufferSize()), i = 0) {
+        const bytesWritten = buffer.write(JSON.stringify(this.toJSON()), i + 4);
+        buffer.writeUInt32LE(bytesWritten, i);
+        return buffer;
+    }
+    /**
+     * The size of the API error in buffer-form in bytes.
+     */
+    bufferSize() {
+        // 4 bytes for string length.
+        return 4 + Buffer.byteLength(JSON.stringify(this.toJSON()));
+    }
+}

package/dist/backends/AsyncMirror.d.ts

@@ -0,0 +1,102 @@
+import { type FileSystem, SynchronousFileSystem, FileSystemMetadata } from '../filesystem';
+import { File, FileFlag, PreloadFile } from '../file';
+import { Stats } from '../stats';
+import { Cred } from '../cred';
+import { type BackendOptions } from './backend';
+export declare namespace AsyncMirror {
+    /**
+     * Configuration options for the AsyncMirror file system.
+     */
+    interface Options {
+        /**
+         * The synchronous file system to mirror the asynchronous file system to.
+         */
+        sync: FileSystem;
+        /**
+         * The asynchronous file system to mirror.
+         */
+        async: FileSystem;
+    }
+}
+/**
+ * AsyncMirrorFS mirrors a synchronous filesystem into an asynchronous filesystem
+ * by:
+ *
+ * * Performing operations over the in-memory copy, while asynchronously pipelining them
+ *   to the backing store.
+ * * During application loading, the contents of the async file system can be reloaded into
+ *   the synchronous store, if desired.
+ *
+ * The two stores will be kept in sync. The most common use-case is to pair a synchronous
+ * in-memory filesystem with an asynchronous backing store.
+ *
+ * Example: Mirroring an IndexedDB file system to an in memory file system. Now, you can use
+ * IndexedDB synchronously.
+ *
+ * ```javascript
+ * ZenFS.configure({
+ *   fs: "AsyncMirror",
+ *   options: {
+ *     sync: { fs: "InMemory" },
+ *     async: { fs: "IndexedDB" }
+ *   }
+ * }, function(e) {
+ *   // ZenFS is initialized and ready-to-use!
+ * });
+ * ```
+ *
+ * Or, alternatively:
+ *
+ * ```javascript
+ * ZenFS.Backend.IndexedDB.Create(function(e, idbfs) {
+ *   ZenFS.Backend.InMemory.Create(function(e, inMemory) {
+ *     ZenFS.Backend.AsyncMirror({
+ *       sync: inMemory, async: idbfs
+ *     }, function(e, mirrored) {
+ *       ZenFS.initialize(mirrored);
+ *     });
+ *   });
+ * });
+ * ```
+ */
+export declare class AsyncMirror extends SynchronousFileSystem {
+    static readonly Name = "AsyncMirror";
+    static Create: any;
+    static readonly Options: BackendOptions;
+    static isAvailable(): boolean;
+    /**
+     * Queue of pending asynchronous operations.
+     */
+    private _queue;
+    private _queueRunning;
+    private _sync;
+    private _async;
+    private _isInitialized;
+    private _initializeCallbacks;
+    /**
+     *
+     * Mirrors the synchronous file system into the asynchronous file system.
+     *
+     * @param sync The synchronous file system to mirror the asynchronous file system to.
+     * @param async The asynchronous file system to mirror.
+     */
+    constructor({ sync, async }: AsyncMirror.Options);
+    get metadata(): FileSystemMetadata;
+    _syncSync(fd: PreloadFile<AsyncMirror>): void;
+    renameSync(oldPath: string, newPath: string, cred: Cred): void;
+    statSync(p: string, cred: Cred): Stats;
+    openSync(p: string, flag: FileFlag, mode: number, cred: Cred): File;
+    unlinkSync(p: string, cred: Cred): void;
+    rmdirSync(p: string, cred: Cred): void;
+    mkdirSync(p: string, mode: number, cred: Cred): void;
+    readdirSync(p: string, cred: Cred): string[];
+    existsSync(p: string, cred: Cred): boolean;
+    chmodSync(p: string, mode: number, cred: Cred): void;
+    chownSync(p: string, new_uid: number, new_gid: number, cred: Cred): void;
+    utimesSync(p: string, atime: Date, mtime: Date, cred: Cred): void;
+    /**
+     * Called once to load up files from async storage into sync storage.
+     */
+    private _initialize;
+    private enqueueOp;
+}