@dxos/random-access-storage 0.4.2 → 0.4.3-main.085f4d3

package/src/browser/web-fs.ts

@@ -13,18 +13,12 @@ import { TimeSeriesCounter, trace } from '@dxos/tracing';
 
 import { Directory, type DiskInfo, type File, type Storage, StorageType, getFullPath } from '../common';
 
-/**
- * When handles weren't shared by different WebFS instances, browser tests
- * were sporadically failing with NonReadableError on test cases like:
- * webFs1.createFile->createWritable->write->close
- * webFs2.createFile->read
- * Presumably due to some handle-level write buffering.
- */
-const files = new Map<string, WebFile>();
 /**
  * Web file systems.
  */
 export class WebFS implements Storage {
+  private readonly _files = new Map<string, WebFile>();
+
   readonly type = StorageType.WEBFS;
 
   protected _root?: FileSystemDirectoryHandle;
@@ -32,13 +26,13 @@ export class WebFS implements Storage {
   constructor(public readonly path: string) {}
 
   public get size() {
-    return files.size;
+    return this._files.size;
   }
 
   private _getFiles(path: string): Map<string, WebFile> {
     const fullName = this._getFullFilename(this.path, path);
     return new Map(
-      [...files.entries()].filter(([path, file]) => {
+      [...this._files.entries()].filter(([path, file]) => {
         return path.includes(fullName) && !file.destroyed;
       }),
     );
@@ -88,19 +82,19 @@ export class WebFS implements Storage {
       getOrCreateFile: (...args) => this.getOrCreateFile(...args),
       remove: () => this._delete(sub),
       onFlush: async () => {
-        await Promise.all(Array.from(this._getFiles(sub)).map(([path, file]) => file.flush()));
+        await Promise.all(Array.from(this._getFiles(sub)).map(([_, file]) => file.flush()));
       },
     });
   }
 
   getOrCreateFile(path: string, filename: string, opts?: any): File {
     const fullName = this._getFullFilename(path, filename);
-    const existingFile = files.get(fullName);
+    const existingFile = this._files.get(fullName);
     if (existingFile) {
       return existingFile;
     }
     const file = this._createFile(fullName);
-    files.set(fullName, file);
+    this._files.set(fullName, file);
     return file;
   }
 
@@ -109,7 +103,7 @@ export class WebFS implements Storage {
       fileName: fullName,
       file: this._initialize().then((root) => root.getFileHandle(fullName, { create: true })),
       destroy: async () => {
-        files.delete(fullName);
+        this._files.delete(fullName);
         const root = await this._initialize();
         return root.removeEntry(fullName);
       },
@@ -120,7 +114,7 @@ export class WebFS implements Storage {
     await Promise.all(
       Array.from(this._getFiles(path)).map(async ([path, file]) => {
         await file.destroy().catch((err: any) => log.warn(err));
-        files.delete(path);
+        this._files.delete(path);
       }),
     );
   }
@@ -128,22 +122,28 @@ export class WebFS implements Storage {
   async reset() {
     await this._initialize();
     for await (const filename of await (this._root as any).keys()) {
-      files.delete(filename);
-      await this._root!.removeEntry(filename, { recursive: true }).catch((err: any) => log.warn(err));
+      await this._root!.removeEntry(filename, { recursive: true }).catch((err: any) =>
+        log.warn('failed to remove an entry', { filename, err }),
+      );
+      this._files.delete(filename);
     }
     this._root = undefined;
   }
 
   async close() {
-    await Promise.all(Array.from(files.values()).map((file) => file.close()));
+    await Promise.all(
+      Array.from(this._files.values()).map((file) => {
+        return file.close().catch((e) => log.warn('failed to close a file', { file: file.fileName, e }));
+      }),
+    );
   }
 
   private _getFullFilename(path: string, filename?: string) {
     // Replace slashes with underscores. Because we can't have slashes in filenames in Browser File Handle API.
     if (filename) {
-      return getFullPath(path, filename).split('/').join('_');
+      return getFullPath(path, filename).replace(/\//g, '_');
     } else {
-      return path.split('/').join('_');
+      return path.replace(/\//g, '_');
     }
   }
 
@@ -158,7 +158,7 @@ export class WebFS implements Storage {
           (async () => {
             switch (entry.kind) {
               case 'file':
-                used += await (entry as FileSystemFileHandle).getFile().then((f) => (used += f.size));
+                used += await (entry as FileSystemFileHandle).getFile().then((f) => f.size);
                 break;
               case 'directory':
                 await recurse(entry as FileSystemDirectoryHandle);
@@ -182,7 +182,7 @@ export class WebFS implements Storage {
 // @trace.resource()
 export class WebFile extends EventEmitter implements File {
   @trace.info()
-  private readonly _fileName: string;
+  readonly fileName: string;
 
   private readonly _fileHandle: Promise<FileSystemFileHandle>;
   private readonly _destroy: () => Promise<void>;
@@ -195,8 +195,12 @@ export class WebFile extends EventEmitter implements File {
   private _loadBufferPromise: Promise<void> | null = null;
 
   private _flushScheduled = false;
-
-  private _flushPromise: Promise<void> | null = null;
+  private _flushPromise: Promise<void> = Promise.resolve();
+  /**
+   * Used to discard unnecessary scheduled flushes.
+   * If _flushNow() is called with a lower sequence number it should early exit.
+   */
+  private _flushSequence = 0;
 
   //
   // Metrics
@@ -235,7 +239,7 @@ export class WebFile extends EventEmitter implements File {
     destroy: () => Promise<void>;
   }) {
     super();
-    this._fileName = fileName;
+    this.fileName = fileName;
     this._fileHandle = file;
     this._destroy = destroy;
 
@@ -284,10 +288,11 @@ export class WebFile extends EventEmitter implements File {
   }
 
   // Do not call directly, use _flushLater or _flushNow.
-  private async _flushCache() {
-    if (this.destroyed) {
+  private async _flushCache(sequence: number) {
+    if (this.destroyed || sequence < this._flushSequence) {
       return;
     }
+    this._flushSequence = sequence + 1;
 
     this._flushes.inc();
 
@@ -305,69 +310,55 @@ export class WebFile extends EventEmitter implements File {
       return;
     }
 
+    const sequence = this._flushSequence;
     setTimeout(async () => {
       // Making sure only one flush can run at a time.
-      const promiseBefore = this._flushPromise;
       await this._flushPromise;
       this._flushScheduled = false;
-
-      // _flushNow might have been called. In that case we don't want to run the flush again.
-      if (promiseBefore !== this._flushPromise) {
-        return;
-      }
-
-      this._flushPromise = this._flushCache().catch((err) => log.warn(err));
+      this._flushPromise = this._flushCache(sequence).catch((err) => log.warn(err));
     });
 
     this._flushScheduled = true;
   }
 
   private async _flushNow() {
-    this._flushPromise = (this._flushPromise ?? Promise.resolve())
-      .then(() => this._flushCache())
-      .catch((err) => log.warn(err));
-
+    await this._flushPromise;
+    this._flushPromise = this._flushCache(this._flushSequence).catch((err) => log.warn(err));
     await this._flushPromise;
   }
 
   async read(offset: number, size: number) {
-    if (this.destroyed) {
-      throw new Error('Read of a destroyed file');
-    }
+    this.assertNotDestroyed('Read');
 
     this._operations.inc();
     this._reads.inc();
     this._readBytes.inc(size);
 
-    if (!this._buffer) {
-      await this._loadBufferGuarded();
-      invariant(this._buffer);
-    }
+    const buffer = await this.requireBuffer();
 
-    if (offset + size > this._buffer.length) {
+    if (offset + size > buffer.length) {
       throw new Error('Read out of bounds');
     }
 
     // Copy data into a new buffer.
-    return Buffer.from(this._buffer.slice(offset, offset + size));
+    return Buffer.from(buffer.slice(offset, offset + size));
   }
 
   async write(offset: number, data: Buffer) {
+    this.assertNotDestroyed('Write');
+
     this._operations.inc();
     this._writes.inc();
     this._writeBytes.inc(data.length);
 
-    if (!this._buffer) {
-      await this._loadBufferGuarded();
-      invariant(this._buffer);
-    }
+    const buffer = await this.requireBuffer();
 
-    if (offset + data.length <= this._buffer.length) {
-      this._buffer.set(data, offset);
+    if (offset + data.length <= buffer.length) {
+      buffer.set(data, offset);
     } else {
       // TODO(dmaretskyi): Optimize re-allocations.
       const newCache = new Uint8Array(offset + data.length);
-      newCache.set(this._buffer);
+      newCache.set(buffer);
       newCache.set(data, offset);
       this._buffer = newCache;
     }
@@ -376,67 +367,86 @@ export class WebFile extends EventEmitter implements File {
   }
 
   async del(offset: number, size: number) {
+    this.assertNotDestroyed('Del');
+
     this._operations.inc();
 
-    if (offset < 0 || size < 0) {
+    if (offset < 0 || size <= 0) {
       return;
     }
 
-    if (!this._buffer) {
-      await this._loadBufferGuarded();
-      invariant(this._buffer);
-    }
+    const buffer = await this.requireBuffer();
 
     let leftoverSize = 0;
-    if (offset + size < this._buffer.length) {
-      leftoverSize = this._buffer.length - (offset + size);
-      this._buffer.set(this._buffer.slice(offset + size, offset + size + leftoverSize), offset);
+    if (offset + size < buffer.length) {
+      leftoverSize = buffer.length - (offset + size);
+      buffer.set(buffer.slice(offset + size, offset + size + leftoverSize), offset);
     }
 
-    this._buffer = this._buffer.slice(0, offset + leftoverSize);
+    this._buffer = buffer.slice(0, offset + leftoverSize);
 
     this._flushLater();
   }
 
   async stat() {
+    this.assertNotDestroyed('Truncate');
+
     this._operations.inc();
 
     // NOTE: This will load all data from the file just to get it's size. While this is a lot of overhead, this works ok for out use cases.
-    if (!this._buffer) {
-      await this._loadBufferGuarded();
-      invariant(this._buffer);
-    }
+    const buffer = await this.requireBuffer();
 
     return {
-      size: this._buffer.length,
+      size: buffer.length,
     };
   }
 
   async truncate(offset: number) {
-    this._operations.inc();
+    this.assertNotDestroyed('Truncate');
 
-    if (!this._buffer) {
-      await this._loadBufferGuarded();
-      invariant(this._buffer);
-    }
+    this._operations.inc();
 
-    this._buffer = this._buffer.slice(0, offset);
+    this._buffer = (await this.requireBuffer()).slice(0, offset);
 
     this._flushLater();
   }
 
   async flush() {
+    this.assertNotDestroyed('Flush');
+
     await this._flushNow();
   }
 
+  /**
+   * It's best to avoid using this method as it doesn't really close a file.
+   * We could update the _opened flag and add a guard like for destroyed, but this would break
+   * the FileSystemFileHandle sharing required for browser tests to run, where writes are
+   * not immediately visible if using different file handles.
+   */
   async close(): Promise<void> {
     await this._flushNow();
   }
 
   @synchronized
   async destroy() {
-    await this._flushNow();
-    this.destroyed = true;
-    return await this._destroy();
+    if (!this.destroyed) {
+      this.destroyed = true;
+      await this._flushNow();
+      return await this._destroy();
+    }
+  }
+
+  private assertNotDestroyed(operation: string) {
+    if (this.destroyed) {
+      throw new Error(`${operation} on a destroyed or closed file`);
+    }
+  }
+
+  private async requireBuffer(): Promise<Uint8Array> {
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer);
+    }
+    return this._buffer;
   }
 }

package/src/common/memory-storage.ts

@@ -8,7 +8,7 @@ import { type Callback, type RandomAccessStorage } from 'random-access-storage';
 import { arrayToBuffer } from '@dxos/util';
 
 import { AbstractStorage } from './abstract-storage';
-import { StorageType } from './storage';
+import { type DiskInfo, StorageType } from './storage';
 
 /**
  * Storage interface implementation for RAM.
@@ -42,4 +42,17 @@ export class MemoryStorage extends AbstractStorage {
 
     return file;
   }
+
+  async getDiskInfo(): Promise<DiskInfo> {
+    let used = 0;
+
+    for (const file of this._files.values()) {
+      const size = (file as any).length;
+      used += Number.isNaN(size) ? 0 : size;
+    }
+
+    return {
+      used,
+    };
+  }
 }

package/src/node/node-storage.ts

@@ -69,11 +69,11 @@ export class NodeStorage extends AbstractStorage implements Storage {
     const recurse = async (path: string) => {
       const pathStats = await stat(path);
 
-      used += pathStats.size;
-
       if (pathStats.isDirectory()) {
         const entries = await readdir(path);
         await Promise.all(entries.map((entry) => recurse(join(path, entry))));
+      } else {
+        used += pathStats.size;
       }
     };
 

package/src/testing/storage.blueprint-test.ts

@@ -2,13 +2,14 @@
 // Copyright 2021 DXOS.org
 //
 
-import { expect, describe, test } from 'vitest';
+import * as uuid from 'uuid';
+import { describe, expect, test } from 'vitest';
 
 import { asyncTimeout } from '@dxos/async';
 
-import { StorageType, type File, type Storage } from '../common';
+import { type File, type Storage, StorageType } from '../common';
 
-export const randomText = () => Math.random().toString(36).substring(2);
+export const randomText = () => uuid.v4();
 
 export const storageTests = (testGroupName: StorageType, createStorage: () => Storage) => {
   const writeAndCheck = async (file: File, data: Buffer, offset = 0) => {
@@ -51,7 +52,7 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const directory = storage.createDirectory(directoryName);
 
       const count = 10;
-      const files = [...Array(count)].map((name) => directory.getOrCreateFile(randomText()));
+      const files = [...Array(count)].map(() => directory.getOrCreateFile(randomText()));
 
       {
         // Create and check files amount.
@@ -173,7 +174,7 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const dir1 = storage.createDirectory('dir1');
       const dir2 = storage.createDirectory('dir2');
 
-      const fileName = 'file';
+      const fileName = randomText();
       const buffer1 = Buffer.from(randomText());
       const buffer2 = Buffer.from(randomText());
 
@@ -195,7 +196,7 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const dir = storage.createDirectory('directory');
       const subDir = dir.createDirectory('subDirectory');
 
-      const file = subDir.getOrCreateFile('file');
+      const file = subDir.getOrCreateFile(randomText());
       const buffer = Buffer.from(randomText());
       await file.write(0, buffer);
 
@@ -204,10 +205,82 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       await file.close();
     });
 
+    test('all writes are flushed', async (t) => {
+      if (testGroupName === StorageType.RAM) {
+        t.skip();
+      }
+
+      const fileName = randomText();
+      {
+        const storage = createStorage();
+        const directory = storage.createDirectory();
+        const file = directory.getOrCreateFile(fileName);
+        await file.write(0, Buffer.alloc(10, '0'));
+        for (let i = 1; i <= 9; i++) {
+          void file.write(i, Buffer.from(String(i)));
+        }
+        await file.close();
+      }
+
+      {
+        const storage = createStorage();
+        const directory = storage.createDirectory();
+        const file = directory.getOrCreateFile(fileName);
+        const allContent = await file.read(0, (await file.stat()).size);
+        expect(allContent.toString()).toBe('0123456789');
+      }
+    });
+
+    test('flush interleaved with write', async (t) => {
+      if (testGroupName === StorageType.RAM) {
+        t.skip();
+      }
+
+      const fileName = randomText();
+      {
+        const storage = createStorage();
+        const directory = storage.createDirectory();
+        const file = directory.getOrCreateFile(fileName);
+        await file.write(0, Buffer.alloc(3, '0'));
+        await file.write(1, Buffer.from('1'));
+        void file.flush?.();
+        await file.write(2, Buffer.from('2'));
+        await file.close();
+      }
+
+      {
+        const storage = createStorage();
+        const directory = storage.createDirectory();
+        const file = directory.getOrCreateFile(fileName);
+        const allContent = await file.read(0, (await file.stat()).size);
+        expect(allContent.toString()).toBe('012');
+      }
+    });
+
+    test('operations on a destroyed file are rejected', async () => {
+      const storage = createStorage();
+      const directory = storage.createDirectory();
+      const file = directory.getOrCreateFile(randomText());
+
+      const buffer = Buffer.from(randomText());
+      await writeAndCheck(file, buffer);
+      await file.destroy();
+
+      expect(file.destroyed).toBeTruthy();
+      await expect(file.destroy()).resolves.toBeUndefined();
+      await expect(file.read(0, 1)).rejects.toThrow();
+      await expect(file.write(0, buffer)).rejects.toThrow();
+      await expect(file.del(0, 1)).rejects.toThrow();
+      await expect(file.stat()).rejects.toThrow();
+      if (file.truncate) {
+        await expect(file.truncate(0)).rejects.toThrow();
+      }
+    });
+
     test('delete directory', async () => {
       const storage = createStorage();
       const directory = storage.createDirectory();
-      const file = directory.getOrCreateFile('file');
+      const file = directory.getOrCreateFile(randomText());
 
       const buffer = Buffer.from(randomText());
       await writeAndCheck(file, buffer);
@@ -364,5 +437,21 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const entries = await directory.list();
       expect(entries).toEqual(expect.arrayContaining(FILES));
     });
+
+    test('getDiskInfo returns correct size', async (t) => {
+      const storage = createStorage();
+      if (storage.type === StorageType.IDB) {
+        t.skip();
+      }
+      await storage.reset();
+      const directory = storage.createDirectory('dir');
+      const file = directory.getOrCreateFile(randomText());
+      const content = 'Hello, world!';
+      await file.write(0, Buffer.from(content));
+      await file.close();
+      await expect(storage.getDiskInfo?.() ?? Promise.resolve(-1)).resolves.toEqual({
+        used: content.length,
+      });
+    });
   });
 };