@dxos/random-access-storage 0.4.3-main.dda98d3 → 0.4.3-main.e5bbcda

package/src/browser/web-fs.ts

@@ -13,12 +13,18 @@ import { TimeSeriesCounter, trace } from '@dxos/tracing';
 
 import { Directory, type DiskInfo, type File, type Storage, StorageType, getFullPath } from '../common';
 
+/**
+ * When handles weren't shared by different WebFS instances, browser tests
+ * were sporadically failing with NonReadableError on test cases like:
+ * webFs1.createFile->createWritable->write->close
+ * webFs2.createFile->read
+ * Presumably due to some handle-level write buffering.
+ */
+const files = new Map<string, WebFile>();
 /**
  * Web file systems.
  */
 export class WebFS implements Storage {
-  private readonly _files = new Map<string, WebFile>();
-
   readonly type = StorageType.WEBFS;
 
   protected _root?: FileSystemDirectoryHandle;
@@ -26,13 +32,13 @@ export class WebFS implements Storage {
   constructor(public readonly path: string) {}
 
   public get size() {
-    return this._files.size;
+    return files.size;
   }
 
   private _getFiles(path: string): Map<string, WebFile> {
     const fullName = this._getFullFilename(this.path, path);
     return new Map(
-      [...this._files.entries()].filter(([path, file]) => {
+      [...files.entries()].filter(([path, file]) => {
         return path.includes(fullName) && !file.destroyed;
       }),
     );
@@ -82,19 +88,19 @@ export class WebFS implements Storage {
       getOrCreateFile: (...args) => this.getOrCreateFile(...args),
       remove: () => this._delete(sub),
       onFlush: async () => {
-        await Promise.all(Array.from(this._getFiles(sub)).map(([_, file]) => file.flush()));
+        await Promise.all(Array.from(this._getFiles(sub)).map(([path, file]) => file.flush()));
       },
     });
   }
 
   getOrCreateFile(path: string, filename: string, opts?: any): File {
     const fullName = this._getFullFilename(path, filename);
-    const existingFile = this._files.get(fullName);
+    const existingFile = files.get(fullName);
     if (existingFile) {
       return existingFile;
     }
     const file = this._createFile(fullName);
-    this._files.set(fullName, file);
+    files.set(fullName, file);
     return file;
   }
 
@@ -103,7 +109,7 @@ export class WebFS implements Storage {
       fileName: fullName,
       file: this._initialize().then((root) => root.getFileHandle(fullName, { create: true })),
       destroy: async () => {
-        this._files.delete(fullName);
+        files.delete(fullName);
         const root = await this._initialize();
         return root.removeEntry(fullName);
       },
@@ -114,7 +120,7 @@ export class WebFS implements Storage {
     await Promise.all(
       Array.from(this._getFiles(path)).map(async ([path, file]) => {
         await file.destroy().catch((err: any) => log.warn(err));
-        this._files.delete(path);
+        files.delete(path);
       }),
     );
   }
@@ -122,28 +128,22 @@ export class WebFS implements Storage {
   async reset() {
     await this._initialize();
     for await (const filename of await (this._root as any).keys()) {
-      await this._root!.removeEntry(filename, { recursive: true }).catch((err: any) =>
-        log.warn('failed to remove an entry', { filename, err }),
-      );
-      this._files.delete(filename);
+      files.delete(filename);
+      await this._root!.removeEntry(filename, { recursive: true }).catch((err: any) => log.warn(err));
     }
     this._root = undefined;
   }
 
   async close() {
-    await Promise.all(
-      Array.from(this._files.values()).map((file) => {
-        return file.close().catch((e) => log.warn('failed to close a file', { file: file.fileName, e }));
-      }),
-    );
+    await Promise.all(Array.from(files.values()).map((file) => file.close()));
   }
 
   private _getFullFilename(path: string, filename?: string) {
     // Replace slashes with underscores. Because we can't have slashes in filenames in Browser File Handle API.
     if (filename) {
-      return getFullPath(path, filename).replace(/\//g, '_');
+      return getFullPath(path, filename).split('/').join('_');
     } else {
-      return path.replace(/\//g, '_');
+      return path.split('/').join('_');
     }
   }
 
@@ -158,7 +158,7 @@ export class WebFS implements Storage {
           (async () => {
             switch (entry.kind) {
               case 'file':
-                used += await (entry as FileSystemFileHandle).getFile().then((f) => f.size);
+                used += await (entry as FileSystemFileHandle).getFile().then((f) => (used += f.size));
                 break;
               case 'directory':
                 await recurse(entry as FileSystemDirectoryHandle);
@@ -182,7 +182,7 @@ export class WebFS implements Storage {
 // @trace.resource()
 export class WebFile extends EventEmitter implements File {
   @trace.info()
-  readonly fileName: string;
+  private readonly _fileName: string;
 
   private readonly _fileHandle: Promise<FileSystemFileHandle>;
   private readonly _destroy: () => Promise<void>;
@@ -195,12 +195,8 @@ export class WebFile extends EventEmitter implements File {
   private _loadBufferPromise: Promise<void> | null = null;
 
   private _flushScheduled = false;
-  private _flushPromise: Promise<void> = Promise.resolve();
-  /**
-   * Used to discard unnecessary scheduled flushes.
-   * If _flushNow() is called with a lower sequence number it should early exit.
-   */
-  private _flushSequence = 0;
+
+  private _flushPromise: Promise<void> | null = null;
 
   //
   // Metrics
@@ -239,7 +235,7 @@ export class WebFile extends EventEmitter implements File {
     destroy: () => Promise<void>;
   }) {
     super();
-    this.fileName = fileName;
+    this._fileName = fileName;
     this._fileHandle = file;
     this._destroy = destroy;
 
@@ -288,11 +284,10 @@ export class WebFile extends EventEmitter implements File {
   }
 
   // Do not call directly, use _flushLater or _flushNow.
-  private async _flushCache(sequence: number) {
-    if (this.destroyed || sequence < this._flushSequence) {
+  private async _flushCache() {
+    if (this.destroyed) {
       return;
     }
-    this._flushSequence = sequence + 1;
 
     this._flushes.inc();
 
@@ -310,55 +305,69 @@ export class WebFile extends EventEmitter implements File {
       return;
     }
 
-    const sequence = this._flushSequence;
     setTimeout(async () => {
       // Making sure only one flush can run at a time.
+      const promiseBefore = this._flushPromise;
       await this._flushPromise;
       this._flushScheduled = false;
-      this._flushPromise = this._flushCache(sequence).catch((err) => log.warn(err));
+
+      // _flushNow might have been called. In that case we don't want to run the flush again.
+      if (promiseBefore !== this._flushPromise) {
+        return;
+      }
+
+      this._flushPromise = this._flushCache().catch((err) => log.warn(err));
     });
 
     this._flushScheduled = true;
   }
 
   private async _flushNow() {
-    await this._flushPromise;
-    this._flushPromise = this._flushCache(this._flushSequence).catch((err) => log.warn(err));
+    this._flushPromise = (this._flushPromise ?? Promise.resolve())
+      .then(() => this._flushCache())
+      .catch((err) => log.warn(err));
+
     await this._flushPromise;
   }
 
   async read(offset: number, size: number) {
-    this.assertNotDestroyed('Read');
+    if (this.destroyed) {
+      throw new Error('Read of a destroyed file');
+    }
 
     this._operations.inc();
     this._reads.inc();
     this._readBytes.inc(size);
 
-    const buffer = await this.requireBuffer();
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer);
+    }
 
-    if (offset + size > buffer.length) {
+    if (offset + size > this._buffer.length) {
       throw new Error('Read out of bounds');
     }
 
     // Copy data into a new buffer.
-    return Buffer.from(buffer.slice(offset, offset + size));
+    return Buffer.from(this._buffer.slice(offset, offset + size));
   }
 
   async write(offset: number, data: Buffer) {
-    this.assertNotDestroyed('Write');
-
     this._operations.inc();
     this._writes.inc();
     this._writeBytes.inc(data.length);
 
-    const buffer = await this.requireBuffer();
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer);
+    }
 
-    if (offset + data.length <= buffer.length) {
-      buffer.set(data, offset);
+    if (offset + data.length <= this._buffer.length) {
+      this._buffer.set(data, offset);
     } else {
       // TODO(dmaretskyi): Optimize re-allocations.
       const newCache = new Uint8Array(offset + data.length);
-      newCache.set(buffer);
+      newCache.set(this._buffer);
       newCache.set(data, offset);
       this._buffer = newCache;
     }
@@ -367,86 +376,67 @@ export class WebFile extends EventEmitter implements File {
   }
 
   async del(offset: number, size: number) {
-    this.assertNotDestroyed('Del');
-
     this._operations.inc();
 
-    if (offset < 0 || size <= 0) {
+    if (offset < 0 || size < 0) {
       return;
     }
 
-    const buffer = await this.requireBuffer();
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer);
+    }
 
     let leftoverSize = 0;
-    if (offset + size < buffer.length) {
-      leftoverSize = buffer.length - (offset + size);
-      buffer.set(buffer.slice(offset + size, offset + size + leftoverSize), offset);
+    if (offset + size < this._buffer.length) {
+      leftoverSize = this._buffer.length - (offset + size);
+      this._buffer.set(this._buffer.slice(offset + size, offset + size + leftoverSize), offset);
     }
 
-    this._buffer = buffer.slice(0, offset + leftoverSize);
+    this._buffer = this._buffer.slice(0, offset + leftoverSize);
 
     this._flushLater();
   }
 
   async stat() {
-    this.assertNotDestroyed('Truncate');
-
     this._operations.inc();
 
     // NOTE: This will load all data from the file just to get it's size. While this is a lot of overhead, this works ok for out use cases.
-    const buffer = await this.requireBuffer();
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer);
+    }
 
     return {
-      size: buffer.length,
+      size: this._buffer.length,
     };
   }
 
   async truncate(offset: number) {
-    this.assertNotDestroyed('Truncate');
-
     this._operations.inc();
 
-    this._buffer = (await this.requireBuffer()).slice(0, offset);
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer);
+    }
+
+    this._buffer = this._buffer.slice(0, offset);
 
     this._flushLater();
   }
 
   async flush() {
-    this.assertNotDestroyed('Flush');
-
     await this._flushNow();
   }
 
-  /**
-   * It's best to avoid using this method as it doesn't really close a file.
-   * We could update the _opened flag and add a guard like for destroyed, but this would break
-   * the FileSystemFileHandle sharing required for browser tests to run, where writes are
-   * not immediately visible if using different file handles.
-   */
   async close(): Promise<void> {
     await this._flushNow();
   }
 
   @synchronized
   async destroy() {
-    if (!this.destroyed) {
-      this.destroyed = true;
-      await this._flushNow();
-      return await this._destroy();
-    }
-  }
-
-  private assertNotDestroyed(operation: string) {
-    if (this.destroyed) {
-      throw new Error(`${operation} on a destroyed or closed file`);
-    }
-  }
-
-  private async requireBuffer(): Promise<Uint8Array> {
-    if (!this._buffer) {
-      await this._loadBufferGuarded();
-      invariant(this._buffer);
-    }
-    return this._buffer;
+    await this._flushNow();
+    this.destroyed = true;
+    return await this._destroy();
   }
 }

package/src/common/memory-storage.ts

@@ -8,7 +8,7 @@ import { type Callback, type RandomAccessStorage } from 'random-access-storage';
 import { arrayToBuffer } from '@dxos/util';
 
 import { AbstractStorage } from './abstract-storage';
-import { type DiskInfo, StorageType } from './storage';
+import { StorageType } from './storage';
 
 /**
  * Storage interface implementation for RAM.
@@ -42,17 +42,4 @@ export class MemoryStorage extends AbstractStorage {
 
     return file;
   }
-
-  async getDiskInfo(): Promise<DiskInfo> {
-    let used = 0;
-
-    for (const file of this._files.values()) {
-      const size = (file as any).length;
-      used += Number.isNaN(size) ? 0 : size;
-    }
-
-    return {
-      used,
-    };
-  }
 }

package/src/node/node-storage.ts

@@ -69,11 +69,11 @@ export class NodeStorage extends AbstractStorage implements Storage {
     const recurse = async (path: string) => {
       const pathStats = await stat(path);
 
+      used += pathStats.size;
+
       if (pathStats.isDirectory()) {
         const entries = await readdir(path);
         await Promise.all(entries.map((entry) => recurse(join(path, entry))));
-      } else {
-        used += pathStats.size;
       }
     };
 

package/src/testing/storage.blueprint-test.ts

@@ -2,14 +2,13 @@
 // Copyright 2021 DXOS.org
 //
 
-import * as uuid from 'uuid';
-import { describe, expect, test } from 'vitest';
+import { expect, describe, test } from 'vitest';
 
 import { asyncTimeout } from '@dxos/async';
 
-import { type File, type Storage, StorageType } from '../common';
+import { StorageType, type File, type Storage } from '../common';
 
-export const randomText = () => uuid.v4();
+export const randomText = () => Math.random().toString(36).substring(2);
 
 export const storageTests = (testGroupName: StorageType, createStorage: () => Storage) => {
   const writeAndCheck = async (file: File, data: Buffer, offset = 0) => {
@@ -52,7 +51,7 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const directory = storage.createDirectory(directoryName);
 
       const count = 10;
-      const files = [...Array(count)].map(() => directory.getOrCreateFile(randomText()));
+      const files = [...Array(count)].map((name) => directory.getOrCreateFile(randomText()));
 
       {
         // Create and check files amount.
@@ -174,7 +173,7 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const dir1 = storage.createDirectory('dir1');
       const dir2 = storage.createDirectory('dir2');
 
-      const fileName = randomText();
+      const fileName = 'file';
       const buffer1 = Buffer.from(randomText());
       const buffer2 = Buffer.from(randomText());
 
@@ -196,7 +195,7 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const dir = storage.createDirectory('directory');
       const subDir = dir.createDirectory('subDirectory');
 
-      const file = subDir.getOrCreateFile(randomText());
+      const file = subDir.getOrCreateFile('file');
       const buffer = Buffer.from(randomText());
       await file.write(0, buffer);
 
@@ -205,82 +204,10 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       await file.close();
     });
 
-    test('all writes are flushed', async (t) => {
-      if (testGroupName === StorageType.RAM) {
-        t.skip();
-      }
-
-      const fileName = randomText();
-      {
-        const storage = createStorage();
-        const directory = storage.createDirectory();
-        const file = directory.getOrCreateFile(fileName);
-        await file.write(0, Buffer.alloc(10, '0'));
-        for (let i = 1; i <= 9; i++) {
-          void file.write(i, Buffer.from(String(i)));
-        }
-        await file.close();
-      }
-
-      {
-        const storage = createStorage();
-        const directory = storage.createDirectory();
-        const file = directory.getOrCreateFile(fileName);
-        const allContent = await file.read(0, (await file.stat()).size);
-        expect(allContent.toString()).toBe('0123456789');
-      }
-    });
-
-    test('flush interleaved with write', async (t) => {
-      if (testGroupName === StorageType.RAM) {
-        t.skip();
-      }
-
-      const fileName = randomText();
-      {
-        const storage = createStorage();
-        const directory = storage.createDirectory();
-        const file = directory.getOrCreateFile(fileName);
-        await file.write(0, Buffer.alloc(3, '0'));
-        await file.write(1, Buffer.from('1'));
-        void file.flush?.();
-        await file.write(2, Buffer.from('2'));
-        await file.close();
-      }
-
-      {
-        const storage = createStorage();
-        const directory = storage.createDirectory();
-        const file = directory.getOrCreateFile(fileName);
-        const allContent = await file.read(0, (await file.stat()).size);
-        expect(allContent.toString()).toBe('012');
-      }
-    });
-
-    test('operations on a destroyed file are rejected', async () => {
-      const storage = createStorage();
-      const directory = storage.createDirectory();
-      const file = directory.getOrCreateFile(randomText());
-
-      const buffer = Buffer.from(randomText());
-      await writeAndCheck(file, buffer);
-      await file.destroy();
-
-      expect(file.destroyed).toBeTruthy();
-      await expect(file.destroy()).resolves.toBeUndefined();
-      await expect(file.read(0, 1)).rejects.toThrow();
-      await expect(file.write(0, buffer)).rejects.toThrow();
-      await expect(file.del(0, 1)).rejects.toThrow();
-      await expect(file.stat()).rejects.toThrow();
-      if (file.truncate) {
-        await expect(file.truncate(0)).rejects.toThrow();
-      }
-    });
-
     test('delete directory', async () => {
       const storage = createStorage();
       const directory = storage.createDirectory();
-      const file = directory.getOrCreateFile(randomText());
+      const file = directory.getOrCreateFile('file');
 
       const buffer = Buffer.from(randomText());
       await writeAndCheck(file, buffer);
@@ -437,21 +364,5 @@ export const storageTests = (testGroupName: StorageType, createStorage: () => St
       const entries = await directory.list();
       expect(entries).toEqual(expect.arrayContaining(FILES));
     });
-
-    test('getDiskInfo returns correct size', async (t) => {
-      const storage = createStorage();
-      if (storage.type === StorageType.IDB) {
-        t.skip();
-      }
-      await storage.reset();
-      const directory = storage.createDirectory('dir');
-      const file = directory.getOrCreateFile(randomText());
-      const content = 'Hello, world!';
-      await file.write(0, Buffer.from(content));
-      await file.close();
-      await expect(storage.getDiskInfo?.() ?? Promise.resolve(-1)).resolves.toEqual({
-        used: content.length,
-      });
-    });
   });
 };