@dxos/random-access-storage 0.1.57-main.e87098f → 0.1.57

package/dist/lib/browser/index.mjs

@@ -13,12 +13,13 @@ var getFullPath = (root, path) => join(root, stringDiff(path, root));
 
 // packages/common/random-access-storage/src/common/directory.ts
 var Directory = class _Directory {
-  constructor(type, path, _list, _getOrCreateFile, _delete) {
+  constructor(type, path, _list, _getOrCreateFile, _delete, _onFlush) {
     this.type = type;
     this.path = path;
     this._list = _list;
     this._getOrCreateFile = _getOrCreateFile;
     this._delete = _delete;
+    this._onFlush = _onFlush;
   }
   toString() {
     return `Directory(${JSON.stringify({
@@ -44,6 +45,9 @@ var Directory = class _Directory {
   getOrCreateFile(filename, opts) {
     return this._getOrCreateFile(this.path, filename, opts);
   }
+  async flush() {
+    await this._onFlush?.();
+  }
   /**
   * Close and delete all files in the directory and all its sub-directories.
   */
@@ -55,7 +59,7 @@ var Directory = class _Directory {
 // packages/common/random-access-storage/src/common/file.ts
 import pify from "pify";
 import { log } from "@dxos/log";
-var __dxlog_file = "/home/runner/work/dxos/dxos/packages/common/random-access-storage/src/common/file.ts";
+var __dxlog_file = "/mnt/ramdisk/work/packages/common/random-access-storage/src/common/file.ts";
 var pifyFields = (object, type, fields) => {
   for (const field of fields) {
     if (!object[field]) {
@@ -100,7 +104,7 @@ var wrapFile = (native, type) => {
 };
 
 // packages/common/random-access-storage/src/common/abstract-storage.ts
-var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/common/random-access-storage/src/common/abstract-storage.ts";
+var __dxlog_file2 = "/mnt/ramdisk/work/packages/common/random-access-storage/src/common/abstract-storage.ts";
 var AbstractStorage = class {
   // TODO(burdon): Make required.
   constructor(path) {
@@ -328,97 +332,7 @@ import { callbackify } from "@dxos/node-std/util";
 import { synchronized } from "@dxos/async";
 import { invariant } from "@dxos/invariant";
 import { log as log3 } from "@dxos/log";
-import { TimeSeriesCounter, TimeUsageCounter, trace } from "@dxos/tracing";
-
-// packages/common/random-access-storage/src/monitor.ts
-import { unrefTimeout } from "@dxos/async";
-var BUCKET_COLLECTION_INTERVAL = 1e3;
-var StorageMonitor = class {
-  constructor() {
-    this._enabled = false;
-    /**
-    * By resource.
-    */
-    this._buckets = /* @__PURE__ */ new Map();
-  }
-  enable() {
-    this._enabled = true;
-    const timer = setInterval(() => {
-      this._finalizeBuckets();
-    }, BUCKET_COLLECTION_INTERVAL);
-    unrefTimeout(timer);
-  }
-  beginOp(op) {
-    if (!this._enabled) {
-      return {
-        end: () => {
-        }
-      };
-    }
-    const beginTime = performance.now();
-    return {
-      end: () => {
-        const endTime = performance.now();
-        const duration = endTime - beginTime;
-        const bucket = this._getOrInitBucket(op.resource);
-        switch (op.type) {
-          case "read":
-            bucket.readOps++;
-            bucket.readBytes += op.size ?? 0;
-            bucket.readTime += duration;
-            break;
-          case "write":
-            bucket.writeOps++;
-            bucket.writeBytes += op.size ?? 0;
-            bucket.writeTime += duration;
-            break;
-          default:
-            bucket.otherOps++;
-            bucket.otherTime += duration;
-        }
-      }
-    };
-  }
-  _getOrInitBucket(resource) {
-    const bucket = this._buckets.get(resource);
-    if (bucket) {
-      return bucket;
-    }
-    const newBucket = {
-      resource,
-      beginTime: performance.now(),
-      endTime: 0,
-      usage: 0,
-      readOps: 0,
-      readBytes: 0,
-      readTime: 0,
-      writeOps: 0,
-      writeBytes: 0,
-      writeTime: 0,
-      otherOps: 0,
-      otherTime: 0
-    };
-    this._buckets.set(resource, newBucket);
-    return newBucket;
-  }
-  _finalizeBuckets() {
-    const now = performance.now();
-    const toEmit = [];
-    for (const bucket of this._buckets.values()) {
-      bucket.endTime = now;
-      bucket.usage = (bucket.readTime + bucket.writeTime + bucket.otherTime) / (bucket.endTime - bucket.beginTime);
-      toEmit.push(bucket);
-    }
-    this._buckets.clear();
-    for (const bucket of toEmit) {
-      this._getOrInitBucket(bucket.resource);
-    }
-    (globalThis.__STORAGE_METRICS ??= []).push(...toEmit);
-  }
-};
-var STORAGE_MONITOR = new StorageMonitor();
-
-// packages/common/random-access-storage/src/browser/web-fs.ts
+import { TimeSeriesCounter, trace } from "@dxos/tracing";
 function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -429,7 +343,7 @@ function _ts_decorate(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
-var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/common/random-access-storage/src/browser/web-fs.ts";
+var __dxlog_file3 = "/mnt/ramdisk/work/packages/common/random-access-storage/src/browser/web-fs.ts";
 var WebFS = class {
   constructor(path) {
     this.path = path;
@@ -463,7 +377,7 @@ var WebFS = class {
     this._root = await navigator.storage.getDirectory();
     invariant(this._root, "Root is undefined", {
       F: __dxlog_file3,
-      L: 68,
+      L: 67,
       S: this,
       A: [
         "this._root",
@@ -473,7 +387,9 @@ var WebFS = class {
     return this._root;
   }
   createDirectory(sub = "") {
-    return new Directory(this.type, getFullPath(this.path, sub), (path) => this._list(path), (...args) => this.getOrCreateFile(...args), () => this._delete(sub));
+    return new Directory(this.type, getFullPath(this.path, sub), (path) => this._list(path), (...args) => this.getOrCreateFile(...args), () => this._delete(sub), async () => {
+      await Promise.all(Array.from(this._getFiles(sub)).map(([path, file]) => file.flush()));
+    });
   }
   getOrCreateFile(path, filename, opts) {
     const fullName = this._getFullFilename(path, filename);
@@ -502,7 +418,7 @@ var WebFS = class {
     await Promise.all(Array.from(this._getFiles(path)).map(async ([path2, file]) => {
       await file.destroy().catch((err) => log3.warn(err, void 0, {
         F: __dxlog_file3,
-        L: 108,
+        L: 110,
         S: this,
         C: (f, a) => f(...a)
       }));
@@ -517,7 +433,7 @@ var WebFS = class {
         recursive: true
       }).catch((err) => log3.warn(err, void 0, {
         F: __dxlog_file3,
-        L: 118,
+        L: 120,
         S: this,
         C: (f, a) => f(...a)
       }));
@@ -561,6 +477,27 @@ _ts_decorate([
 var WebFile = class WebFile2 extends EventEmitter {
   constructor({ fileName, file, destroy }) {
     super();
+    /**
+    * Current view of the file contents.
+    */
+    this._buffer = null;
+    this._loadBufferPromise = null;
+    this._flushScheduled = false;
+    this._flushPromise = null;
+    //
+    // Metrics
+    //
+    this._flushes = new TimeSeriesCounter();
+    this._operations = new TimeSeriesCounter();
+    this._reads = new TimeSeriesCounter();
+    this._readBytes = new TimeSeriesCounter();
+    this._writes = new TimeSeriesCounter();
+    this._writeBytes = new TimeSeriesCounter();
+    this.type = StorageType.WEBFS;
+    //
+    // random-access-storage library compatibility
+    //
+    // TODO(dmaretskyi): Are those all needed?
     this.opened = true;
     this.suspended = false;
     this.closed = false;
@@ -571,17 +508,10 @@ var WebFile = class WebFile2 extends EventEmitter {
     this.deletable = true;
     this.truncatable = true;
     this.statable = true;
-    this._usage = new TimeUsageCounter();
-    this._operations = new TimeSeriesCounter();
-    this._reads = new TimeSeriesCounter();
-    this._readBytes = new TimeSeriesCounter();
-    this._writes = new TimeSeriesCounter();
-    this._writeBytes = new TimeSeriesCounter();
     this.destroyed = false;
     this.directory = "";
     // TODO(dmaretskyi): is this used?
     this.filename = "";
-    this.type = StorageType.WEBFS;
     this.native = {
       write: callbackify(this.write.bind(this)),
       read: callbackify(this.read.bind(this)),
@@ -593,133 +523,187 @@ var WebFile = class WebFile2 extends EventEmitter {
     this._fileName = fileName;
     this._fileHandle = file;
     this._destroy = destroy;
+    void this._loadBufferGuarded();
   }
-  async write(offset, data) {
-    const span = this._usage.beginRecording();
-    this._operations.inc();
-    this._writes.inc();
-    this._writeBytes.inc(data.length);
-    const metric = STORAGE_MONITOR.beginOp({
-      resource: this._fileName,
+  get _bufferSize() {
+    return this._buffer?.length;
+  }
+  async _loadBuffer() {
+    const fileHandle = await this._fileHandle;
+    const file = await fileHandle.getFile();
+    this._buffer = new Uint8Array(await file.arrayBuffer());
+  }
+  async _loadBufferGuarded() {
+    await (this._loadBufferPromise ??= this._loadBuffer());
+  }
+  // Do not call directly, use _flushLater or _flushNow.
+  async _flushCache() {
+    if (this.destroyed) {
+      return;
+    }
+    this._flushes.inc();
+    await this._loadBufferGuarded();
+    invariant(this._buffer, void 0, {
+      F: __dxlog_file3,
+      L: 279,
+      S: this,
+      A: [
+        "this._buffer",
+        ""
+      ]
+    });
+    const fileHandle = await this._fileHandle;
+    const writable = await fileHandle.createWritable({
+      keepExistingData: true
+    });
+    await writable.write({
       type: "write",
-      size: data.length
+      data: this._buffer,
+      position: 0
     });
-    try {
-      const fileHandle = await this._fileHandle;
-      const writable = await fileHandle.createWritable({
-        keepExistingData: true
-      });
-      await writable.write({
-        type: "write",
-        data,
-        position: offset
-      });
-      await writable.close();
-    } finally {
-      span.end();
-      metric.end();
+    await writable.close();
+  }
+  _flushLater() {
+    if (this._flushScheduled) {
+      return;
     }
+    setTimeout(async () => {
+      const promiseBefore = this._flushPromise;
+      await this._flushPromise;
+      this._flushScheduled = false;
+      if (promiseBefore !== this._flushPromise) {
+        return;
+      }
+      this._flushPromise = this._flushCache().catch((err) => log3.warn(err, void 0, {
+        F: __dxlog_file3,
+        L: 303,
+        S: this,
+        C: (f, a) => f(...a)
+      }));
+    });
+    this._flushScheduled = true;
+  }
+  async _flushNow() {
+    this._flushPromise = (this._flushPromise ?? Promise.resolve()).then(() => this._flushCache()).catch((err) => log3.warn(err, void 0, {
+      F: __dxlog_file3,
+      L: 312,
+      S: this,
+      C: (f, a) => f(...a)
+    }));
+    await this._flushPromise;
   }
   async read(offset, size) {
-    const span = this._usage.beginRecording();
     this._operations.inc();
     this._reads.inc();
     this._readBytes.inc(size);
-    const metric = STORAGE_MONITOR.beginOp({
-      resource: this._fileName,
-      type: "read",
-      size
-    });
-    try {
-      const fileHandle = await this._fileHandle;
-      const file = await fileHandle.getFile();
-      if (offset + size > file.size) {
-        throw new Error("Read out of bounds");
-      }
-      return Buffer.from(await file.slice(offset, offset + size).arrayBuffer());
-    } finally {
-      span.end();
-      metric.end();
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer, void 0, {
+        F: __dxlog_file3,
+        L: 324,
+        S: this,
+        A: [
+          "this._buffer",
+          ""
+        ]
+      });
+    }
+    if (offset + size > this._buffer.length) {
+      throw new Error("Read out of bounds");
     }
+    return Buffer.from(this._buffer.slice(offset, offset + size));
   }
-  async del(offset, size) {
-    const span = this._usage.beginRecording();
+  async write(offset, data) {
     this._operations.inc();
-    const metric = STORAGE_MONITOR.beginOp({
-      resource: this._fileName,
-      type: "delete",
-      size
-    });
-    try {
-      if (offset < 0 || size < 0) {
-        return;
-      }
-      const fileHandle = await this._fileHandle;
-      const writable = await fileHandle.createWritable({
-        keepExistingData: true
+    this._writes.inc();
+    this._writeBytes.inc(data.length);
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer, void 0, {
+        F: __dxlog_file3,
+        L: 342,
+        S: this,
+        A: [
+          "this._buffer",
+          ""
+        ]
       });
-      const file = await fileHandle.getFile();
-      let leftoverSize = 0;
-      if (offset + size < file.size) {
-        const leftover = Buffer.from(await file.slice(offset + size, file.size).arrayBuffer());
-        leftoverSize = leftover.length;
-        await writable.write({
-          type: "write",
-          data: leftover,
-          position: offset
-        });
-      }
-      await writable.write({
-        type: "truncate",
-        size: offset + leftoverSize
+    }
+    if (offset + data.length <= this._buffer.length) {
+      this._buffer.set(data, offset);
+    } else {
+      const newCache = new Uint8Array(offset + data.length);
+      newCache.set(this._buffer);
+      newCache.set(data, offset);
+      this._buffer = newCache;
+    }
+    this._flushLater();
+  }
+  async del(offset, size) {
+    this._operations.inc();
+    if (offset < 0 || size < 0) {
+      return;
+    }
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer, void 0, {
+        F: __dxlog_file3,
+        L: 367,
+        S: this,
+        A: [
+          "this._buffer",
+          ""
+        ]
       });
-      await writable.close();
-    } finally {
-      span.end();
-      metric.end();
     }
+    let leftoverSize = 0;
+    if (offset + size < this._buffer.length) {
+      leftoverSize = this._buffer.length - (offset + size);
+      this._buffer.set(this._buffer.slice(offset + size, offset + size + leftoverSize), offset);
+    }
+    this._buffer = this._buffer.slice(0, offset + leftoverSize);
+    this._flushLater();
   }
   async stat() {
-    const span = this._usage.beginRecording();
     this._operations.inc();
-    const metric = STORAGE_MONITOR.beginOp({
-      resource: this._fileName,
-      type: "stat"
-    });
-    try {
-      const fileHandle = await this._fileHandle;
-      const file = await fileHandle.getFile();
-      return {
-        size: file.size
-      };
-    } finally {
-      span.end();
-      metric.end();
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer, void 0, {
+        F: __dxlog_file3,
+        L: 387,
+        S: this,
+        A: [
+          "this._buffer",
+          ""
+        ]
+      });
     }
+    return {
+      size: this._buffer.length
+    };
   }
   async truncate(offset) {
-    const span = this._usage.beginRecording();
     this._operations.inc();
-    const metric = STORAGE_MONITOR.beginOp({
-      resource: this._fileName,
-      type: "truncate"
-    });
-    try {
-      const fileHandle = await this._fileHandle;
-      const writable = await fileHandle.createWritable({
-        keepExistingData: true
-      });
-      await writable.write({
-        type: "truncate",
-        size: offset
+    if (!this._buffer) {
+      await this._loadBufferGuarded();
+      invariant(this._buffer, void 0, {
+        F: __dxlog_file3,
+        L: 400,
+        S: this,
+        A: [
+          "this._buffer",
+          ""
+        ]
       });
-      await writable.close();
-    } finally {
-      span.end();
-      metric.end();
     }
+    this._buffer = this._buffer.slice(0, offset);
+    this._flushLater();
+  }
+  async flush() {
+    await this._flushNow();
   }
   async close() {
+    await this._flushNow();
   }
   async destroy() {
     this.destroyed = true;
@@ -731,7 +715,7 @@ _ts_decorate([
 ], WebFile.prototype, "_fileName", void 0);
 _ts_decorate([
   trace.metricsCounter()
-], WebFile.prototype, "_usage", void 0);
+], WebFile.prototype, "_flushes", void 0);
 _ts_decorate([
   trace.metricsCounter()
 ], WebFile.prototype, "_operations", void 0);
@@ -748,20 +732,8 @@ _ts_decorate([
   trace.metricsCounter()
 ], WebFile.prototype, "_writeBytes", void 0);
 _ts_decorate([
-  synchronized
-], WebFile.prototype, "write", null);
-_ts_decorate([
-  synchronized
-], WebFile.prototype, "read", null);
-_ts_decorate([
-  synchronized
-], WebFile.prototype, "del", null);
-_ts_decorate([
-  synchronized
-], WebFile.prototype, "stat", null);
-_ts_decorate([
-  synchronized
-], WebFile.prototype, "truncate", null);
+  trace.info()
+], WebFile.prototype, "_bufferSize", null);
 _ts_decorate([
   synchronized
 ], WebFile.prototype, "destroy", null);