@dxos/teleport-extension-object-sync 0.8.4-main.fffef41 → 0.9.0

package/dist/lib/browser/index.mjs

@@ -1,647 +0,0 @@
-import "@dxos/node-std/globals";
-
-// src/blob-sync-extension.ts
-import { DeferredTask, sleep, synchronized } from "@dxos/async";
-import { Context } from "@dxos/context";
-import { invariant } from "@dxos/invariant";
-import { log } from "@dxos/log";
-import { RpcClosedError } from "@dxos/protocols";
-import { schema } from "@dxos/protocols/proto";
-import { RpcExtension } from "@dxos/teleport";
-import { BitField } from "@dxos/util";
-function _ts_decorate(decorators, target, key, desc) {
-  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-  return c > 3 && r && Object.defineProperty(target, key, r), r;
-}
-var __dxlog_file = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
-var MIN_WANT_LIST_UPDATE_INTERVAL = false ? 5 : 500;
-var MAX_CONCURRENT_UPLOADS = 20;
-var BlobSyncExtension = class extends RpcExtension {
-  _params;
-  _ctx = new Context({
-    onError: (err) => log.catch(err, void 0, {
-      F: __dxlog_file,
-      L: 35,
-      S: this,
-      C: (f, a) => f(...a)
-    })
-  }, {
-    F: __dxlog_file,
-    L: 35
-  });
-  _lastWantListUpdate = 0;
-  _localWantList = {
-    blobs: []
-  };
-  _updateWantList = new DeferredTask(this._ctx, async () => {
-    if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {
-      await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());
-      if (this._ctx.disposed) {
-        return;
-      }
-    }
-    log("want", {
-      list: this._localWantList
-    }, {
-      F: __dxlog_file,
-      L: 49,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await this.rpc.BlobSyncService.want(this._localWantList);
-    this._lastWantListUpdate = Date.now();
-  });
-  _currentUploads = 0;
-  _upload = new DeferredTask(this._ctx, async () => {
-    if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {
-      return;
-    }
-    const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);
-    if (!blobChunks) {
-      return;
-    }
-    for (const blobChunk of blobChunks) {
-      if (this._ctx.disposed) {
-        break;
-      }
-      this._currentUploads++;
-      this.push(blobChunk).catch((err) => {
-        if (err instanceof RpcClosedError) {
-          return;
-        }
-        log.warn("push failed", {
-          err
-        }, {
-          F: __dxlog_file,
-          L: 76,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-      }).finally(() => {
-        this._currentUploads--;
-        this.reconcileUploads();
-      });
-    }
-  });
-  /**
-  * Set of id's remote peer wants.
-  */
-  remoteWantList = {
-    blobs: []
-  };
-  constructor(_params) {
-    super({
-      exposed: {
-        BlobSyncService: schema.getService("dxos.mesh.teleport.blobsync.BlobSyncService")
-      },
-      requested: {
-        BlobSyncService: schema.getService("dxos.mesh.teleport.blobsync.BlobSyncService")
-      },
-      timeout: 2e4,
-      encodingOptions: {
-        preserveAny: true
-      }
-    }), this._params = _params;
-  }
-  async onOpen(context) {
-    log("open", void 0, {
-      F: __dxlog_file,
-      L: 108,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await super.onOpen(context);
-    await this._params.onOpen();
-  }
-  async onClose(err) {
-    log("close", void 0, {
-      F: __dxlog_file,
-      L: 114,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await this._ctx.dispose();
-    await this._params.onClose();
-    await super.onClose(err);
-  }
-  async onAbort(err) {
-    log("abort", void 0, {
-      F: __dxlog_file,
-      L: 121,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await this._ctx.dispose();
-    await this._params.onAbort();
-    await super.onAbort(err);
-  }
-  async getHandlers() {
-    return {
-      BlobSyncService: {
-        want: async (wantList) => {
-          log("remote want", {
-            remoteWantList: wantList
-          }, {
-            F: __dxlog_file,
-            L: 131,
-            S: this,
-            C: (f, a) => f(...a)
-          });
-          this.remoteWantList = wantList;
-          this.reconcileUploads();
-        },
-        push: async (data) => {
-          log("received", {
-            data
-          }, {
-            F: __dxlog_file,
-            L: 136,
-            S: this,
-            C: (f, a) => f(...a)
-          });
-          await this._params.onPush(data);
-        }
-      }
-    };
-  }
-  async push(data) {
-    if (this._ctx.disposed) {
-      return;
-    }
-    log("push", {
-      data
-    }, {
-      F: __dxlog_file,
-      L: 148,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await this.rpc.BlobSyncService.push(data);
-  }
-  updateWantList(wantList) {
-    if (this._ctx.disposed) {
-      return;
-    }
-    this._localWantList = wantList;
-    this._updateWantList.schedule();
-  }
-  reconcileUploads() {
-    if (this._ctx.disposed) {
-      return;
-    }
-    this._upload.schedule();
-  }
-  async _pickBlobChunks(amount = 1) {
-    if (this._ctx.disposed) {
-      return;
-    }
-    if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {
-      return;
-    }
-    const shuffled = [
-      ...this.remoteWantList.blobs
-    ].sort(() => Math.random() - 0.5);
-    const chunks = [];
-    for (const header of shuffled) {
-      const meta = await this._params.blobStore.getMeta(header.id);
-      if (!meta) {
-        continue;
-      }
-      invariant(meta.bitfield, void 0, {
-        F: __dxlog_file,
-        L: 187,
-        S: this,
-        A: [
-          "meta.bitfield",
-          ""
-        ]
-      });
-      invariant(meta.chunkSize, void 0, {
-        F: __dxlog_file,
-        L: 188,
-        S: this,
-        A: [
-          "meta.chunkSize",
-          ""
-        ]
-      });
-      invariant(meta.length, void 0, {
-        F: __dxlog_file,
-        L: 189,
-        S: this,
-        A: [
-          "meta.length",
-          ""
-        ]
-      });
-      if (header.chunkSize && header.chunkSize !== meta.chunkSize) {
-        log.warn("Invalid chunk size", {
-          header,
-          meta
-        }, {
-          F: __dxlog_file,
-          L: 192,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        continue;
-      }
-      const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);
-      const presentData = BitField.and(requestBitfield, meta.bitfield);
-      const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);
-      for (const idx of chunkIndices) {
-        const chunkData = await this._params.blobStore.get(header.id, {
-          offset: idx * meta.chunkSize,
-          length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize)
-        });
-        chunks.push({
-          id: header.id,
-          totalLength: meta.length,
-          chunkSize: meta.chunkSize,
-          chunkOffset: idx * meta.chunkSize,
-          payload: chunkData
-        });
-        if (chunks.length >= amount) {
-          return chunks;
-        }
-      }
-    }
-    return chunks;
-  }
-};
-_ts_decorate([
-  synchronized
-], BlobSyncExtension.prototype, "push", null);
-
-// src/blob-sync.ts
-import { Mutex, Trigger, trackLeaks } from "@dxos/async";
-import { Context as Context2, cancelWithContext } from "@dxos/context";
-import { invariant as invariant2 } from "@dxos/invariant";
-import { PublicKey } from "@dxos/keys";
-import { log as log2 } from "@dxos/log";
-import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
-import { BitField as BitField2, ComplexMap } from "@dxos/util";
-function _ts_decorate2(decorators, target, key, desc) {
-  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-  return c > 3 && r && Object.defineProperty(target, key, r), r;
-}
-var __dxlog_file2 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
-var BlobSync = class {
-  _params;
-  _ctx = new Context2(void 0, {
-    F: __dxlog_file2,
-    L: 30
-  });
-  _mutex = new Mutex();
-  _downloadRequests = new ComplexMap((key) => PublicKey.from(key).toHex());
-  _extensions = /* @__PURE__ */ new Set();
-  constructor(_params) {
-    this._params = _params;
-  }
-  async open() {
-  }
-  async close() {
-    await this._ctx.dispose();
-  }
-  /**
-  * Resolves when the object with the given id is fully downloaded in the blob store.
-  *
-  * @param id hex-encoded id of the object to download.
-  */
-  async download(ctx, id) {
-    log2("download", {
-      id
-    }, {
-      F: __dxlog_file2,
-      L: 53,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    const request = await this._mutex.executeSynchronized(async () => {
-      const existingRequest = this._downloadRequests.get(id);
-      if (existingRequest) {
-        existingRequest.counter++;
-        return existingRequest;
-      }
-      const meta = await this._params.blobStore.getMeta(id);
-      const request2 = {
-        trigger: new Trigger(),
-        counter: 1,
-        want: {
-          id,
-          chunkSize: meta?.chunkSize,
-          bitfield: meta?.bitfield && Uint8Array.from(BitField2.invert(meta.bitfield))
-        }
-      };
-      if (meta?.state === BlobMeta.State.FULLY_PRESENT) {
-        request2.trigger.wake();
-      } else {
-        this._downloadRequests.set(id, request2);
-        this._updateExtensionsWantList();
-      }
-      return request2;
-    });
-    ctx?.onDispose(() => this._mutex.executeSynchronized(async () => {
-      const request2 = this._downloadRequests.get(id);
-      if (!request2) {
-        return;
-      }
-      if (--request2.counter === 0) {
-        this._downloadRequests.delete(id);
-      }
-      this._updateExtensionsWantList();
-    }));
-    return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();
-  }
-  createExtension() {
-    const extension = new BlobSyncExtension({
-      blobStore: this._params.blobStore,
-      onOpen: async () => {
-        log2("extension opened", void 0, {
-          F: __dxlog_file2,
-          L: 105,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        this._extensions.add(extension);
-        extension.updateWantList(this._getWantList());
-      },
-      onClose: async () => {
-        log2("extension closed", void 0, {
-          F: __dxlog_file2,
-          L: 110,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        this._extensions.delete(extension);
-      },
-      onAbort: async () => {
-        log2("extension aborted", void 0, {
-          F: __dxlog_file2,
-          L: 114,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        this._extensions.delete(extension);
-      },
-      onPush: async (blobChunk) => {
-        if (!this._downloadRequests.has(blobChunk.id)) {
-          return;
-        }
-        log2("received", {
-          blobChunk
-        }, {
-          F: __dxlog_file2,
-          L: 121,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        const meta = await this._params.blobStore.setChunk(blobChunk);
-        if (meta.state === BlobMeta.State.FULLY_PRESENT) {
-          this._downloadRequests.get(blobChunk.id)?.trigger.wake();
-          this._downloadRequests.delete(blobChunk.id);
-        } else {
-          invariant2(meta.bitfield, void 0, {
-            F: __dxlog_file2,
-            L: 127,
-            S: this,
-            A: [
-              "meta.bitfield",
-              ""
-            ]
-          });
-          this._downloadRequests.get(blobChunk.id).want.bitfield = BitField2.invert(meta.bitfield);
-        }
-        this._updateExtensionsWantList();
-        this._reconcileUploads();
-      }
-    });
-    return extension;
-  }
-  /**
-  * Notify extensions that a blob with the given id was added to the blob store.
-  */
-  async notifyBlobAdded(_id) {
-    this._reconcileUploads();
-  }
-  _getWantList() {
-    return {
-      blobs: Array.from(this._downloadRequests.values()).map((request) => request.want)
-    };
-  }
-  _reconcileUploads() {
-    for (const extension of this._extensions) {
-      extension.reconcileUploads();
-    }
-  }
-  _updateExtensionsWantList() {
-    for (const extension of this._extensions) {
-      extension.updateWantList(this._getWantList());
-    }
-  }
-};
-BlobSync = _ts_decorate2([
-  trackLeaks("open", "close")
-], BlobSync);
-
-// src/blob-store.ts
-import path from "@dxos/node-std/path";
-import { synchronized as synchronized2 } from "@dxos/async";
-import { subtleCrypto } from "@dxos/crypto";
-import { invariant as invariant3 } from "@dxos/invariant";
-import { PublicKey as PublicKey2 } from "@dxos/keys";
-import { schema as schema2 } from "@dxos/protocols/proto";
-import { BlobMeta as BlobMeta2 } from "@dxos/protocols/proto/dxos/echo/blob";
-import { BitField as BitField3, arrayToBuffer } from "@dxos/util";
-function _ts_decorate3(decorators, target, key, desc) {
-  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-  return c > 3 && r && Object.defineProperty(target, key, r), r;
-}
-var __dxlog_file3 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
-var DEFAULT_CHUNK_SIZE = 4096;
-var BlobMetaCodec = schema2.getCodecForType("dxos.echo.blob.BlobMeta");
-var BlobStore = class {
-  _directory;
-  constructor(_directory) {
-    this._directory = _directory;
-  }
-  async getMeta(id) {
-    return this._getMeta(id);
-  }
-  /**
-  * @throws If range is not available.
-  */
-  async get(id, options = {}) {
-    const metadata = await this._getMeta(id);
-    if (!metadata) {
-      throw new Error("Blob not available");
-    }
-    const { offset = 0, length = metadata.length } = options;
-    if (offset + length > metadata.length) {
-      throw new Error("Invalid range");
-    }
-    if (metadata.state === BlobMeta2.State.FULLY_PRESENT) {
-      const file2 = this._getDataFile(id);
-      return file2.read(offset, length);
-    } else if (options.offset === void 0 && options.length === void 0) {
-      throw new Error("Blob not available");
-    }
-    const beginChunk = Math.floor(offset / metadata.chunkSize);
-    const endChunk = Math.ceil((offset + length) / metadata.chunkSize);
-    invariant3(metadata.bitfield, "Bitfield not present", {
-      F: __dxlog_file3,
-      L: 61,
-      S: this,
-      A: [
-        "metadata.bitfield",
-        "'Bitfield not present'"
-      ]
-    });
-    invariant3(metadata.bitfield.length * 8 >= endChunk, "Invalid bitfield length", {
-      F: __dxlog_file3,
-      L: 62,
-      S: this,
-      A: [
-        "metadata.bitfield.length * 8 >= endChunk",
-        "'Invalid bitfield length'"
-      ]
-    });
-    const present = BitField3.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;
-    if (!present) {
-      throw new Error("Blob not available");
-    }
-    const file = this._getDataFile(id);
-    return file.read(offset, length);
-  }
-  async list() {
-    const files = new Set((await this._directory.list()).map((f) => f.split("_")[0]));
-    const res = [];
-    for (const file of files) {
-      const id = PublicKey2.from(file).asUint8Array();
-      const meta = await this._getMeta(id);
-      if (meta) {
-        res.push(meta);
-      }
-    }
-    return res;
-  }
-  async set(data) {
-    const id = new Uint8Array(await subtleCrypto.digest("SHA-256", data));
-    const bitfield = BitField3.ones(data.length / DEFAULT_CHUNK_SIZE);
-    const meta = {
-      id,
-      state: BlobMeta2.State.FULLY_PRESENT,
-      length: data.length,
-      chunkSize: DEFAULT_CHUNK_SIZE,
-      bitfield,
-      created: /* @__PURE__ */ new Date(),
-      updated: /* @__PURE__ */ new Date()
-    };
-    await this._getDataFile(id).write(0, arrayToBuffer(data));
-    await this._writeMeta(id, meta);
-    return meta;
-  }
-  // TODO(dmaretskyi): Optimize locking.
-  async setChunk(chunk) {
-    let meta = await this._getMeta(chunk.id);
-    if (!meta) {
-      invariant3(chunk.totalLength, "totalLength is not present", {
-        F: __dxlog_file3,
-        L: 124,
-        S: this,
-        A: [
-          "chunk.totalLength",
-          "'totalLength is not present'"
-        ]
-      });
-      meta = {
-        id: chunk.id,
-        state: BlobMeta2.State.PARTIALLY_PRESENT,
-        length: chunk.totalLength,
-        chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,
-        created: /* @__PURE__ */ new Date()
-      };
-      meta.bitfield = BitField3.zeros(meta.length / meta.chunkSize);
-    }
-    if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {
-      throw new Error("Invalid chunk size");
-    }
-    invariant3(meta.bitfield, "Bitfield not present", {
-      F: __dxlog_file3,
-      L: 139,
-      S: this,
-      A: [
-        "meta.bitfield",
-        "'Bitfield not present'"
-      ]
-    });
-    invariant3(chunk.chunkOffset !== void 0, "chunkOffset is not present", {
-      F: __dxlog_file3,
-      L: 140,
-      S: this,
-      A: [
-        "chunk.chunkOffset !== undefined",
-        "'chunkOffset is not present'"
-      ]
-    });
-    await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));
-    BitField3.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);
-    if (BitField3.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {
-      meta.state = BlobMeta2.State.FULLY_PRESENT;
-    }
-    meta.updated = /* @__PURE__ */ new Date();
-    await this._writeMeta(chunk.id, meta);
-    return meta;
-  }
-  async _writeMeta(id, meta) {
-    const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));
-    const data = Buffer.alloc(encoded.length + 4);
-    data.writeUInt32LE(encoded.length, 0);
-    encoded.copy(data, 4);
-    await this._getMetaFile(id).write(0, data);
-  }
-  async _getMeta(id) {
-    const file = this._getMetaFile(id);
-    const size = (await file.stat()).size;
-    if (size === 0) {
-      return;
-    }
-    const data = await file.read(0, size);
-    const protoSize = data.readUInt32LE(0);
-    return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));
-  }
-  _getMetaFile(id) {
-    return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString("hex"), "meta"));
-  }
-  _getDataFile(id) {
-    return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString("hex"), "data"));
-  }
-};
-_ts_decorate3([
-  synchronized2
-], BlobStore.prototype, "getMeta", null);
-_ts_decorate3([
-  synchronized2
-], BlobStore.prototype, "get", null);
-_ts_decorate3([
-  synchronized2
-], BlobStore.prototype, "list", null);
-_ts_decorate3([
-  synchronized2
-], BlobStore.prototype, "set", null);
-_ts_decorate3([
-  synchronized2
-], BlobStore.prototype, "setChunk", null);
-export {
-  BlobStore,
-  BlobSync,
-  BlobSyncExtension,
-  DEFAULT_CHUNK_SIZE
-};
-//# sourceMappingURL=index.mjs.map