@dxos/teleport-extension-replicator 0.1.56-main.c19b7cd → 0.1.56-main.c277f68

package/dist/lib/browser/index.mjs

@@ -25,14 +25,13 @@ var ReplicatorExtension = class {
   constructor() {
     this._ctx = new Context({
       onError: (err) => {
-        var _a;
         log.catch(err, void 0, {
           F: __dxlog_file,
           L: 31,
           S: this,
           C: (f, a) => f(...a)
         });
-        (_a = this._extensionContext) == null ? void 0 : _a.close(err);
+        this._extensionContext?.close(err);
       }
     });
     this._feeds = new ComplexMap(PublicKey.hash);
@@ -62,11 +61,10 @@ var ReplicatorExtension = class {
     });
   }
   get extensionInfo() {
-    var _a, _b, _c;
     return {
-      initiator: (_a = this._extensionContext) == null ? void 0 : _a.initiator,
-      localPeerId: (_b = this._extensionContext) == null ? void 0 : _b.localPeerId,
-      remotePeerId: (_c = this._extensionContext) == null ? void 0 : _c.remotePeerId,
+      initiator: this._extensionContext?.initiator,
+      localPeerId: this._extensionContext?.localPeerId,
+      remotePeerId: this._extensionContext?.remotePeerId,
       feeds: Array.from(this._feeds.keys())
     };
   }
@@ -189,7 +187,6 @@ var ReplicatorExtension = class {
     this._updateTask.schedule();
   }
   async onClose(err) {
-    var _a;
     log("close", {
       err
     }, {
@@ -199,13 +196,12 @@ var ReplicatorExtension = class {
       C: (f, a) => f(...a)
     });
     await this._ctx.dispose();
-    await ((_a = this._rpc) == null ? void 0 : _a.close());
+    await this._rpc?.close();
     for (const feedKey of this._streams.keys()) {
       await this._stopReplication(feedKey);
     }
   }
   async _reevaluateFeeds() {
-    var _a;
     log("_reevaluateFeeds", void 0, {
       F: __dxlog_file,
       L: 150,
@@ -216,7 +212,7 @@ var ReplicatorExtension = class {
       if (this._ctx.disposed) {
         return;
       }
-      if (this._streams.has(feedKey) && this._options.upload !== ((_a = this._streams.get(feedKey)) == null ? void 0 : _a.info.upload)) {
+      if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {
         try {
           await asyncTimeout(this._stopReplication(feedKey), 1e3);
         } catch (err) {
@@ -300,7 +296,6 @@ var ReplicatorExtension = class {
     return tag;
   }
   async _replicateFeed(info, streamTag) {
-    var _a;
     log("replicate", {
       info,
       streamTag
@@ -319,7 +314,7 @@ var ReplicatorExtension = class {
         "`Replication already in progress for feed: ${info.feedKey}`"
       ]
     });
-    const feed = (_a = this._feeds.get(info.feedKey)) != null ? _a : failUndefined();
+    const feed = this._feeds.get(info.feedKey) ?? failUndefined();
     const networkStream = await this._extensionContext.createStream(streamTag, {
       contentType: "application/x-hypercore"
     });
@@ -328,10 +323,11 @@ var ReplicatorExtension = class {
       upload: info.upload,
       download: info.download,
       noise: false,
-      encrypted: false
+      encrypted: false,
+      maxRequests: 1024
     });
     replicationStream.on("error", (err) => {
-      if ((err == null ? void 0 : err.message) === "Writable stream closed prematurely" || (err == null ? void 0 : err.message) === "Cannot call write after a stream was destroyed") {
+      if (err?.message === "Writable stream closed prematurely" || err?.message === "Cannot call write after a stream was destroyed") {
         return;
       }
       log.warn("replication stream error", {
@@ -339,7 +335,7 @@ var ReplicatorExtension = class {
         info
       }, {
         F: __dxlog_file,
-        L: 240,
+        L: 251,
         S: this,
         C: (f, a) => f(...a)
       });

package/dist/lib/browser/index.mjs.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../src/replicator-extension.ts"],
-  "sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\n\nimport { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { failUndefined } from '@dxos/debug';\nimport { FeedWrapper } from '@dxos/feed-store';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log, logInfo } from '@dxos/log';\nimport { schema, RpcClosedError } from '@dxos/protocols';\nimport { FeedInfo, ReplicatorService } from '@dxos/protocols/proto/dxos/mesh/teleport/replicator';\nimport { createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';\nimport { ExtensionContext, TeleportExtension } from '@dxos/teleport';\nimport { ComplexMap } from '@dxos/util';\n\nexport type ReplicationOptions = {\n  upload: boolean;\n};\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class ReplicatorExtension implements TeleportExtension {\n  private readonly _ctx = new Context({\n    onError: (err) => {\n      log.catch(err);\n      this._extensionContext?.close(err);\n    },\n  });\n\n  private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);\n  private readonly _streams = new ComplexMap<PublicKey, ActiveStream>(PublicKey.hash);\n\n  private _rpc?: ProtoRpcPeer<ServiceBundle>;\n  private _extensionContext?: ExtensionContext;\n\n  private _options: ReplicationOptions = {\n    upload: false,\n  };\n\n  private readonly _updateTask = new DeferredTask(this._ctx, async () => {\n    try {\n      if (this._extensionContext!.initiator === false) {\n        await this._rpc!.rpc.ReplicatorService.updateFeeds({\n          feeds: Array.from(this._feeds.values()).map((feed) => ({\n            feedKey: feed.key,\n            download: true,\n            upload: this._options.upload,\n          })),\n        });\n      } else if (this._extensionContext!.initiator === true) {\n        await this._reevaluateFeeds();\n      }\n    } catch (err) {\n      if (err instanceof RpcClosedError) {\n        return; // Some RPC requests might be pending while closing.\n      }\n      throw err;\n    }\n  });\n\n  @logInfo\n  private get extensionInfo() {\n    return {\n      initiator: this._extensionContext?.initiator,\n      localPeerId: this._extensionContext?.localPeerId,\n      remotePeerId: this._extensionContext?.remotePeerId,\n      feeds: Array.from(this._feeds.keys()),\n    };\n  }\n\n  setOptions(options: ReplicationOptions): this {\n    this._options = options;\n    log('setOptions', { options });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n    return this;\n  }\n\n  addFeed(feed: FeedWrapper<any>) {\n    this._feeds.set(feed.key, feed);\n    log('addFeed', { feedKey: feed.key });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n  }\n\n  async onOpen(context: ExtensionContext) {\n    this._extensionContext = context;\n    log('open');\n\n    this._rpc = createProtoRpcPeer<ServiceBundle, ServiceBundle>({\n      requested: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      exposed: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      handlers: {\n        ReplicatorService: {\n          updateFeeds: async ({ feeds }) => {\n            log('received feed info', { feeds });\n            invariant(this._extensionContext!.initiator === true, 'Invalid call');\n            this._updateTask.schedule();\n          },\n          startReplication: async ({ info }) => {\n            log('starting replication...', { info });\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            const streamTag = await this._acceptReplication(info);\n            return {\n              streamTag,\n            };\n          },\n          stopReplication: async ({ info }) => {\n            log('stopping replication...', { info });\n            // TODO(dmaretskyi): Make sure any peer can stop replication.\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            await this._stopReplication(info.feedKey);\n          },\n        },\n      },\n      port: await context.createPort('rpc', {\n        contentType: 'application/x-protobuf; messageType=\"dxos.rpc.Message\"',\n      }),\n    });\n    await this._rpc.open();\n\n    this._updateTask.schedule();\n  }\n\n  async onClose(err?: Error | undefined) {\n    log('close', { err });\n    await this._ctx.dispose();\n    await this._rpc?.close();\n    for (const feedKey of this._streams.keys()) {\n      await this._stopReplication(feedKey);\n    }\n  }\n\n  @synchronized\n  private async _reevaluateFeeds() {\n    log('_reevaluateFeeds');\n    for (const feedKey of this._feeds.keys()) {\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {\n        try {\n          await asyncTimeout(this._stopReplication(feedKey), 1000);\n        } catch (err) {\n          log.catch(err);\n        }\n      }\n\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (!this._streams.has(feedKey)) {\n        await this._initiateReplication({\n          feedKey,\n          download: true,\n          upload: this._options.upload,\n        });\n      }\n    }\n  }\n\n  /**\n   * Try to initiate feed replication.\n   */\n  private async _initiateReplication(feedInfo: FeedInfo) {\n    log('initiating replication', { feedInfo });\n    invariant(this._extensionContext!.initiator === true, 'Invalid call');\n    invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`);\n    const { streamTag } = await this._rpc!.rpc.ReplicatorService.startReplication({ info: feedInfo });\n    if (!streamTag) {\n      return;\n    }\n\n    await this._replicateFeed(feedInfo, streamTag);\n  }\n\n  /**\n   * Respond to a remote request to replicate a feed.\n   * @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.\n   */\n  @synchronized\n  private async _acceptReplication(feedInfo: FeedInfo): Promise<string | undefined> {\n    invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n    if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {\n      return undefined; // We don't have the feed or we are already replicating it.\n    }\n\n    const tag = `feed-${feedInfo.feedKey.toHex()}-${PublicKey.random().toHex().slice(0, 8)}`; // Generate a unique tag for the stream.\n    await this._replicateFeed(feedInfo, tag);\n    return tag;\n  }\n\n  private async _replicateFeed(info: FeedInfo, streamTag: string) {\n    log('replicate', { info, streamTag });\n    invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`);\n\n    const feed = this._feeds.get(info.feedKey) ?? failUndefined();\n    const networkStream = await this._extensionContext!.createStream(streamTag, {\n      contentType: 'application/x-hypercore',\n    });\n    const replicationStream = feed.replicate(true, {\n      live: true,\n      upload: info.upload,\n      download: info.download,\n      noise: false,\n      encrypted: false,\n    });\n\n    // left for testing\n    // feed.on('download', (index: number, data: any) => {\n    //   log('download', {\n    //     key: info.feedKey,\n    //     index\n    //   });\n    // });\n\n    replicationStream.on('error', (err) => {\n      if (\n        err?.message === 'Writable stream closed prematurely' ||\n        err?.message === 'Cannot call write after a stream was destroyed'\n      ) {\n        return;\n      }\n\n      log.warn('replication stream error', { err, info });\n    });\n\n    this._streams.set(info.feedKey, {\n      streamTag,\n      networkStream,\n      replicationStream,\n      info,\n    });\n\n    networkStream.pipe(replicationStream as any).pipe(networkStream);\n  }\n\n  private async _stopReplication(feedKey: PublicKey) {\n    const stream = this._streams.get(feedKey);\n    if (!stream) {\n      return;\n    }\n\n    stream.networkStream.destroy();\n    this._streams.delete(feedKey);\n  }\n}\n\ntype ServiceBundle = {\n  ReplicatorService: ReplicatorService;\n};\n\ntype ActiveStream = {\n  streamTag: string;\n  networkStream: Duplex;\n  replicationStream: ProtocolStream;\n  info: FeedInfo;\n};\n"],
-  "mappings": ";;;AAOA,SAASA,cAAcC,cAAcC,oBAAoB;AACzD,SAASC,eAAe;AACxB,SAASC,qBAAqB;AAE9B,SAASC,iBAAiB;AAC1B,SAASC,iBAAiB;AAC1B,SAASC,KAAKC,eAAe;AAC7B,SAASC,QAAQC,sBAAsB;AAEvC,SAASC,0BAAwC;AAEjD,SAASC,kBAAkB;;;;;;;;;;;;AASpB,IAAMC,sBAAN,MAAMA;EAAN;AACYC,gBAAO,IAAIX,QAAQ;MAClCY,SAAS,CAACC,QAAAA;AA7Bd;AA8BMT,YAAIU,MAAMD,KAAAA,QAAAA;;;;;;AACV,mBAAKE,sBAAL,mBAAwBC,MAAMH;MAChC;IACF,CAAA;AAEiBI,kBAAS,IAAIR,WAAwCN,UAAUe,IAAI;AACnEC,oBAAW,IAAIV,WAAoCN,UAAUe,IAAI;AAK1EE,oBAA+B;MACrCC,QAAQ;IACV;AAEiBC,uBAAc,IAAIxB,aAAa,KAAKa,MAAM,YAAA;AACzD,UAAI;AACF,YAAI,KAAKI,kBAAmBQ,cAAc,OAAO;AAC/C,gBAAM,KAAKC,KAAMC,IAAIC,kBAAkBC,YAAY;YACjDC,OAAOC,MAAMC,KAAK,KAAKb,OAAOc,OAAM,CAAA,EAAIC,IAAI,CAACC,UAAU;cACrDC,SAASD,KAAKE;cACdC,UAAU;cACVf,QAAQ,KAAKD,SAASC;YACxB,EAAA;UACF,CAAA;QACF,WAAW,KAAKN,kBAAmBQ,cAAc,MAAM;AACrD,gBAAM,KAAKc,iBAAgB;QAC7B;MACF,SAASxB,KAAP;AACA,YAAIA,eAAeN,gBAAgB;AACjC;QACF;AACA,cAAMM;MACR;IACF,CAAA;;EAEA,IACYyB,gBAAgB;AAnE9B;AAoEI,WAAO;MACLf,YAAW,UAAKR,sBAAL,mBAAwBQ;MACnCgB,cAAa,UAAKxB,sBAAL,mBAAwBwB;MACrCC,eAAc,UAAKzB,sBAAL,mBAAwByB;MACtCZ,OAAOC,MAAMC,KAAK,KAAKb,OAAOwB,KAAI,CAAA;IACpC;EACF;EAEAC,WAAWC,SAAmC;AAC5C,SAAKvB,WAAWuB;AAChBvC,QAAI,cAAc;MAAEuC;IAAQ,GAAA;;;;;;AAC5B,QAAI,KAAK5B,mBAAmB;AAC1B,WAAKO,YAAYsB,SAAQ;IAC3B;AACA,WAAO;EACT;EAEAC,QAAQZ,MAAwB;AAC9B,SAAKhB,OAAO6B,IAAIb,KAAKE,KAAKF,IAAAA;AAC1B7B,QAAI,WAAW;MAAE8B,SAASD,KAAKE;IAAI,GAAA;;;;;;AACnC,QAAI,KAAKpB,mBAAmB;AAC1B,WAAKO,YAAYsB,SAAQ;IAC3B;EACF;EAEA,MAAMG,OAAOC,SAA2B;AACtC,SAAKjC,oBAAoBiC;AACzB5C,QAAI,QAAA,QAAA;;;;;;AAEJ,SAAKoB,OAAOhB,mBAAiD;MAC3DyC,WAAW;QACTvB,mBAAmBpB,OAAO4C,WAAW,iDAAA;MACvC;MACAC,SAAS;QACPzB,mBAAmBpB,OAAO4C,WAAW,iDAAA;MACvC;MACAE,UAAU;QACR1B,mBAAmB;UACjBC,aAAa,OAAO,EAAEC,MAAK,MAAE;AAC3BxB,gBAAI,sBAAsB;cAAEwB;YAAM,GAAA;;;;;;AAClC1B,sBAAU,KAAKa,kBAAmBQ,cAAc,MAAM,gBAAA;;;;;;;;;AACtD,iBAAKD,YAAYsB,SAAQ;UAC3B;UACAS,kBAAkB,OAAO,EAAEC,KAAI,MAAE;AAC/BlD,gBAAI,2BAA2B;cAAEkD;YAAK,GAAA;;;;;;AACtCpD,sBAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAMgC,YAAY,MAAM,KAAKC,mBAAmBF,IAAAA;AAChD,mBAAO;cACLC;YACF;UACF;UACAE,iBAAiB,OAAO,EAAEH,KAAI,MAAE;AAC9BlD,gBAAI,2BAA2B;cAAEkD;YAAK,GAAA;;;;;;AAEtCpD,sBAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAM,KAAKmC,iBAAiBJ,KAAKpB,OAAO;UAC1C;QACF;MACF;MACAyB,MAAM,MAAMX,QAAQY,WAAW,OAAO;QACpCC,aAAa;MACf,CAAA;IACF,CAAA;AACA,UAAM,KAAKrC,KAAKsC,KAAI;AAEpB,SAAKxC,YAAYsB,SAAQ;EAC3B;EAEA,MAAMmB,QAAQlD,KAAyB;AA1IzC;AA2IIT,QAAI,SAAS;MAAES;IAAI,GAAA;;;;;;AACnB,UAAM,KAAKF,KAAKqD,QAAO;AACvB,YAAM,UAAKxC,SAAL,mBAAWR;AACjB,eAAWkB,WAAW,KAAKf,SAASsB,KAAI,GAAI;AAC1C,YAAM,KAAKiB,iBAAiBxB,OAAAA;IAC9B;EACF;EAEA,MACcG,mBAAmB;AApJnC;AAqJIjC,QAAI,oBAAA,QAAA;;;;;;AACJ,eAAW8B,WAAW,KAAKjB,OAAOwB,KAAI,GAAI;AACxC,UAAI,KAAK9B,KAAKsD,UAAU;AACtB;MACF;AACA,UAAI,KAAK9C,SAAS+C,IAAIhC,OAAAA,KAAY,KAAKd,SAASC,aAAW,UAAKF,SAASgD,IAAIjC,OAAAA,MAAlB,mBAA4BoB,KAAKjC,SAAQ;AAClG,YAAI;AACF,gBAAMxB,aAAa,KAAK6D,iBAAiBxB,OAAAA,GAAU,GAAA;QACrD,SAASrB,KAAP;AACAT,cAAIU,MAAMD,KAAAA,QAAAA;;;;;;QACZ;MACF;AAEA,UAAI,KAAKF,KAAKsD,UAAU;AACtB;MACF;AACA,UAAI,CAAC,KAAK9C,SAAS+C,IAAIhC,OAAAA,GAAU;AAC/B,cAAM,KAAKkC,qBAAqB;UAC9BlC;UACAE,UAAU;UACVf,QAAQ,KAAKD,SAASC;QACxB,CAAA;MACF;IACF;EACF;;;;EAKA,MAAc+C,qBAAqBC,UAAoB;AACrDjE,QAAI,0BAA0B;MAAEiE;IAAS,GAAA;;;;;;AACzCnE,cAAU,KAAKa,kBAAmBQ,cAAc,MAAM,gBAAA;;;;;;;;;AACtDrB,cAAU,CAAC,KAAKiB,SAAS+C,IAAIG,SAASnC,OAAO,GAAG,6CAA6CmC,SAASnC,WAAS;;;;;;;;;AAC/G,UAAM,EAAEqB,UAAS,IAAK,MAAM,KAAK/B,KAAMC,IAAIC,kBAAkB2B,iBAAiB;MAAEC,MAAMe;IAAS,CAAA;AAC/F,QAAI,CAACd,WAAW;AACd;IACF;AAEA,UAAM,KAAKe,eAAeD,UAAUd,SAAAA;EACtC;;;;;EAMA,MACcC,mBAAmBa,UAAiD;AAChFnE,cAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,QAAI,CAAC,KAAKN,OAAOiD,IAAIG,SAASnC,OAAO,KAAK,KAAKf,SAAS+C,IAAIG,SAASnC,OAAO,GAAG;AAC7E,aAAOqC;IACT;AAEA,UAAMC,MAAM,QAAQH,SAASnC,QAAQuC,MAAK,KAAMtE,UAAUuE,OAAM,EAAGD,MAAK,EAAGE,MAAM,GAAG,CAAA;AACpF,UAAM,KAAKL,eAAeD,UAAUG,GAAAA;AACpC,WAAOA;EACT;EAEA,MAAcF,eAAehB,MAAgBC,WAAmB;AA/MlE;AAgNInD,QAAI,aAAa;MAAEkD;MAAMC;IAAU,GAAA;;;;;;AACnCrD,cAAU,CAAC,KAAKiB,SAAS+C,IAAIZ,KAAKpB,OAAO,GAAG,6CAA6CoB,KAAKpB,WAAS;;;;;;;;;AAEvG,UAAMD,QAAO,UAAKhB,OAAOkD,IAAIb,KAAKpB,OAAO,MAA5B,YAAiCjC,cAAAA;AAC9C,UAAM2E,gBAAgB,MAAM,KAAK7D,kBAAmB8D,aAAatB,WAAW;MAC1EM,aAAa;IACf,CAAA;AACA,UAAMiB,oBAAoB7C,KAAK8C,UAAU,MAAM;MAC7CC,MAAM;MACN3D,QAAQiC,KAAKjC;MACbe,UAAUkB,KAAKlB;MACf6C,OAAO;MACPC,WAAW;IACb,CAAA;AAUAJ,sBAAkBK,GAAG,SAAS,CAACtE,QAAAA;AAC7B,WACEA,2BAAKuE,aAAY,yCACjBvE,2BAAKuE,aAAY,kDACjB;AACA;MACF;AAEAhF,UAAIiF,KAAK,4BAA4B;QAAExE;QAAKyC;MAAK,GAAA;;;;;;IACnD,CAAA;AAEA,SAAKnC,SAAS2B,IAAIQ,KAAKpB,SAAS;MAC9BqB;MACAqB;MACAE;MACAxB;IACF,CAAA;AAEAsB,kBAAcU,KAAKR,iBAAAA,EAA0BQ,KAAKV,aAAAA;EACpD;EAEA,MAAclB,iBAAiBxB,SAAoB;AACjD,UAAMqD,SAAS,KAAKpE,SAASgD,IAAIjC,OAAAA;AACjC,QAAI,CAACqD,QAAQ;AACX;IACF;AAEAA,WAAOX,cAAcY,QAAO;AAC5B,SAAKrE,SAASsE,OAAOvD,OAAAA;EACvB;AACF;;EAnMG7B;GAvCUK,oBAAAA,WAAAA,iBAAAA,IAAAA;;EAwHVX;GAxHUW,oBAAAA,WAAAA,oBAAAA,IAAAA;;EAuKVX;GAvKUW,oBAAAA,WAAAA,sBAAAA,IAAAA;",
-  "names": ["asyncTimeout", "DeferredTask", "synchronized", "Context", "failUndefined", "invariant", "PublicKey", "log", "logInfo", "schema", "RpcClosedError", "createProtoRpcPeer", "ComplexMap", "ReplicatorExtension", "_ctx", "onError", "err", "catch", "_extensionContext", "close", "_feeds", "hash", "_streams", "_options", "upload", "_updateTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "requested", "getService", "exposed", "handlers", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "on", "message", "warn", "pipe", "stream", "destroy", "delete"]
+  "sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\n\nimport { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { failUndefined } from '@dxos/debug';\nimport { FeedWrapper } from '@dxos/feed-store';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log, logInfo } from '@dxos/log';\nimport { schema, RpcClosedError } from '@dxos/protocols';\nimport { FeedInfo, ReplicatorService } from '@dxos/protocols/proto/dxos/mesh/teleport/replicator';\nimport { createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';\nimport { ExtensionContext, TeleportExtension } from '@dxos/teleport';\nimport { ComplexMap } from '@dxos/util';\n\nexport type ReplicationOptions = {\n  upload: boolean;\n};\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class ReplicatorExtension implements TeleportExtension {\n  private readonly _ctx = new Context({\n    onError: (err) => {\n      log.catch(err);\n      this._extensionContext?.close(err);\n    },\n  });\n\n  private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);\n  private readonly _streams = new ComplexMap<PublicKey, ActiveStream>(PublicKey.hash);\n\n  private _rpc?: ProtoRpcPeer<ServiceBundle>;\n  private _extensionContext?: ExtensionContext;\n\n  private _options: ReplicationOptions = {\n    upload: false,\n  };\n\n  private readonly _updateTask = new DeferredTask(this._ctx, async () => {\n    try {\n      if (this._extensionContext!.initiator === false) {\n        await this._rpc!.rpc.ReplicatorService.updateFeeds({\n          feeds: Array.from(this._feeds.values()).map((feed) => ({\n            feedKey: feed.key,\n            download: true,\n            upload: this._options.upload,\n          })),\n        });\n      } else if (this._extensionContext!.initiator === true) {\n        await this._reevaluateFeeds();\n      }\n    } catch (err) {\n      if (err instanceof RpcClosedError) {\n        return; // Some RPC requests might be pending while closing.\n      }\n      throw err;\n    }\n  });\n\n  @logInfo\n  private get extensionInfo() {\n    return {\n      initiator: this._extensionContext?.initiator,\n      localPeerId: this._extensionContext?.localPeerId,\n      remotePeerId: this._extensionContext?.remotePeerId,\n      feeds: Array.from(this._feeds.keys()),\n    };\n  }\n\n  setOptions(options: ReplicationOptions): this {\n    this._options = options;\n    log('setOptions', { options });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n    return this;\n  }\n\n  addFeed(feed: FeedWrapper<any>) {\n    this._feeds.set(feed.key, feed);\n    log('addFeed', { feedKey: feed.key });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n  }\n\n  async onOpen(context: ExtensionContext) {\n    this._extensionContext = context;\n    log('open');\n\n    this._rpc = createProtoRpcPeer<ServiceBundle, ServiceBundle>({\n      requested: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      exposed: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      handlers: {\n        ReplicatorService: {\n          updateFeeds: async ({ feeds }) => {\n            log('received feed info', { feeds });\n            invariant(this._extensionContext!.initiator === true, 'Invalid call');\n            this._updateTask.schedule();\n          },\n          startReplication: async ({ info }) => {\n            log('starting replication...', { info });\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            const streamTag = await this._acceptReplication(info);\n            return {\n              streamTag,\n            };\n          },\n          stopReplication: async ({ info }) => {\n            log('stopping replication...', { info });\n            // TODO(dmaretskyi): Make sure any peer can stop replication.\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            await this._stopReplication(info.feedKey);\n          },\n        },\n      },\n      port: await context.createPort('rpc', {\n        contentType: 'application/x-protobuf; messageType=\"dxos.rpc.Message\"',\n      }),\n    });\n    await this._rpc.open();\n\n    this._updateTask.schedule();\n  }\n\n  async onClose(err?: Error | undefined) {\n    log('close', { err });\n    await this._ctx.dispose();\n    await this._rpc?.close();\n    for (const feedKey of this._streams.keys()) {\n      await this._stopReplication(feedKey);\n    }\n  }\n\n  @synchronized\n  private async _reevaluateFeeds() {\n    log('_reevaluateFeeds');\n    for (const feedKey of this._feeds.keys()) {\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {\n        try {\n          await asyncTimeout(this._stopReplication(feedKey), 1000);\n        } catch (err) {\n          log.catch(err);\n        }\n      }\n\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (!this._streams.has(feedKey)) {\n        await this._initiateReplication({\n          feedKey,\n          download: true,\n          upload: this._options.upload,\n        });\n      }\n    }\n  }\n\n  /**\n   * Try to initiate feed replication.\n   */\n  private async _initiateReplication(feedInfo: FeedInfo) {\n    log('initiating replication', { feedInfo });\n    invariant(this._extensionContext!.initiator === true, 'Invalid call');\n    invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`);\n    const { streamTag } = await this._rpc!.rpc.ReplicatorService.startReplication({ info: feedInfo });\n    if (!streamTag) {\n      return;\n    }\n\n    await this._replicateFeed(feedInfo, streamTag);\n  }\n\n  /**\n   * Respond to a remote request to replicate a feed.\n   * @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.\n   */\n  @synchronized\n  private async _acceptReplication(feedInfo: FeedInfo): Promise<string | undefined> {\n    invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n    if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {\n      return undefined; // We don't have the feed or we are already replicating it.\n    }\n\n    const tag = `feed-${feedInfo.feedKey.toHex()}-${PublicKey.random().toHex().slice(0, 8)}`; // Generate a unique tag for the stream.\n    await this._replicateFeed(feedInfo, tag);\n    return tag;\n  }\n\n  private async _replicateFeed(info: FeedInfo, streamTag: string) {\n    log('replicate', { info, streamTag });\n    invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`);\n\n    const feed = this._feeds.get(info.feedKey) ?? failUndefined();\n    const networkStream = await this._extensionContext!.createStream(streamTag, {\n      contentType: 'application/x-hypercore',\n    });\n\n    // https://github.com/holepunchto/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options\n    const replicationStream = feed.replicate(true, {\n      live: true,\n      upload: info.upload,\n      download: info.download,\n      noise: false,\n      encrypted: false,\n      maxRequests: 1024,\n    });\n\n    // Left for testing.\n    // const debug = true;\n    // if (debug) {\n    //   feed.on('sync', () => {\n    //     log.info('sync', { key: feed.key, length: feed.length });\n    //   });\n    //   feed.on('download', (index: number, data: any) => {\n    //     log.info('download', {\n    //       key: feed.key,\n    //       index,\n    //       length: feed.length,\n    //       data: data.length,\n    //     });\n    //   });\n    // }\n\n    replicationStream.on('error', (err) => {\n      if (\n        err?.message === 'Writable stream closed prematurely' ||\n        err?.message === 'Cannot call write after a stream was destroyed'\n      ) {\n        return;\n      }\n\n      log.warn('replication stream error', { err, info });\n    });\n\n    this._streams.set(info.feedKey, {\n      streamTag,\n      networkStream,\n      replicationStream,\n      info,\n    });\n\n    networkStream.pipe(replicationStream as any).pipe(networkStream);\n  }\n\n  private async _stopReplication(feedKey: PublicKey) {\n    const stream = this._streams.get(feedKey);\n    if (!stream) {\n      return;\n    }\n\n    stream.networkStream.destroy();\n    this._streams.delete(feedKey);\n  }\n}\n\ntype ServiceBundle = {\n  ReplicatorService: ReplicatorService;\n};\n\ntype ActiveStream = {\n  streamTag: string;\n  networkStream: Duplex;\n  replicationStream: ProtocolStream;\n  info: FeedInfo;\n};\n"],
+  "mappings": ";;;AAOA,SAASA,cAAcC,cAAcC,oBAAoB;AACzD,SAASC,eAAe;AACxB,SAASC,qBAAqB;AAE9B,SAASC,iBAAiB;AAC1B,SAASC,iBAAiB;AAC1B,SAASC,KAAKC,eAAe;AAC7B,SAASC,QAAQC,sBAAsB;AAEvC,SAASC,0BAAwC;AAEjD,SAASC,kBAAkB;;;;;;;;;;;;AASpB,IAAMC,sBAAN,MAAMA;EAAN;AACYC,gBAAO,IAAIX,QAAQ;MAClCY,SAAS,CAACC,QAAAA;AACRT,YAAIU,MAAMD,KAAAA,QAAAA;;;;;;AACV,aAAKE,mBAAmBC,MAAMH,GAAAA;MAChC;IACF,CAAA;AAEiBI,kBAAS,IAAIR,WAAwCN,UAAUe,IAAI;AACnEC,oBAAW,IAAIV,WAAoCN,UAAUe,IAAI;AAK1EE,oBAA+B;MACrCC,QAAQ;IACV;AAEiBC,uBAAc,IAAIxB,aAAa,KAAKa,MAAM,YAAA;AACzD,UAAI;AACF,YAAI,KAAKI,kBAAmBQ,cAAc,OAAO;AAC/C,gBAAM,KAAKC,KAAMC,IAAIC,kBAAkBC,YAAY;YACjDC,OAAOC,MAAMC,KAAK,KAAKb,OAAOc,OAAM,CAAA,EAAIC,IAAI,CAACC,UAAU;cACrDC,SAASD,KAAKE;cACdC,UAAU;cACVf,QAAQ,KAAKD,SAASC;YACxB,EAAA;UACF,CAAA;QACF,WAAW,KAAKN,kBAAmBQ,cAAc,MAAM;AACrD,gBAAM,KAAKc,iBAAgB;QAC7B;MACF,SAASxB,KAAK;AACZ,YAAIA,eAAeN,gBAAgB;AACjC;QACF;AACA,cAAMM;MACR;IACF,CAAA;;EAEA,IACYyB,gBAAgB;AAC1B,WAAO;MACLf,WAAW,KAAKR,mBAAmBQ;MACnCgB,aAAa,KAAKxB,mBAAmBwB;MACrCC,cAAc,KAAKzB,mBAAmByB;MACtCZ,OAAOC,MAAMC,KAAK,KAAKb,OAAOwB,KAAI,CAAA;IACpC;EACF;EAEAC,WAAWC,SAAmC;AAC5C,SAAKvB,WAAWuB;AAChBvC,QAAI,cAAc;MAAEuC;IAAQ,GAAA;;;;;;AAC5B,QAAI,KAAK5B,mBAAmB;AAC1B,WAAKO,YAAYsB,SAAQ;IAC3B;AACA,WAAO;EACT;EAEAC,QAAQZ,MAAwB;AAC9B,SAAKhB,OAAO6B,IAAIb,KAAKE,KAAKF,IAAAA;AAC1B7B,QAAI,WAAW;MAAE8B,SAASD,KAAKE;IAAI,GAAA;;;;;;AACnC,QAAI,KAAKpB,mBAAmB;AAC1B,WAAKO,YAAYsB,SAAQ;IAC3B;EACF;EAEA,MAAMG,OAAOC,SAA2B;AACtC,SAAKjC,oBAAoBiC;AACzB5C,QAAI,QAAA,QAAA;;;;;;AAEJ,SAAKoB,OAAOhB,mBAAiD;MAC3DyC,WAAW;QACTvB,mBAAmBpB,OAAO4C,WAAW,iDAAA;MACvC;MACAC,SAAS;QACPzB,mBAAmBpB,OAAO4C,WAAW,iDAAA;MACvC;MACAE,UAAU;QACR1B,mBAAmB;UACjBC,aAAa,OAAO,EAAEC,MAAK,MAAE;AAC3BxB,gBAAI,sBAAsB;cAAEwB;YAAM,GAAA;;;;;;AAClC1B,sBAAU,KAAKa,kBAAmBQ,cAAc,MAAM,gBAAA;;;;;;;;;AACtD,iBAAKD,YAAYsB,SAAQ;UAC3B;UACAS,kBAAkB,OAAO,EAAEC,KAAI,MAAE;AAC/BlD,gBAAI,2BAA2B;cAAEkD;YAAK,GAAA;;;;;;AACtCpD,sBAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAMgC,YAAY,MAAM,KAAKC,mBAAmBF,IAAAA;AAChD,mBAAO;cACLC;YACF;UACF;UACAE,iBAAiB,OAAO,EAAEH,KAAI,MAAE;AAC9BlD,gBAAI,2BAA2B;cAAEkD;YAAK,GAAA;;;;;;AAEtCpD,sBAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAM,KAAKmC,iBAAiBJ,KAAKpB,OAAO;UAC1C;QACF;MACF;MACAyB,MAAM,MAAMX,QAAQY,WAAW,OAAO;QACpCC,aAAa;MACf,CAAA;IACF,CAAA;AACA,UAAM,KAAKrC,KAAKsC,KAAI;AAEpB,SAAKxC,YAAYsB,SAAQ;EAC3B;EAEA,MAAMmB,QAAQlD,KAAyB;AACrCT,QAAI,SAAS;MAAES;IAAI,GAAA;;;;;;AACnB,UAAM,KAAKF,KAAKqD,QAAO;AACvB,UAAM,KAAKxC,MAAMR,MAAAA;AACjB,eAAWkB,WAAW,KAAKf,SAASsB,KAAI,GAAI;AAC1C,YAAM,KAAKiB,iBAAiBxB,OAAAA;IAC9B;EACF;EAEA,MACcG,mBAAmB;AAC/BjC,QAAI,oBAAA,QAAA;;;;;;AACJ,eAAW8B,WAAW,KAAKjB,OAAOwB,KAAI,GAAI;AACxC,UAAI,KAAK9B,KAAKsD,UAAU;AACtB;MACF;AACA,UAAI,KAAK9C,SAAS+C,IAAIhC,OAAAA,KAAY,KAAKd,SAASC,WAAW,KAAKF,SAASgD,IAAIjC,OAAAA,GAAUoB,KAAKjC,QAAQ;AAClG,YAAI;AACF,gBAAMxB,aAAa,KAAK6D,iBAAiBxB,OAAAA,GAAU,GAAA;QACrD,SAASrB,KAAK;AACZT,cAAIU,MAAMD,KAAAA,QAAAA;;;;;;QACZ;MACF;AAEA,UAAI,KAAKF,KAAKsD,UAAU;AACtB;MACF;AACA,UAAI,CAAC,KAAK9C,SAAS+C,IAAIhC,OAAAA,GAAU;AAC/B,cAAM,KAAKkC,qBAAqB;UAC9BlC;UACAE,UAAU;UACVf,QAAQ,KAAKD,SAASC;QACxB,CAAA;MACF;IACF;EACF;;;;EAKA,MAAc+C,qBAAqBC,UAAoB;AACrDjE,QAAI,0BAA0B;MAAEiE;IAAS,GAAA;;;;;;AACzCnE,cAAU,KAAKa,kBAAmBQ,cAAc,MAAM,gBAAA;;;;;;;;;AACtDrB,cAAU,CAAC,KAAKiB,SAAS+C,IAAIG,SAASnC,OAAO,GAAG,6CAA6CmC,SAASnC,OAAO,IAAE;;;;;;;;;AAC/G,UAAM,EAAEqB,UAAS,IAAK,MAAM,KAAK/B,KAAMC,IAAIC,kBAAkB2B,iBAAiB;MAAEC,MAAMe;IAAS,CAAA;AAC/F,QAAI,CAACd,WAAW;AACd;IACF;AAEA,UAAM,KAAKe,eAAeD,UAAUd,SAAAA;EACtC;;;;;EAMA,MACcC,mBAAmBa,UAAiD;AAChFnE,cAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,QAAI,CAAC,KAAKN,OAAOiD,IAAIG,SAASnC,OAAO,KAAK,KAAKf,SAAS+C,IAAIG,SAASnC,OAAO,GAAG;AAC7E,aAAOqC;IACT;AAEA,UAAMC,MAAM,QAAQH,SAASnC,QAAQuC,MAAK,CAAA,IAAMtE,UAAUuE,OAAM,EAAGD,MAAK,EAAGE,MAAM,GAAG,CAAA,CAAA;AACpF,UAAM,KAAKL,eAAeD,UAAUG,GAAAA;AACpC,WAAOA;EACT;EAEA,MAAcF,eAAehB,MAAgBC,WAAmB;AAC9DnD,QAAI,aAAa;MAAEkD;MAAMC;IAAU,GAAA;;;;;;AACnCrD,cAAU,CAAC,KAAKiB,SAAS+C,IAAIZ,KAAKpB,OAAO,GAAG,6CAA6CoB,KAAKpB,OAAO,IAAE;;;;;;;;;AAEvG,UAAMD,OAAO,KAAKhB,OAAOkD,IAAIb,KAAKpB,OAAO,KAAKjC,cAAAA;AAC9C,UAAM2E,gBAAgB,MAAM,KAAK7D,kBAAmB8D,aAAatB,WAAW;MAC1EM,aAAa;IACf,CAAA;AAGA,UAAMiB,oBAAoB7C,KAAK8C,UAAU,MAAM;MAC7CC,MAAM;MACN3D,QAAQiC,KAAKjC;MACbe,UAAUkB,KAAKlB;MACf6C,OAAO;MACPC,WAAW;MACXC,aAAa;IACf,CAAA;AAkBAL,sBAAkBM,GAAG,SAAS,CAACvE,QAAAA;AAC7B,UACEA,KAAKwE,YAAY,wCACjBxE,KAAKwE,YAAY,kDACjB;AACA;MACF;AAEAjF,UAAIkF,KAAK,4BAA4B;QAAEzE;QAAKyC;MAAK,GAAA;;;;;;IACnD,CAAA;AAEA,SAAKnC,SAAS2B,IAAIQ,KAAKpB,SAAS;MAC9BqB;MACAqB;MACAE;MACAxB;IACF,CAAA;AAEAsB,kBAAcW,KAAKT,iBAAAA,EAA0BS,KAAKX,aAAAA;EACpD;EAEA,MAAclB,iBAAiBxB,SAAoB;AACjD,UAAMsD,SAAS,KAAKrE,SAASgD,IAAIjC,OAAAA;AACjC,QAAI,CAACsD,QAAQ;AACX;IACF;AAEAA,WAAOZ,cAAca,QAAO;AAC5B,SAAKtE,SAASuE,OAAOxD,OAAAA;EACvB;AACF;;EA9MG7B;GAvCUK,oBAAAA,WAAAA,iBAAAA,IAAAA;;EAwHVX;GAxHUW,oBAAAA,WAAAA,oBAAAA,IAAAA;;EAuKVX;GAvKUW,oBAAAA,WAAAA,sBAAAA,IAAAA;",
+  "names": ["asyncTimeout", "DeferredTask", "synchronized", "Context", "failUndefined", "invariant", "PublicKey", "log", "logInfo", "schema", "RpcClosedError", "createProtoRpcPeer", "ComplexMap", "ReplicatorExtension", "_ctx", "onError", "err", "catch", "_extensionContext", "close", "_feeds", "hash", "_streams", "_options", "upload", "_updateTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "requested", "getService", "exposed", "handlers", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "maxRequests", "on", "message", "warn", "pipe", "stream", "destroy", "delete"]
 }

package/dist/lib/browser/meta.json

@@ -1 +1 @@
-{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":33894,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}]},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytes":520,"imports":[{"path":"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts","kind":"import-statement","original":"./replicator-extension"}]}},"outputs":{"packages/core/mesh/teleport-extension-replicator/dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":14689},"packages/core/mesh/teleport-extension-replicator/dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["ReplicatorExtension"],"entryPoint":"packages/core/mesh/teleport-extension-replicator/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytesInOutput":10856},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytesInOutput":0}},"bytes":11039}}}
+{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":35023,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytes":520,"imports":[{"path":"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts","kind":"import-statement","original":"./replicator-extension"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-replicator/dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":15060},"packages/core/mesh/teleport-extension-replicator/dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["ReplicatorExtension"],"entryPoint":"packages/core/mesh/teleport-extension-replicator/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytesInOutput":10570},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytesInOutput":0}},"bytes":10753}}}

package/dist/lib/node/index.cjs

@@ -49,14 +49,13 @@ var ReplicatorExtension = class {
   constructor() {
     this._ctx = new import_context.Context({
       onError: (err) => {
-        var _a;
         import_log.log.catch(err, void 0, {
           F: __dxlog_file,
           L: 31,
           S: this,
           C: (f, a) => f(...a)
         });
-        (_a = this._extensionContext) == null ? void 0 : _a.close(err);
+        this._extensionContext?.close(err);
       }
     });
     this._feeds = new import_util.ComplexMap(import_keys.PublicKey.hash);
@@ -86,11 +85,10 @@ var ReplicatorExtension = class {
     });
   }
   get extensionInfo() {
-    var _a, _b, _c;
     return {
-      initiator: (_a = this._extensionContext) == null ? void 0 : _a.initiator,
-      localPeerId: (_b = this._extensionContext) == null ? void 0 : _b.localPeerId,
-      remotePeerId: (_c = this._extensionContext) == null ? void 0 : _c.remotePeerId,
+      initiator: this._extensionContext?.initiator,
+      localPeerId: this._extensionContext?.localPeerId,
+      remotePeerId: this._extensionContext?.remotePeerId,
       feeds: Array.from(this._feeds.keys())
     };
   }
@@ -213,7 +211,6 @@ var ReplicatorExtension = class {
     this._updateTask.schedule();
   }
   async onClose(err) {
-    var _a;
     (0, import_log.log)("close", {
       err
     }, {
@@ -223,13 +220,12 @@ var ReplicatorExtension = class {
       C: (f, a) => f(...a)
     });
     await this._ctx.dispose();
-    await ((_a = this._rpc) == null ? void 0 : _a.close());
+    await this._rpc?.close();
     for (const feedKey of this._streams.keys()) {
       await this._stopReplication(feedKey);
     }
   }
   async _reevaluateFeeds() {
-    var _a;
     (0, import_log.log)("_reevaluateFeeds", void 0, {
       F: __dxlog_file,
       L: 150,
@@ -240,7 +236,7 @@ var ReplicatorExtension = class {
       if (this._ctx.disposed) {
         return;
       }
-      if (this._streams.has(feedKey) && this._options.upload !== ((_a = this._streams.get(feedKey)) == null ? void 0 : _a.info.upload)) {
+      if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {
         try {
           await (0, import_async.asyncTimeout)(this._stopReplication(feedKey), 1e3);
         } catch (err) {
@@ -324,7 +320,6 @@ var ReplicatorExtension = class {
     return tag;
   }
   async _replicateFeed(info, streamTag) {
-    var _a;
     (0, import_log.log)("replicate", {
       info,
       streamTag
@@ -343,7 +338,7 @@ var ReplicatorExtension = class {
         "`Replication already in progress for feed: ${info.feedKey}`"
       ]
     });
-    const feed = (_a = this._feeds.get(info.feedKey)) != null ? _a : (0, import_debug.failUndefined)();
+    const feed = this._feeds.get(info.feedKey) ?? (0, import_debug.failUndefined)();
     const networkStream = await this._extensionContext.createStream(streamTag, {
       contentType: "application/x-hypercore"
     });
@@ -352,10 +347,11 @@ var ReplicatorExtension = class {
       upload: info.upload,
       download: info.download,
       noise: false,
-      encrypted: false
+      encrypted: false,
+      maxRequests: 1024
     });
     replicationStream.on("error", (err) => {
-      if ((err == null ? void 0 : err.message) === "Writable stream closed prematurely" || (err == null ? void 0 : err.message) === "Cannot call write after a stream was destroyed") {
+      if (err?.message === "Writable stream closed prematurely" || err?.message === "Cannot call write after a stream was destroyed") {
         return;
       }
       import_log.log.warn("replication stream error", {
@@ -363,7 +359,7 @@ var ReplicatorExtension = class {
         info
       }, {
         F: __dxlog_file,
-        L: 240,
+        L: 251,
         S: this,
         C: (f, a) => f(...a)
       });

package/dist/lib/node/index.cjs.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../src/index.ts", "../../../src/replicator-extension.ts"],
-  "sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './replicator-extension';\n", "//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\n\nimport { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { failUndefined } from '@dxos/debug';\nimport { FeedWrapper } from '@dxos/feed-store';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log, logInfo } from '@dxos/log';\nimport { schema, RpcClosedError } from '@dxos/protocols';\nimport { FeedInfo, ReplicatorService } from '@dxos/protocols/proto/dxos/mesh/teleport/replicator';\nimport { createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';\nimport { ExtensionContext, TeleportExtension } from '@dxos/teleport';\nimport { ComplexMap } from '@dxos/util';\n\nexport type ReplicationOptions = {\n  upload: boolean;\n};\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class ReplicatorExtension implements TeleportExtension {\n  private readonly _ctx = new Context({\n    onError: (err) => {\n      log.catch(err);\n      this._extensionContext?.close(err);\n    },\n  });\n\n  private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);\n  private readonly _streams = new ComplexMap<PublicKey, ActiveStream>(PublicKey.hash);\n\n  private _rpc?: ProtoRpcPeer<ServiceBundle>;\n  private _extensionContext?: ExtensionContext;\n\n  private _options: ReplicationOptions = {\n    upload: false,\n  };\n\n  private readonly _updateTask = new DeferredTask(this._ctx, async () => {\n    try {\n      if (this._extensionContext!.initiator === false) {\n        await this._rpc!.rpc.ReplicatorService.updateFeeds({\n          feeds: Array.from(this._feeds.values()).map((feed) => ({\n            feedKey: feed.key,\n            download: true,\n            upload: this._options.upload,\n          })),\n        });\n      } else if (this._extensionContext!.initiator === true) {\n        await this._reevaluateFeeds();\n      }\n    } catch (err) {\n      if (err instanceof RpcClosedError) {\n        return; // Some RPC requests might be pending while closing.\n      }\n      throw err;\n    }\n  });\n\n  @logInfo\n  private get extensionInfo() {\n    return {\n      initiator: this._extensionContext?.initiator,\n      localPeerId: this._extensionContext?.localPeerId,\n      remotePeerId: this._extensionContext?.remotePeerId,\n      feeds: Array.from(this._feeds.keys()),\n    };\n  }\n\n  setOptions(options: ReplicationOptions): this {\n    this._options = options;\n    log('setOptions', { options });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n    return this;\n  }\n\n  addFeed(feed: FeedWrapper<any>) {\n    this._feeds.set(feed.key, feed);\n    log('addFeed', { feedKey: feed.key });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n  }\n\n  async onOpen(context: ExtensionContext) {\n    this._extensionContext = context;\n    log('open');\n\n    this._rpc = createProtoRpcPeer<ServiceBundle, ServiceBundle>({\n      requested: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      exposed: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      handlers: {\n        ReplicatorService: {\n          updateFeeds: async ({ feeds }) => {\n            log('received feed info', { feeds });\n            invariant(this._extensionContext!.initiator === true, 'Invalid call');\n            this._updateTask.schedule();\n          },\n          startReplication: async ({ info }) => {\n            log('starting replication...', { info });\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            const streamTag = await this._acceptReplication(info);\n            return {\n              streamTag,\n            };\n          },\n          stopReplication: async ({ info }) => {\n            log('stopping replication...', { info });\n            // TODO(dmaretskyi): Make sure any peer can stop replication.\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            await this._stopReplication(info.feedKey);\n          },\n        },\n      },\n      port: await context.createPort('rpc', {\n        contentType: 'application/x-protobuf; messageType=\"dxos.rpc.Message\"',\n      }),\n    });\n    await this._rpc.open();\n\n    this._updateTask.schedule();\n  }\n\n  async onClose(err?: Error | undefined) {\n    log('close', { err });\n    await this._ctx.dispose();\n    await this._rpc?.close();\n    for (const feedKey of this._streams.keys()) {\n      await this._stopReplication(feedKey);\n    }\n  }\n\n  @synchronized\n  private async _reevaluateFeeds() {\n    log('_reevaluateFeeds');\n    for (const feedKey of this._feeds.keys()) {\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {\n        try {\n          await asyncTimeout(this._stopReplication(feedKey), 1000);\n        } catch (err) {\n          log.catch(err);\n        }\n      }\n\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (!this._streams.has(feedKey)) {\n        await this._initiateReplication({\n          feedKey,\n          download: true,\n          upload: this._options.upload,\n        });\n      }\n    }\n  }\n\n  /**\n   * Try to initiate feed replication.\n   */\n  private async _initiateReplication(feedInfo: FeedInfo) {\n    log('initiating replication', { feedInfo });\n    invariant(this._extensionContext!.initiator === true, 'Invalid call');\n    invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`);\n    const { streamTag } = await this._rpc!.rpc.ReplicatorService.startReplication({ info: feedInfo });\n    if (!streamTag) {\n      return;\n    }\n\n    await this._replicateFeed(feedInfo, streamTag);\n  }\n\n  /**\n   * Respond to a remote request to replicate a feed.\n   * @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.\n   */\n  @synchronized\n  private async _acceptReplication(feedInfo: FeedInfo): Promise<string | undefined> {\n    invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n    if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {\n      return undefined; // We don't have the feed or we are already replicating it.\n    }\n\n    const tag = `feed-${feedInfo.feedKey.toHex()}-${PublicKey.random().toHex().slice(0, 8)}`; // Generate a unique tag for the stream.\n    await this._replicateFeed(feedInfo, tag);\n    return tag;\n  }\n\n  private async _replicateFeed(info: FeedInfo, streamTag: string) {\n    log('replicate', { info, streamTag });\n    invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`);\n\n    const feed = this._feeds.get(info.feedKey) ?? failUndefined();\n    const networkStream = await this._extensionContext!.createStream(streamTag, {\n      contentType: 'application/x-hypercore',\n    });\n    const replicationStream = feed.replicate(true, {\n      live: true,\n      upload: info.upload,\n      download: info.download,\n      noise: false,\n      encrypted: false,\n    });\n\n    // left for testing\n    // feed.on('download', (index: number, data: any) => {\n    //   log('download', {\n    //     key: info.feedKey,\n    //     index\n    //   });\n    // });\n\n    replicationStream.on('error', (err) => {\n      if (\n        err?.message === 'Writable stream closed prematurely' ||\n        err?.message === 'Cannot call write after a stream was destroyed'\n      ) {\n        return;\n      }\n\n      log.warn('replication stream error', { err, info });\n    });\n\n    this._streams.set(info.feedKey, {\n      streamTag,\n      networkStream,\n      replicationStream,\n      info,\n    });\n\n    networkStream.pipe(replicationStream as any).pipe(networkStream);\n  }\n\n  private async _stopReplication(feedKey: PublicKey) {\n    const stream = this._streams.get(feedKey);\n    if (!stream) {\n      return;\n    }\n\n    stream.networkStream.destroy();\n    this._streams.delete(feedKey);\n  }\n}\n\ntype ServiceBundle = {\n  ReplicatorService: ReplicatorService;\n};\n\ntype ActiveStream = {\n  streamTag: string;\n  networkStream: Duplex;\n  replicationStream: ProtocolStream;\n  info: FeedInfo;\n};\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACOA,mBAAyD;AACzD,qBAAwB;AACxB,mBAA8B;AAE9B,uBAA0B;AAC1B,kBAA0B;AAC1B,iBAA6B;AAC7B,uBAAuC;AAEvC,iBAAiD;AAEjD,kBAA2B;;;;;;;;;;;;AASpB,IAAMA,sBAAN,MAAMA;EAAN;AACYC,gBAAO,IAAIC,uBAAQ;MAClCC,SAAS,CAACC,QAAAA;AA7Bd;AA8BMC,uBAAIC,MAAMF,KAAAA,QAAAA;;;;;;AACV,mBAAKG,sBAAL,mBAAwBC,MAAMJ;MAChC;IACF,CAAA;AAEiBK,kBAAS,IAAIC,uBAAwCC,sBAAUC,IAAI;AACnEC,oBAAW,IAAIH,uBAAoCC,sBAAUC,IAAI;AAK1EE,oBAA+B;MACrCC,QAAQ;IACV;AAEiBC,uBAAc,IAAIC,0BAAa,KAAKhB,MAAM,YAAA;AACzD,UAAI;AACF,YAAI,KAAKM,kBAAmBW,cAAc,OAAO;AAC/C,gBAAM,KAAKC,KAAMC,IAAIC,kBAAkBC,YAAY;YACjDC,OAAOC,MAAMC,KAAK,KAAKhB,OAAOiB,OAAM,CAAA,EAAIC,IAAI,CAACC,UAAU;cACrDC,SAASD,KAAKE;cACdC,UAAU;cACVhB,QAAQ,KAAKD,SAASC;YACxB,EAAA;UACF,CAAA;QACF,WAAW,KAAKR,kBAAmBW,cAAc,MAAM;AACrD,gBAAM,KAAKc,iBAAgB;QAC7B;MACF,SAAS5B,KAAP;AACA,YAAIA,eAAe6B,iCAAgB;AACjC;QACF;AACA,cAAM7B;MACR;IACF,CAAA;;EAEA,IACY8B,gBAAgB;AAnE9B;AAoEI,WAAO;MACLhB,YAAW,UAAKX,sBAAL,mBAAwBW;MACnCiB,cAAa,UAAK5B,sBAAL,mBAAwB4B;MACrCC,eAAc,UAAK7B,sBAAL,mBAAwB6B;MACtCb,OAAOC,MAAMC,KAAK,KAAKhB,OAAO4B,KAAI,CAAA;IACpC;EACF;EAEAC,WAAWC,SAAmC;AAC5C,SAAKzB,WAAWyB;AAChBlC,wBAAI,cAAc;MAAEkC;IAAQ,GAAA;;;;;;AAC5B,QAAI,KAAKhC,mBAAmB;AAC1B,WAAKS,YAAYwB,SAAQ;IAC3B;AACA,WAAO;EACT;EAEAC,QAAQb,MAAwB;AAC9B,SAAKnB,OAAOiC,IAAId,KAAKE,KAAKF,IAAAA;AAC1BvB,wBAAI,WAAW;MAAEwB,SAASD,KAAKE;IAAI,GAAA;;;;;;AACnC,QAAI,KAAKvB,mBAAmB;AAC1B,WAAKS,YAAYwB,SAAQ;IAC3B;EACF;EAEA,MAAMG,OAAOC,SAA2B;AACtC,SAAKrC,oBAAoBqC;AACzBvC,wBAAI,QAAA,QAAA;;;;;;AAEJ,SAAKc,WAAO0B,+BAAiD;MAC3DC,WAAW;QACTzB,mBAAmB0B,wBAAOC,WAAW,iDAAA;MACvC;MACAC,SAAS;QACP5B,mBAAmB0B,wBAAOC,WAAW,iDAAA;MACvC;MACAE,UAAU;QACR7B,mBAAmB;UACjBC,aAAa,OAAO,EAAEC,MAAK,MAAE;AAC3BlB,gCAAI,sBAAsB;cAAEkB;YAAM,GAAA;;;;;;AAClC4B,4CAAU,KAAK5C,kBAAmBW,cAAc,MAAM,gBAAA;;;;;;;;;AACtD,iBAAKF,YAAYwB,SAAQ;UAC3B;UACAY,kBAAkB,OAAO,EAAEC,KAAI,MAAE;AAC/BhD,gCAAI,2BAA2B;cAAEgD;YAAK,GAAA;;;;;;AACtCF,4CAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAMoC,YAAY,MAAM,KAAKC,mBAAmBF,IAAAA;AAChD,mBAAO;cACLC;YACF;UACF;UACAE,iBAAiB,OAAO,EAAEH,KAAI,MAAE;AAC9BhD,gCAAI,2BAA2B;cAAEgD;YAAK,GAAA;;;;;;AAEtCF,4CAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAM,KAAKuC,iBAAiBJ,KAAKxB,OAAO;UAC1C;QACF;MACF;MACA6B,MAAM,MAAMd,QAAQe,WAAW,OAAO;QACpCC,aAAa;MACf,CAAA;IACF,CAAA;AACA,UAAM,KAAKzC,KAAK0C,KAAI;AAEpB,SAAK7C,YAAYwB,SAAQ;EAC3B;EAEA,MAAMsB,QAAQ1D,KAAyB;AA1IzC;AA2IIC,wBAAI,SAAS;MAAED;IAAI,GAAA;;;;;;AACnB,UAAM,KAAKH,KAAK8D,QAAO;AACvB,YAAM,UAAK5C,SAAL,mBAAWX;AACjB,eAAWqB,WAAW,KAAKhB,SAASwB,KAAI,GAAI;AAC1C,YAAM,KAAKoB,iBAAiB5B,OAAAA;IAC9B;EACF;EAEA,MACcG,mBAAmB;AApJnC;AAqJI3B,wBAAI,oBAAA,QAAA;;;;;;AACJ,eAAWwB,WAAW,KAAKpB,OAAO4B,KAAI,GAAI;AACxC,UAAI,KAAKpC,KAAK+D,UAAU;AACtB;MACF;AACA,UAAI,KAAKnD,SAASoD,IAAIpC,OAAAA,KAAY,KAAKf,SAASC,aAAW,UAAKF,SAASqD,IAAIrC,OAAAA,MAAlB,mBAA4BwB,KAAKtC,SAAQ;AAClG,YAAI;AACF,oBAAMoD,2BAAa,KAAKV,iBAAiB5B,OAAAA,GAAU,GAAA;QACrD,SAASzB,KAAP;AACAC,yBAAIC,MAAMF,KAAAA,QAAAA;;;;;;QACZ;MACF;AAEA,UAAI,KAAKH,KAAK+D,UAAU;AACtB;MACF;AACA,UAAI,CAAC,KAAKnD,SAASoD,IAAIpC,OAAAA,GAAU;AAC/B,cAAM,KAAKuC,qBAAqB;UAC9BvC;UACAE,UAAU;UACVhB,QAAQ,KAAKD,SAASC;QACxB,CAAA;MACF;IACF;EACF;;;;EAKA,MAAcqD,qBAAqBC,UAAoB;AACrDhE,wBAAI,0BAA0B;MAAEgE;IAAS,GAAA;;;;;;AACzClB,oCAAU,KAAK5C,kBAAmBW,cAAc,MAAM,gBAAA;;;;;;;;;AACtDiC,oCAAU,CAAC,KAAKtC,SAASoD,IAAII,SAASxC,OAAO,GAAG,6CAA6CwC,SAASxC,WAAS;;;;;;;;;AAC/G,UAAM,EAAEyB,UAAS,IAAK,MAAM,KAAKnC,KAAMC,IAAIC,kBAAkB+B,iBAAiB;MAAEC,MAAMgB;IAAS,CAAA;AAC/F,QAAI,CAACf,WAAW;AACd;IACF;AAEA,UAAM,KAAKgB,eAAeD,UAAUf,SAAAA;EACtC;;;;;EAMA,MACcC,mBAAmBc,UAAiD;AAChFlB,oCAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,QAAI,CAAC,KAAKT,OAAOwD,IAAII,SAASxC,OAAO,KAAK,KAAKhB,SAASoD,IAAII,SAASxC,OAAO,GAAG;AAC7E,aAAO0C;IACT;AAEA,UAAMC,MAAM,QAAQH,SAASxC,QAAQ4C,MAAK,KAAM9D,sBAAU+D,OAAM,EAAGD,MAAK,EAAGE,MAAM,GAAG,CAAA;AACpF,UAAM,KAAKL,eAAeD,UAAUG,GAAAA;AACpC,WAAOA;EACT;EAEA,MAAcF,eAAejB,MAAgBC,WAAmB;AA/MlE;AAgNIjD,wBAAI,aAAa;MAAEgD;MAAMC;IAAU,GAAA;;;;;;AACnCH,oCAAU,CAAC,KAAKtC,SAASoD,IAAIZ,KAAKxB,OAAO,GAAG,6CAA6CwB,KAAKxB,WAAS;;;;;;;;;AAEvG,UAAMD,QAAO,UAAKnB,OAAOyD,IAAIb,KAAKxB,OAAO,MAA5B,gBAAiC+C,4BAAAA;AAC9C,UAAMC,gBAAgB,MAAM,KAAKtE,kBAAmBuE,aAAaxB,WAAW;MAC1EM,aAAa;IACf,CAAA;AACA,UAAMmB,oBAAoBnD,KAAKoD,UAAU,MAAM;MAC7CC,MAAM;MACNlE,QAAQsC,KAAKtC;MACbgB,UAAUsB,KAAKtB;MACfmD,OAAO;MACPC,WAAW;IACb,CAAA;AAUAJ,sBAAkBK,GAAG,SAAS,CAAChF,QAAAA;AAC7B,WACEA,2BAAKiF,aAAY,yCACjBjF,2BAAKiF,aAAY,kDACjB;AACA;MACF;AAEAhF,qBAAIiF,KAAK,4BAA4B;QAAElF;QAAKiD;MAAK,GAAA;;;;;;IACnD,CAAA;AAEA,SAAKxC,SAAS6B,IAAIW,KAAKxB,SAAS;MAC9ByB;MACAuB;MACAE;MACA1B;IACF,CAAA;AAEAwB,kBAAcU,KAAKR,iBAAAA,EAA0BQ,KAAKV,aAAAA;EACpD;EAEA,MAAcpB,iBAAiB5B,SAAoB;AACjD,UAAM2D,SAAS,KAAK3E,SAASqD,IAAIrC,OAAAA;AACjC,QAAI,CAAC2D,QAAQ;AACX;IACF;AAEAA,WAAOX,cAAcY,QAAO;AAC5B,SAAK5E,SAAS6E,OAAO7D,OAAAA;EACvB;AACF;;EAnMG8D;GAvCU3F,oBAAAA,WAAAA,iBAAAA,IAAAA;;EAwHV4F;GAxHU5F,oBAAAA,WAAAA,oBAAAA,IAAAA;;EAuKV4F;GAvKU5F,oBAAAA,WAAAA,sBAAAA,IAAAA;",
-  "names": ["ReplicatorExtension", "_ctx", "Context", "onError", "err", "log", "catch", "_extensionContext", "close", "_feeds", "ComplexMap", "PublicKey", "hash", "_streams", "_options", "upload", "_updateTask", "DeferredTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "RpcClosedError", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "createProtoRpcPeer", "requested", "schema", "getService", "exposed", "handlers", "invariant", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "asyncTimeout", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "failUndefined", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "on", "message", "warn", "pipe", "stream", "destroy", "delete", "logInfo", "synchronized"]
+  "sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './replicator-extension';\n", "//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\n\nimport { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { failUndefined } from '@dxos/debug';\nimport { FeedWrapper } from '@dxos/feed-store';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log, logInfo } from '@dxos/log';\nimport { schema, RpcClosedError } from '@dxos/protocols';\nimport { FeedInfo, ReplicatorService } from '@dxos/protocols/proto/dxos/mesh/teleport/replicator';\nimport { createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';\nimport { ExtensionContext, TeleportExtension } from '@dxos/teleport';\nimport { ComplexMap } from '@dxos/util';\n\nexport type ReplicationOptions = {\n  upload: boolean;\n};\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class ReplicatorExtension implements TeleportExtension {\n  private readonly _ctx = new Context({\n    onError: (err) => {\n      log.catch(err);\n      this._extensionContext?.close(err);\n    },\n  });\n\n  private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);\n  private readonly _streams = new ComplexMap<PublicKey, ActiveStream>(PublicKey.hash);\n\n  private _rpc?: ProtoRpcPeer<ServiceBundle>;\n  private _extensionContext?: ExtensionContext;\n\n  private _options: ReplicationOptions = {\n    upload: false,\n  };\n\n  private readonly _updateTask = new DeferredTask(this._ctx, async () => {\n    try {\n      if (this._extensionContext!.initiator === false) {\n        await this._rpc!.rpc.ReplicatorService.updateFeeds({\n          feeds: Array.from(this._feeds.values()).map((feed) => ({\n            feedKey: feed.key,\n            download: true,\n            upload: this._options.upload,\n          })),\n        });\n      } else if (this._extensionContext!.initiator === true) {\n        await this._reevaluateFeeds();\n      }\n    } catch (err) {\n      if (err instanceof RpcClosedError) {\n        return; // Some RPC requests might be pending while closing.\n      }\n      throw err;\n    }\n  });\n\n  @logInfo\n  private get extensionInfo() {\n    return {\n      initiator: this._extensionContext?.initiator,\n      localPeerId: this._extensionContext?.localPeerId,\n      remotePeerId: this._extensionContext?.remotePeerId,\n      feeds: Array.from(this._feeds.keys()),\n    };\n  }\n\n  setOptions(options: ReplicationOptions): this {\n    this._options = options;\n    log('setOptions', { options });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n    return this;\n  }\n\n  addFeed(feed: FeedWrapper<any>) {\n    this._feeds.set(feed.key, feed);\n    log('addFeed', { feedKey: feed.key });\n    if (this._extensionContext) {\n      this._updateTask.schedule();\n    }\n  }\n\n  async onOpen(context: ExtensionContext) {\n    this._extensionContext = context;\n    log('open');\n\n    this._rpc = createProtoRpcPeer<ServiceBundle, ServiceBundle>({\n      requested: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      exposed: {\n        ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n      },\n      handlers: {\n        ReplicatorService: {\n          updateFeeds: async ({ feeds }) => {\n            log('received feed info', { feeds });\n            invariant(this._extensionContext!.initiator === true, 'Invalid call');\n            this._updateTask.schedule();\n          },\n          startReplication: async ({ info }) => {\n            log('starting replication...', { info });\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            const streamTag = await this._acceptReplication(info);\n            return {\n              streamTag,\n            };\n          },\n          stopReplication: async ({ info }) => {\n            log('stopping replication...', { info });\n            // TODO(dmaretskyi): Make sure any peer can stop replication.\n            invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n            await this._stopReplication(info.feedKey);\n          },\n        },\n      },\n      port: await context.createPort('rpc', {\n        contentType: 'application/x-protobuf; messageType=\"dxos.rpc.Message\"',\n      }),\n    });\n    await this._rpc.open();\n\n    this._updateTask.schedule();\n  }\n\n  async onClose(err?: Error | undefined) {\n    log('close', { err });\n    await this._ctx.dispose();\n    await this._rpc?.close();\n    for (const feedKey of this._streams.keys()) {\n      await this._stopReplication(feedKey);\n    }\n  }\n\n  @synchronized\n  private async _reevaluateFeeds() {\n    log('_reevaluateFeeds');\n    for (const feedKey of this._feeds.keys()) {\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {\n        try {\n          await asyncTimeout(this._stopReplication(feedKey), 1000);\n        } catch (err) {\n          log.catch(err);\n        }\n      }\n\n      if (this._ctx.disposed) {\n        return;\n      }\n      if (!this._streams.has(feedKey)) {\n        await this._initiateReplication({\n          feedKey,\n          download: true,\n          upload: this._options.upload,\n        });\n      }\n    }\n  }\n\n  /**\n   * Try to initiate feed replication.\n   */\n  private async _initiateReplication(feedInfo: FeedInfo) {\n    log('initiating replication', { feedInfo });\n    invariant(this._extensionContext!.initiator === true, 'Invalid call');\n    invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`);\n    const { streamTag } = await this._rpc!.rpc.ReplicatorService.startReplication({ info: feedInfo });\n    if (!streamTag) {\n      return;\n    }\n\n    await this._replicateFeed(feedInfo, streamTag);\n  }\n\n  /**\n   * Respond to a remote request to replicate a feed.\n   * @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.\n   */\n  @synchronized\n  private async _acceptReplication(feedInfo: FeedInfo): Promise<string | undefined> {\n    invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n    if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {\n      return undefined; // We don't have the feed or we are already replicating it.\n    }\n\n    const tag = `feed-${feedInfo.feedKey.toHex()}-${PublicKey.random().toHex().slice(0, 8)}`; // Generate a unique tag for the stream.\n    await this._replicateFeed(feedInfo, tag);\n    return tag;\n  }\n\n  private async _replicateFeed(info: FeedInfo, streamTag: string) {\n    log('replicate', { info, streamTag });\n    invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`);\n\n    const feed = this._feeds.get(info.feedKey) ?? failUndefined();\n    const networkStream = await this._extensionContext!.createStream(streamTag, {\n      contentType: 'application/x-hypercore',\n    });\n\n    // https://github.com/holepunchto/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options\n    const replicationStream = feed.replicate(true, {\n      live: true,\n      upload: info.upload,\n      download: info.download,\n      noise: false,\n      encrypted: false,\n      maxRequests: 1024,\n    });\n\n    // Left for testing.\n    // const debug = true;\n    // if (debug) {\n    //   feed.on('sync', () => {\n    //     log.info('sync', { key: feed.key, length: feed.length });\n    //   });\n    //   feed.on('download', (index: number, data: any) => {\n    //     log.info('download', {\n    //       key: feed.key,\n    //       index,\n    //       length: feed.length,\n    //       data: data.length,\n    //     });\n    //   });\n    // }\n\n    replicationStream.on('error', (err) => {\n      if (\n        err?.message === 'Writable stream closed prematurely' ||\n        err?.message === 'Cannot call write after a stream was destroyed'\n      ) {\n        return;\n      }\n\n      log.warn('replication stream error', { err, info });\n    });\n\n    this._streams.set(info.feedKey, {\n      streamTag,\n      networkStream,\n      replicationStream,\n      info,\n    });\n\n    networkStream.pipe(replicationStream as any).pipe(networkStream);\n  }\n\n  private async _stopReplication(feedKey: PublicKey) {\n    const stream = this._streams.get(feedKey);\n    if (!stream) {\n      return;\n    }\n\n    stream.networkStream.destroy();\n    this._streams.delete(feedKey);\n  }\n}\n\ntype ServiceBundle = {\n  ReplicatorService: ReplicatorService;\n};\n\ntype ActiveStream = {\n  streamTag: string;\n  networkStream: Duplex;\n  replicationStream: ProtocolStream;\n  info: FeedInfo;\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACOA,mBAAyD;AACzD,qBAAwB;AACxB,mBAA8B;AAE9B,uBAA0B;AAC1B,kBAA0B;AAC1B,iBAA6B;AAC7B,uBAAuC;AAEvC,iBAAiD;AAEjD,kBAA2B;;;;;;;;;;;;AASpB,IAAMA,sBAAN,MAAMA;EAAN;AACYC,gBAAO,IAAIC,uBAAQ;MAClCC,SAAS,CAACC,QAAAA;AACRC,uBAAIC,MAAMF,KAAAA,QAAAA;;;;;;AACV,aAAKG,mBAAmBC,MAAMJ,GAAAA;MAChC;IACF,CAAA;AAEiBK,kBAAS,IAAIC,uBAAwCC,sBAAUC,IAAI;AACnEC,oBAAW,IAAIH,uBAAoCC,sBAAUC,IAAI;AAK1EE,oBAA+B;MACrCC,QAAQ;IACV;AAEiBC,uBAAc,IAAIC,0BAAa,KAAKhB,MAAM,YAAA;AACzD,UAAI;AACF,YAAI,KAAKM,kBAAmBW,cAAc,OAAO;AAC/C,gBAAM,KAAKC,KAAMC,IAAIC,kBAAkBC,YAAY;YACjDC,OAAOC,MAAMC,KAAK,KAAKhB,OAAOiB,OAAM,CAAA,EAAIC,IAAI,CAACC,UAAU;cACrDC,SAASD,KAAKE;cACdC,UAAU;cACVhB,QAAQ,KAAKD,SAASC;YACxB,EAAA;UACF,CAAA;QACF,WAAW,KAAKR,kBAAmBW,cAAc,MAAM;AACrD,gBAAM,KAAKc,iBAAgB;QAC7B;MACF,SAAS5B,KAAK;AACZ,YAAIA,eAAe6B,iCAAgB;AACjC;QACF;AACA,cAAM7B;MACR;IACF,CAAA;;EAEA,IACY8B,gBAAgB;AAC1B,WAAO;MACLhB,WAAW,KAAKX,mBAAmBW;MACnCiB,aAAa,KAAK5B,mBAAmB4B;MACrCC,cAAc,KAAK7B,mBAAmB6B;MACtCb,OAAOC,MAAMC,KAAK,KAAKhB,OAAO4B,KAAI,CAAA;IACpC;EACF;EAEAC,WAAWC,SAAmC;AAC5C,SAAKzB,WAAWyB;AAChBlC,wBAAI,cAAc;MAAEkC;IAAQ,GAAA;;;;;;AAC5B,QAAI,KAAKhC,mBAAmB;AAC1B,WAAKS,YAAYwB,SAAQ;IAC3B;AACA,WAAO;EACT;EAEAC,QAAQb,MAAwB;AAC9B,SAAKnB,OAAOiC,IAAId,KAAKE,KAAKF,IAAAA;AAC1BvB,wBAAI,WAAW;MAAEwB,SAASD,KAAKE;IAAI,GAAA;;;;;;AACnC,QAAI,KAAKvB,mBAAmB;AAC1B,WAAKS,YAAYwB,SAAQ;IAC3B;EACF;EAEA,MAAMG,OAAOC,SAA2B;AACtC,SAAKrC,oBAAoBqC;AACzBvC,wBAAI,QAAA,QAAA;;;;;;AAEJ,SAAKc,WAAO0B,+BAAiD;MAC3DC,WAAW;QACTzB,mBAAmB0B,wBAAOC,WAAW,iDAAA;MACvC;MACAC,SAAS;QACP5B,mBAAmB0B,wBAAOC,WAAW,iDAAA;MACvC;MACAE,UAAU;QACR7B,mBAAmB;UACjBC,aAAa,OAAO,EAAEC,MAAK,MAAE;AAC3BlB,gCAAI,sBAAsB;cAAEkB;YAAM,GAAA;;;;;;AAClC4B,4CAAU,KAAK5C,kBAAmBW,cAAc,MAAM,gBAAA;;;;;;;;;AACtD,iBAAKF,YAAYwB,SAAQ;UAC3B;UACAY,kBAAkB,OAAO,EAAEC,KAAI,MAAE;AAC/BhD,gCAAI,2BAA2B;cAAEgD;YAAK,GAAA;;;;;;AACtCF,4CAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAMoC,YAAY,MAAM,KAAKC,mBAAmBF,IAAAA;AAChD,mBAAO;cACLC;YACF;UACF;UACAE,iBAAiB,OAAO,EAAEH,KAAI,MAAE;AAC9BhD,gCAAI,2BAA2B;cAAEgD;YAAK,GAAA;;;;;;AAEtCF,4CAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAM,KAAKuC,iBAAiBJ,KAAKxB,OAAO;UAC1C;QACF;MACF;MACA6B,MAAM,MAAMd,QAAQe,WAAW,OAAO;QACpCC,aAAa;MACf,CAAA;IACF,CAAA;AACA,UAAM,KAAKzC,KAAK0C,KAAI;AAEpB,SAAK7C,YAAYwB,SAAQ;EAC3B;EAEA,MAAMsB,QAAQ1D,KAAyB;AACrCC,wBAAI,SAAS;MAAED;IAAI,GAAA;;;;;;AACnB,UAAM,KAAKH,KAAK8D,QAAO;AACvB,UAAM,KAAK5C,MAAMX,MAAAA;AACjB,eAAWqB,WAAW,KAAKhB,SAASwB,KAAI,GAAI;AAC1C,YAAM,KAAKoB,iBAAiB5B,OAAAA;IAC9B;EACF;EAEA,MACcG,mBAAmB;AAC/B3B,wBAAI,oBAAA,QAAA;;;;;;AACJ,eAAWwB,WAAW,KAAKpB,OAAO4B,KAAI,GAAI;AACxC,UAAI,KAAKpC,KAAK+D,UAAU;AACtB;MACF;AACA,UAAI,KAAKnD,SAASoD,IAAIpC,OAAAA,KAAY,KAAKf,SAASC,WAAW,KAAKF,SAASqD,IAAIrC,OAAAA,GAAUwB,KAAKtC,QAAQ;AAClG,YAAI;AACF,oBAAMoD,2BAAa,KAAKV,iBAAiB5B,OAAAA,GAAU,GAAA;QACrD,SAASzB,KAAK;AACZC,yBAAIC,MAAMF,KAAAA,QAAAA;;;;;;QACZ;MACF;AAEA,UAAI,KAAKH,KAAK+D,UAAU;AACtB;MACF;AACA,UAAI,CAAC,KAAKnD,SAASoD,IAAIpC,OAAAA,GAAU;AAC/B,cAAM,KAAKuC,qBAAqB;UAC9BvC;UACAE,UAAU;UACVhB,QAAQ,KAAKD,SAASC;QACxB,CAAA;MACF;IACF;EACF;;;;EAKA,MAAcqD,qBAAqBC,UAAoB;AACrDhE,wBAAI,0BAA0B;MAAEgE;IAAS,GAAA;;;;;;AACzClB,oCAAU,KAAK5C,kBAAmBW,cAAc,MAAM,gBAAA;;;;;;;;;AACtDiC,oCAAU,CAAC,KAAKtC,SAASoD,IAAII,SAASxC,OAAO,GAAG,6CAA6CwC,SAASxC,OAAO,IAAE;;;;;;;;;AAC/G,UAAM,EAAEyB,UAAS,IAAK,MAAM,KAAKnC,KAAMC,IAAIC,kBAAkB+B,iBAAiB;MAAEC,MAAMgB;IAAS,CAAA;AAC/F,QAAI,CAACf,WAAW;AACd;IACF;AAEA,UAAM,KAAKgB,eAAeD,UAAUf,SAAAA;EACtC;;;;;EAMA,MACcC,mBAAmBc,UAAiD;AAChFlB,oCAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,QAAI,CAAC,KAAKT,OAAOwD,IAAII,SAASxC,OAAO,KAAK,KAAKhB,SAASoD,IAAII,SAASxC,OAAO,GAAG;AAC7E,aAAO0C;IACT;AAEA,UAAMC,MAAM,QAAQH,SAASxC,QAAQ4C,MAAK,CAAA,IAAM9D,sBAAU+D,OAAM,EAAGD,MAAK,EAAGE,MAAM,GAAG,CAAA,CAAA;AACpF,UAAM,KAAKL,eAAeD,UAAUG,GAAAA;AACpC,WAAOA;EACT;EAEA,MAAcF,eAAejB,MAAgBC,WAAmB;AAC9DjD,wBAAI,aAAa;MAAEgD;MAAMC;IAAU,GAAA;;;;;;AACnCH,oCAAU,CAAC,KAAKtC,SAASoD,IAAIZ,KAAKxB,OAAO,GAAG,6CAA6CwB,KAAKxB,OAAO,IAAE;;;;;;;;;AAEvG,UAAMD,OAAO,KAAKnB,OAAOyD,IAAIb,KAAKxB,OAAO,SAAK+C,4BAAAA;AAC9C,UAAMC,gBAAgB,MAAM,KAAKtE,kBAAmBuE,aAAaxB,WAAW;MAC1EM,aAAa;IACf,CAAA;AAGA,UAAMmB,oBAAoBnD,KAAKoD,UAAU,MAAM;MAC7CC,MAAM;MACNlE,QAAQsC,KAAKtC;MACbgB,UAAUsB,KAAKtB;MACfmD,OAAO;MACPC,WAAW;MACXC,aAAa;IACf,CAAA;AAkBAL,sBAAkBM,GAAG,SAAS,CAACjF,QAAAA;AAC7B,UACEA,KAAKkF,YAAY,wCACjBlF,KAAKkF,YAAY,kDACjB;AACA;MACF;AAEAjF,qBAAIkF,KAAK,4BAA4B;QAAEnF;QAAKiD;MAAK,GAAA;;;;;;IACnD,CAAA;AAEA,SAAKxC,SAAS6B,IAAIW,KAAKxB,SAAS;MAC9ByB;MACAuB;MACAE;MACA1B;IACF,CAAA;AAEAwB,kBAAcW,KAAKT,iBAAAA,EAA0BS,KAAKX,aAAAA;EACpD;EAEA,MAAcpB,iBAAiB5B,SAAoB;AACjD,UAAM4D,SAAS,KAAK5E,SAASqD,IAAIrC,OAAAA;AACjC,QAAI,CAAC4D,QAAQ;AACX;IACF;AAEAA,WAAOZ,cAAca,QAAO;AAC5B,SAAK7E,SAAS8E,OAAO9D,OAAAA;EACvB;AACF;;EA9MG+D;GAvCU5F,oBAAAA,WAAAA,iBAAAA,IAAAA;;EAwHV6F;GAxHU7F,oBAAAA,WAAAA,oBAAAA,IAAAA;;EAuKV6F;GAvKU7F,oBAAAA,WAAAA,sBAAAA,IAAAA;",
+  "names": ["ReplicatorExtension", "_ctx", "Context", "onError", "err", "log", "catch", "_extensionContext", "close", "_feeds", "ComplexMap", "PublicKey", "hash", "_streams", "_options", "upload", "_updateTask", "DeferredTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "RpcClosedError", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "createProtoRpcPeer", "requested", "schema", "getService", "exposed", "handlers", "invariant", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "asyncTimeout", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "failUndefined", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "maxRequests", "on", "message", "warn", "pipe", "stream", "destroy", "delete", "logInfo", "synchronized"]
 }

package/dist/lib/node/meta.json

@@ -1 +1 @@
-{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":33894,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}]},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytes":520,"imports":[{"path":"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts","kind":"import-statement","original":"./replicator-extension"}]}},"outputs":{"packages/core/mesh/teleport-extension-replicator/dist/lib/node/index.cjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":14791},"packages/core/mesh/teleport-extension-replicator/dist/lib/node/index.cjs":{"imports":[{"path":"@dxos/async","kind":"require-call","external":true},{"path":"@dxos/context","kind":"require-call","external":true},{"path":"@dxos/debug","kind":"require-call","external":true},{"path":"@dxos/invariant","kind":"require-call","external":true},{"path":"@dxos/keys","kind":"require-call","external":true},{"path":"@dxos/log","kind":"require-call","external":true},{"path":"@dxos/protocols","kind":"require-call","external":true},{"path":"@dxos/rpc","kind":"require-call","external":true},{"path":"@dxos/util","kind":"require-call","external":true}],"exports":[],"entryPoint":"packages/core/mesh/teleport-extension-replicator/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytesInOutput":143},"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytesInOutput":11394}},"bytes":12631}}}
+{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":35023,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytes":520,"imports":[{"path":"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts","kind":"import-statement","original":"./replicator-extension"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-replicator/dist/lib/node/index.cjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":15161},"packages/core/mesh/teleport-extension-replicator/dist/lib/node/index.cjs":{"imports":[{"path":"@dxos/async","kind":"require-call","external":true},{"path":"@dxos/context","kind":"require-call","external":true},{"path":"@dxos/debug","kind":"require-call","external":true},{"path":"@dxos/invariant","kind":"require-call","external":true},{"path":"@dxos/keys","kind":"require-call","external":true},{"path":"@dxos/log","kind":"require-call","external":true},{"path":"@dxos/protocols","kind":"require-call","external":true},{"path":"@dxos/rpc","kind":"require-call","external":true},{"path":"@dxos/util","kind":"require-call","external":true}],"exports":[],"entryPoint":"packages/core/mesh/teleport-extension-replicator/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytesInOutput":143},"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytesInOutput":11108}},"bytes":12345}}}

package/dist/types/src/replicator-extension.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replicator-extension.d.ts","sourceRoot":"","sources":["../../../src/replicator-extension.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO/C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGrE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,qBAAa,mBAAoB,YAAW,iBAAiB;IAC3D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAKlB;IAEH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+D;IACtF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2D;IAEpF,OAAO,CAAC,IAAI,CAAC,CAA8B;IAC3C,OAAO,CAAC,iBAAiB,CAAC,CAAmB;IAE7C,OAAO,CAAC,QAAQ,CAEd;IAEF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAmBzB;IAGH,OAAO,KAAK,aAAa,GAOxB;IAED,UAAU,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI;IAS7C,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC;IAQxB,MAAM,CAAC,OAAO,EAAE,gBAAgB;IA6ChC,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS;YAUvB,gBAAgB;IA2B9B;;OAEG;YACW,oBAAoB;IAYlC;;;OAGG;YAEW,kBAAkB;YAYlB,cAAc;YA6Cd,gBAAgB;CAS/B"}
+{"version":3,"file":"replicator-extension.d.ts","sourceRoot":"","sources":["../../../src/replicator-extension.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO/C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGrE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,qBAAa,mBAAoB,YAAW,iBAAiB;IAC3D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAKlB;IAEH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+D;IACtF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2D;IAEpF,OAAO,CAAC,IAAI,CAAC,CAA8B;IAC3C,OAAO,CAAC,iBAAiB,CAAC,CAAmB;IAE7C,OAAO,CAAC,QAAQ,CAEd;IAEF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAmBzB;IAGH,OAAO,KAAK,aAAa,GAOxB;IAED,UAAU,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI;IAS7C,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC;IAQxB,MAAM,CAAC,OAAO,EAAE,gBAAgB;IA6ChC,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS;YAUvB,gBAAgB;IA2B9B;;OAEG;YACW,oBAAoB;IAYlC;;;OAGG;YAEW,kBAAkB;YAYlB,cAAc;YAwDd,gBAAgB;CAS/B"}

package/dist/types/src/testing.d.ts

@@ -1,4 +1,4 @@
-import { FeedStore } from '@dxos/feed-store';
+import { FeedOptions, FeedStore } from '@dxos/feed-store';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { Teleport } from '@dxos/teleport';
@@ -11,7 +11,7 @@ export declare class TestAgent {
     keyring: Keyring;
     feedStore: FeedStore<{}>;
     createWriteFeed(numBlocks?: number): Promise<import("@dxos/feed-store").FeedWrapper<{}>>;
-    createReadFeed(key: PublicKey): Promise<import("@dxos/feed-store").FeedWrapper<{}>>;
+    createReadFeed(key: PublicKey, opts?: FeedOptions): Promise<import("@dxos/feed-store").FeedWrapper<{}>>;
 }
 /**
  * Simulates two peers connected via P2P network.

package/dist/types/src/testing.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../../src/testing.ts"],"names":[],"mappings":"AAMA,OAAO,EAAe,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAI1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,qBAAa,WAAW;IACtB,WAAW,IAAI,SAAS;CAGzB;AAGD,qBAAa,SAAS;IACb,OAAO,gDAA4C;IACnD,OAAO,UAAwD;IAC/D,SAAS,gBAEb;IAEG,eAAe,CAAC,SAAS,SAAI;IAUnC,cAAc,CAAC,GAAG,EAAE,SAAS;CAG9B;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;EAuB5B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB;;;EAUhC,CAAC"}
+{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../../src/testing.ts"],"names":[],"mappings":"AAMA,OAAO,EAAe,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAI1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,qBAAa,WAAW;IACtB,WAAW,IAAI,SAAS;CAGzB;AAGD,qBAAa,SAAS;IACb,OAAO,gDAA4C;IACnD,OAAO,UAAwD;IAC/D,SAAS,gBAEb;IAEG,eAAe,CAAC,SAAS,SAAI;IAUnC,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,WAAW;CAGlD;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;EAuB5B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB;;;EAUhC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/teleport-extension-replicator",
-  "version": "0.1.56-main.c19b7cd",
+  "version": "0.1.56-main.c277f68",
   "description": "Replicator extension for teleport.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -16,25 +16,25 @@
     "src"
   ],
   "dependencies": {
-    "@dxos/async": "0.1.56-main.c19b7cd",
-    "@dxos/context": "0.1.56-main.c19b7cd",
-    "@dxos/debug": "0.1.56-main.c19b7cd",
-    "@dxos/feed-store": "0.1.56-main.c19b7cd",
-    "@dxos/invariant": "0.1.56-main.c19b7cd",
-    "@dxos/keys": "0.1.56-main.c19b7cd",
-    "@dxos/log": "0.1.56-main.c19b7cd",
-    "@dxos/node-std": "0.1.56-main.c19b7cd",
-    "@dxos/protocols": "0.1.56-main.c19b7cd",
-    "@dxos/rpc": "0.1.56-main.c19b7cd",
-    "@dxos/teleport": "0.1.56-main.c19b7cd",
-    "@dxos/util": "0.1.56-main.c19b7cd"
+    "@dxos/async": "0.1.56-main.c277f68",
+    "@dxos/context": "0.1.56-main.c277f68",
+    "@dxos/debug": "0.1.56-main.c277f68",
+    "@dxos/feed-store": "0.1.56-main.c277f68",
+    "@dxos/invariant": "0.1.56-main.c277f68",
+    "@dxos/keys": "0.1.56-main.c277f68",
+    "@dxos/log": "0.1.56-main.c277f68",
+    "@dxos/node-std": "0.1.56-main.c277f68",
+    "@dxos/protocols": "0.1.56-main.c277f68",
+    "@dxos/rpc": "0.1.56-main.c277f68",
+    "@dxos/teleport": "0.1.56-main.c277f68",
+    "@dxos/util": "0.1.56-main.c277f68"
   },
   "devDependencies": {
     "add": "^2.0.6",
     "fast-check": "~3.3.0",
     "typescript": "^5.0.4",
-    "@dxos/keyring": "0.1.56-main.c19b7cd",
-    "@dxos/random-access-storage": "0.1.56-main.c19b7cd"
+    "@dxos/keyring": "0.1.56-main.c277f68",
+    "@dxos/random-access-storage": "0.1.56-main.c277f68"
   },
   "publishConfig": {
     "access": "public"

package/src/replicator-extension.test.ts

@@ -6,6 +6,7 @@ import expect from 'expect';
 
 import { Event, sleep } from '@dxos/async';
 import { describe, test } from '@dxos/test';
+import { range } from '@dxos/util';
 
 import { createReplicatorPair, TestBuilder } from './testing';
 
@@ -103,6 +104,33 @@ describe('ReplicatorExtension', () => {
     await Event.wrap(feed1B, 'download').waitForCondition(() => feed1B.length === 10);
   });
 
+  test('sparse replication', async () => {
+    const builder = new TestBuilder();
+    const agent1 = builder.createAgent();
+    const agent2 = builder.createAgent();
+
+    const { replicator1, replicator2 } = await createReplicatorPair();
+
+    replicator1.setOptions({ upload: true });
+    replicator2.setOptions({ upload: true });
+
+    const feed1 = await agent1.createWriteFeed(10);
+    const feed2 = await agent2.createReadFeed(feed1.key, { sparse: true });
+
+    replicator1.addFeed(feed1);
+    replicator2.addFeed(feed2);
+
+    void feed2.download({ start: 10, linear: true });
+
+    await sleep(50);
+
+    expect(range(10).every((idx) => feed2.has(idx))).toEqual(false);
+
+    void feed2.download({ start: 0, linear: true });
+
+    await Event.wrap(feed2, 'download').waitForCondition(() => range(10).every((idx) => feed2.has(idx)));
+  });
+
   // TODO: not working yet.
   // eslint-disable-next-line mocha/no-skipped-tests
   test.skip('enabling upload mid replication', async () => {

package/src/replicator-extension.ts

@@ -213,21 +213,32 @@ export class ReplicatorExtension implements TeleportExtension {
     const networkStream = await this._extensionContext!.createStream(streamTag, {
       contentType: 'application/x-hypercore',
     });
+
+    // https://github.com/holepunchto/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options
     const replicationStream = feed.replicate(true, {
       live: true,
       upload: info.upload,
       download: info.download,
       noise: false,
       encrypted: false,
+      maxRequests: 1024,
     });
 
-    // left for testing
-    // feed.on('download', (index: number, data: any) => {
-    //   log('download', {
-    //     key: info.feedKey,
-    //     index
+    // Left for testing.
+    // const debug = true;
+    // if (debug) {
+    //   feed.on('sync', () => {
+    //     log.info('sync', { key: feed.key, length: feed.length });
+    //   });
+    //   feed.on('download', (index: number, data: any) => {
+    //     log.info('download', {
+    //       key: feed.key,
+    //       index,
+    //       length: feed.length,
+    //       data: data.length,
+    //     });
     //   });
-    // });
+    // }
 
     replicationStream.on('error', (err) => {
       if (

package/src/stress.test.ts

@@ -209,8 +209,8 @@ describe('stress-tests', () => {
 
       await fc.assert(
         fc.asyncProperty(fc.commands(allCommands, { maxCommands: 100 }), async (cmds) => {
-          console.log('\n=====');
-          console.log([...cmds].map((c) => c.toString()).join('\n'));
+          // console.log('\n=====');
+          // console.log([...cmds].map((c) => c.toString()).join('\n'));
 
           const system = await factory(keyring)();
           try {

package/src/testing.ts

@@ -4,7 +4,7 @@
 
 import { pipeline } from 'stream';
 
-import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { FeedFactory, FeedOptions, FeedStore } from '@dxos/feed-store';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -39,8 +39,8 @@ export class TestAgent {
     return feed;
   }
 
-  createReadFeed(key: PublicKey) {
-    return this.feedStore.openFeed(key);
+  createReadFeed(key: PublicKey, opts?: FeedOptions) {
+    return this.feedStore.openFeed(key, opts);
   }
 }