@dxos/teleport-extension-replicator 0.1.56-main.e47dfd1 → 0.1.56-main.e79d64a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/index.mjs +75 -23
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/node/index.cjs +75 -33
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/types/src/replicator-extension.d.ts.map +1 -1
- package/dist/types/src/testing.d.ts +2 -2
- package/dist/types/src/testing.d.ts.map +1 -1
- package/package.json +15 -15
- package/src/replicator-extension.test.ts +28 -0
- package/src/replicator-extension.ts +18 -7
- package/src/stress.test.ts +2 -2
- package/src/testing.ts +3 -3
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import "@dxos/node-std/globals";
|
|
2
2
|
|
|
3
3
|
// packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts
|
|
4
|
-
import invariant from "tiny-invariant";
|
|
5
4
|
import { asyncTimeout, DeferredTask, synchronized } from "@dxos/async";
|
|
6
5
|
import { Context } from "@dxos/context";
|
|
7
6
|
import { failUndefined } from "@dxos/debug";
|
|
7
|
+
import { invariant } from "@dxos/invariant";
|
|
8
8
|
import { PublicKey } from "@dxos/keys";
|
|
9
9
|
import { log, logInfo } from "@dxos/log";
|
|
10
10
|
import { schema, RpcClosedError } from "@dxos/protocols";
|
|
@@ -25,14 +25,13 @@ var ReplicatorExtension = class {
|
|
|
25
25
|
constructor() {
|
|
26
26
|
this._ctx = new Context({
|
|
27
27
|
onError: (err) => {
|
|
28
|
-
var _a;
|
|
29
28
|
log.catch(err, void 0, {
|
|
30
29
|
F: __dxlog_file,
|
|
31
30
|
L: 31,
|
|
32
31
|
S: this,
|
|
33
32
|
C: (f, a) => f(...a)
|
|
34
33
|
});
|
|
35
|
-
|
|
34
|
+
this._extensionContext?.close(err);
|
|
36
35
|
}
|
|
37
36
|
});
|
|
38
37
|
this._feeds = new ComplexMap(PublicKey.hash);
|
|
@@ -62,11 +61,10 @@ var ReplicatorExtension = class {
|
|
|
62
61
|
});
|
|
63
62
|
}
|
|
64
63
|
get extensionInfo() {
|
|
65
|
-
var _a, _b, _c;
|
|
66
64
|
return {
|
|
67
|
-
initiator:
|
|
68
|
-
localPeerId:
|
|
69
|
-
remotePeerId:
|
|
65
|
+
initiator: this._extensionContext?.initiator,
|
|
66
|
+
localPeerId: this._extensionContext?.localPeerId,
|
|
67
|
+
remotePeerId: this._extensionContext?.remotePeerId,
|
|
70
68
|
feeds: Array.from(this._feeds.keys())
|
|
71
69
|
};
|
|
72
70
|
}
|
|
@@ -125,7 +123,15 @@ var ReplicatorExtension = class {
|
|
|
125
123
|
S: this,
|
|
126
124
|
C: (f, a) => f(...a)
|
|
127
125
|
});
|
|
128
|
-
invariant(this._extensionContext.initiator === true, "Invalid call"
|
|
126
|
+
invariant(this._extensionContext.initiator === true, "Invalid call", {
|
|
127
|
+
F: __dxlog_file,
|
|
128
|
+
L: 109,
|
|
129
|
+
S: this,
|
|
130
|
+
A: [
|
|
131
|
+
"this._extensionContext!.initiator === true",
|
|
132
|
+
"'Invalid call'"
|
|
133
|
+
]
|
|
134
|
+
});
|
|
129
135
|
this._updateTask.schedule();
|
|
130
136
|
},
|
|
131
137
|
startReplication: async ({ info }) => {
|
|
@@ -137,7 +143,15 @@ var ReplicatorExtension = class {
|
|
|
137
143
|
S: this,
|
|
138
144
|
C: (f, a) => f(...a)
|
|
139
145
|
});
|
|
140
|
-
invariant(this._extensionContext.initiator === false, "Invalid call"
|
|
146
|
+
invariant(this._extensionContext.initiator === false, "Invalid call", {
|
|
147
|
+
F: __dxlog_file,
|
|
148
|
+
L: 114,
|
|
149
|
+
S: this,
|
|
150
|
+
A: [
|
|
151
|
+
"this._extensionContext!.initiator === false",
|
|
152
|
+
"'Invalid call'"
|
|
153
|
+
]
|
|
154
|
+
});
|
|
141
155
|
const streamTag = await this._acceptReplication(info);
|
|
142
156
|
return {
|
|
143
157
|
streamTag
|
|
@@ -152,7 +166,15 @@ var ReplicatorExtension = class {
|
|
|
152
166
|
S: this,
|
|
153
167
|
C: (f, a) => f(...a)
|
|
154
168
|
});
|
|
155
|
-
invariant(this._extensionContext.initiator === false, "Invalid call"
|
|
169
|
+
invariant(this._extensionContext.initiator === false, "Invalid call", {
|
|
170
|
+
F: __dxlog_file,
|
|
171
|
+
L: 124,
|
|
172
|
+
S: this,
|
|
173
|
+
A: [
|
|
174
|
+
"this._extensionContext!.initiator === false",
|
|
175
|
+
"'Invalid call'"
|
|
176
|
+
]
|
|
177
|
+
});
|
|
156
178
|
await this._stopReplication(info.feedKey);
|
|
157
179
|
}
|
|
158
180
|
}
|
|
@@ -165,7 +187,6 @@ var ReplicatorExtension = class {
|
|
|
165
187
|
this._updateTask.schedule();
|
|
166
188
|
}
|
|
167
189
|
async onClose(err) {
|
|
168
|
-
var _a;
|
|
169
190
|
log("close", {
|
|
170
191
|
err
|
|
171
192
|
}, {
|
|
@@ -175,13 +196,12 @@ var ReplicatorExtension = class {
|
|
|
175
196
|
C: (f, a) => f(...a)
|
|
176
197
|
});
|
|
177
198
|
await this._ctx.dispose();
|
|
178
|
-
await
|
|
199
|
+
await this._rpc?.close();
|
|
179
200
|
for (const feedKey of this._streams.keys()) {
|
|
180
201
|
await this._stopReplication(feedKey);
|
|
181
202
|
}
|
|
182
203
|
}
|
|
183
204
|
async _reevaluateFeeds() {
|
|
184
|
-
var _a;
|
|
185
205
|
log("_reevaluateFeeds", void 0, {
|
|
186
206
|
F: __dxlog_file,
|
|
187
207
|
L: 150,
|
|
@@ -192,7 +212,7 @@ var ReplicatorExtension = class {
|
|
|
192
212
|
if (this._ctx.disposed) {
|
|
193
213
|
return;
|
|
194
214
|
}
|
|
195
|
-
if (this._streams.has(feedKey) && this._options.upload !==
|
|
215
|
+
if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {
|
|
196
216
|
try {
|
|
197
217
|
await asyncTimeout(this._stopReplication(feedKey), 1e3);
|
|
198
218
|
} catch (err) {
|
|
@@ -228,8 +248,24 @@ var ReplicatorExtension = class {
|
|
|
228
248
|
S: this,
|
|
229
249
|
C: (f, a) => f(...a)
|
|
230
250
|
});
|
|
231
|
-
invariant(this._extensionContext.initiator === true, "Invalid call"
|
|
232
|
-
|
|
251
|
+
invariant(this._extensionContext.initiator === true, "Invalid call", {
|
|
252
|
+
F: __dxlog_file,
|
|
253
|
+
L: 181,
|
|
254
|
+
S: this,
|
|
255
|
+
A: [
|
|
256
|
+
"this._extensionContext!.initiator === true",
|
|
257
|
+
"'Invalid call'"
|
|
258
|
+
]
|
|
259
|
+
});
|
|
260
|
+
invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`, {
|
|
261
|
+
F: __dxlog_file,
|
|
262
|
+
L: 182,
|
|
263
|
+
S: this,
|
|
264
|
+
A: [
|
|
265
|
+
"!this._streams.has(feedInfo.feedKey)",
|
|
266
|
+
"`Replication already in progress for feed: ${feedInfo.feedKey}`"
|
|
267
|
+
]
|
|
268
|
+
});
|
|
233
269
|
const { streamTag } = await this._rpc.rpc.ReplicatorService.startReplication({
|
|
234
270
|
info: feedInfo
|
|
235
271
|
});
|
|
@@ -243,7 +279,15 @@ var ReplicatorExtension = class {
|
|
|
243
279
|
* @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.
|
|
244
280
|
*/
|
|
245
281
|
async _acceptReplication(feedInfo) {
|
|
246
|
-
invariant(this._extensionContext.initiator === false, "Invalid call"
|
|
282
|
+
invariant(this._extensionContext.initiator === false, "Invalid call", {
|
|
283
|
+
F: __dxlog_file,
|
|
284
|
+
L: 197,
|
|
285
|
+
S: this,
|
|
286
|
+
A: [
|
|
287
|
+
"this._extensionContext!.initiator === false",
|
|
288
|
+
"'Invalid call'"
|
|
289
|
+
]
|
|
290
|
+
});
|
|
247
291
|
if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {
|
|
248
292
|
return void 0;
|
|
249
293
|
}
|
|
@@ -252,7 +296,6 @@ var ReplicatorExtension = class {
|
|
|
252
296
|
return tag;
|
|
253
297
|
}
|
|
254
298
|
async _replicateFeed(info, streamTag) {
|
|
255
|
-
var _a;
|
|
256
299
|
log("replicate", {
|
|
257
300
|
info,
|
|
258
301
|
streamTag
|
|
@@ -262,8 +305,16 @@ var ReplicatorExtension = class {
|
|
|
262
305
|
S: this,
|
|
263
306
|
C: (f, a) => f(...a)
|
|
264
307
|
});
|
|
265
|
-
invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}
|
|
266
|
-
|
|
308
|
+
invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`, {
|
|
309
|
+
F: __dxlog_file,
|
|
310
|
+
L: 210,
|
|
311
|
+
S: this,
|
|
312
|
+
A: [
|
|
313
|
+
"!this._streams.has(info.feedKey)",
|
|
314
|
+
"`Replication already in progress for feed: ${info.feedKey}`"
|
|
315
|
+
]
|
|
316
|
+
});
|
|
317
|
+
const feed = this._feeds.get(info.feedKey) ?? failUndefined();
|
|
267
318
|
const networkStream = await this._extensionContext.createStream(streamTag, {
|
|
268
319
|
contentType: "application/x-hypercore"
|
|
269
320
|
});
|
|
@@ -272,10 +323,11 @@ var ReplicatorExtension = class {
|
|
|
272
323
|
upload: info.upload,
|
|
273
324
|
download: info.download,
|
|
274
325
|
noise: false,
|
|
275
|
-
encrypted: false
|
|
326
|
+
encrypted: false,
|
|
327
|
+
maxRequests: 1024
|
|
276
328
|
});
|
|
277
329
|
replicationStream.on("error", (err) => {
|
|
278
|
-
if (
|
|
330
|
+
if (err?.message === "Writable stream closed prematurely" || err?.message === "Cannot call write after a stream was destroyed") {
|
|
279
331
|
return;
|
|
280
332
|
}
|
|
281
333
|
log.warn("replication stream error", {
|
|
@@ -283,7 +335,7 @@ var ReplicatorExtension = class {
|
|
|
283
335
|
info
|
|
284
336
|
}, {
|
|
285
337
|
F: __dxlog_file,
|
|
286
|
-
L:
|
|
338
|
+
L: 251,
|
|
287
339
|
S: this,
|
|
288
340
|
C: (f, a) => f(...a)
|
|
289
341
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/replicator-extension.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\
|
|
5
|
-
"mappings": ";;;
|
|
6
|
-
"names": ["
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\n\nimport { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { failUndefined } from '@dxos/debug';\nimport { FeedWrapper } from '@dxos/feed-store';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log, logInfo } from '@dxos/log';\nimport { schema, RpcClosedError } from '@dxos/protocols';\nimport { FeedInfo, ReplicatorService } from '@dxos/protocols/proto/dxos/mesh/teleport/replicator';\nimport { createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';\nimport { ExtensionContext, TeleportExtension } from '@dxos/teleport';\nimport { ComplexMap } from '@dxos/util';\n\nexport type ReplicationOptions = {\n upload: boolean;\n};\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class ReplicatorExtension implements TeleportExtension {\n private readonly _ctx = new Context({\n onError: (err) => {\n log.catch(err);\n this._extensionContext?.close(err);\n },\n });\n\n private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);\n private readonly _streams = new ComplexMap<PublicKey, ActiveStream>(PublicKey.hash);\n\n private _rpc?: ProtoRpcPeer<ServiceBundle>;\n private _extensionContext?: ExtensionContext;\n\n private _options: ReplicationOptions = {\n upload: false,\n };\n\n private readonly _updateTask = new DeferredTask(this._ctx, async () => {\n try {\n if (this._extensionContext!.initiator === false) {\n await this._rpc!.rpc.ReplicatorService.updateFeeds({\n feeds: Array.from(this._feeds.values()).map((feed) => ({\n feedKey: feed.key,\n download: true,\n upload: this._options.upload,\n })),\n });\n } else if (this._extensionContext!.initiator === true) {\n await this._reevaluateFeeds();\n }\n } catch (err) {\n if (err instanceof RpcClosedError) {\n return; // Some RPC requests might be pending while closing.\n }\n throw err;\n }\n });\n\n @logInfo\n private get extensionInfo() {\n return {\n initiator: this._extensionContext?.initiator,\n localPeerId: this._extensionContext?.localPeerId,\n remotePeerId: this._extensionContext?.remotePeerId,\n feeds: Array.from(this._feeds.keys()),\n };\n }\n\n setOptions(options: ReplicationOptions): this {\n this._options = options;\n log('setOptions', { options });\n if (this._extensionContext) {\n this._updateTask.schedule();\n }\n return this;\n }\n\n addFeed(feed: FeedWrapper<any>) {\n this._feeds.set(feed.key, feed);\n log('addFeed', { feedKey: feed.key });\n if (this._extensionContext) {\n this._updateTask.schedule();\n }\n }\n\n async onOpen(context: ExtensionContext) {\n this._extensionContext = context;\n log('open');\n\n this._rpc = createProtoRpcPeer<ServiceBundle, ServiceBundle>({\n requested: {\n ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n },\n exposed: {\n ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n },\n handlers: {\n ReplicatorService: {\n updateFeeds: async ({ feeds }) => {\n log('received feed info', { feeds });\n invariant(this._extensionContext!.initiator === true, 'Invalid call');\n this._updateTask.schedule();\n },\n startReplication: async ({ info }) => {\n log('starting replication...', { info });\n invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n const streamTag = await this._acceptReplication(info);\n return {\n streamTag,\n };\n },\n stopReplication: async ({ info }) => {\n log('stopping replication...', { info });\n // TODO(dmaretskyi): Make sure any peer can stop replication.\n invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n await this._stopReplication(info.feedKey);\n },\n },\n },\n port: await context.createPort('rpc', {\n contentType: 'application/x-protobuf; messageType=\"dxos.rpc.Message\"',\n }),\n });\n await this._rpc.open();\n\n this._updateTask.schedule();\n }\n\n async onClose(err?: Error | undefined) {\n log('close', { err });\n await this._ctx.dispose();\n await this._rpc?.close();\n for (const feedKey of this._streams.keys()) {\n await this._stopReplication(feedKey);\n }\n }\n\n @synchronized\n private async _reevaluateFeeds() {\n log('_reevaluateFeeds');\n for (const feedKey of this._feeds.keys()) {\n if (this._ctx.disposed) {\n return;\n }\n if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {\n try {\n await asyncTimeout(this._stopReplication(feedKey), 1000);\n } catch (err) {\n log.catch(err);\n }\n }\n\n if (this._ctx.disposed) {\n return;\n }\n if (!this._streams.has(feedKey)) {\n await this._initiateReplication({\n feedKey,\n download: true,\n upload: this._options.upload,\n });\n }\n }\n }\n\n /**\n * Try to initiate feed replication.\n */\n private async _initiateReplication(feedInfo: FeedInfo) {\n log('initiating replication', { feedInfo });\n invariant(this._extensionContext!.initiator === true, 'Invalid call');\n invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`);\n const { streamTag } = await this._rpc!.rpc.ReplicatorService.startReplication({ info: feedInfo });\n if (!streamTag) {\n return;\n }\n\n await this._replicateFeed(feedInfo, streamTag);\n }\n\n /**\n * Respond to a remote request to replicate a feed.\n * @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.\n */\n @synchronized\n private async _acceptReplication(feedInfo: FeedInfo): Promise<string | undefined> {\n invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {\n return undefined; // We don't have the feed or we are already replicating it.\n }\n\n const tag = `feed-${feedInfo.feedKey.toHex()}-${PublicKey.random().toHex().slice(0, 8)}`; // Generate a unique tag for the stream.\n await this._replicateFeed(feedInfo, tag);\n return tag;\n }\n\n private async _replicateFeed(info: FeedInfo, streamTag: string) {\n log('replicate', { info, streamTag });\n invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`);\n\n const feed = this._feeds.get(info.feedKey) ?? failUndefined();\n const networkStream = await this._extensionContext!.createStream(streamTag, {\n contentType: 'application/x-hypercore',\n });\n\n // https://github.com/holepunchto/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options\n const replicationStream = feed.replicate(true, {\n live: true,\n upload: info.upload,\n download: info.download,\n noise: false,\n encrypted: false,\n maxRequests: 1024,\n });\n\n // Left for testing.\n // const debug = true;\n // if (debug) {\n // feed.on('sync', () => {\n // log.info('sync', { key: feed.key, length: feed.length });\n // });\n // feed.on('download', (index: number, data: any) => {\n // log.info('download', {\n // key: feed.key,\n // index,\n // length: feed.length,\n // data: data.length,\n // });\n // });\n // }\n\n replicationStream.on('error', (err) => {\n if (\n err?.message === 'Writable stream closed prematurely' ||\n err?.message === 'Cannot call write after a stream was destroyed'\n ) {\n return;\n }\n\n log.warn('replication stream error', { err, info });\n });\n\n this._streams.set(info.feedKey, {\n streamTag,\n networkStream,\n replicationStream,\n info,\n });\n\n networkStream.pipe(replicationStream as any).pipe(networkStream);\n }\n\n private async _stopReplication(feedKey: PublicKey) {\n const stream = this._streams.get(feedKey);\n if (!stream) {\n return;\n }\n\n stream.networkStream.destroy();\n this._streams.delete(feedKey);\n }\n}\n\ntype ServiceBundle = {\n ReplicatorService: ReplicatorService;\n};\n\ntype ActiveStream = {\n streamTag: string;\n networkStream: Duplex;\n replicationStream: ProtocolStream;\n info: FeedInfo;\n};\n"],
|
|
5
|
+
"mappings": ";;;AAOA,SAASA,cAAcC,cAAcC,oBAAoB;AACzD,SAASC,eAAe;AACxB,SAASC,qBAAqB;AAE9B,SAASC,iBAAiB;AAC1B,SAASC,iBAAiB;AAC1B,SAASC,KAAKC,eAAe;AAC7B,SAASC,QAAQC,sBAAsB;AAEvC,SAASC,0BAAwC;AAEjD,SAASC,kBAAkB;;;;;;;;;;;;AASpB,IAAMC,sBAAN,MAAMA;EAAN;AACYC,gBAAO,IAAIX,QAAQ;MAClCY,SAAS,CAACC,QAAAA;AACRT,YAAIU,MAAMD,KAAAA,QAAAA;;;;;;AACV,aAAKE,mBAAmBC,MAAMH,GAAAA;MAChC;IACF,CAAA;AAEiBI,kBAAS,IAAIR,WAAwCN,UAAUe,IAAI;AACnEC,oBAAW,IAAIV,WAAoCN,UAAUe,IAAI;AAK1EE,oBAA+B;MACrCC,QAAQ;IACV;AAEiBC,uBAAc,IAAIxB,aAAa,KAAKa,MAAM,YAAA;AACzD,UAAI;AACF,YAAI,KAAKI,kBAAmBQ,cAAc,OAAO;AAC/C,gBAAM,KAAKC,KAAMC,IAAIC,kBAAkBC,YAAY;YACjDC,OAAOC,MAAMC,KAAK,KAAKb,OAAOc,OAAM,CAAA,EAAIC,IAAI,CAACC,UAAU;cACrDC,SAASD,KAAKE;cACdC,UAAU;cACVf,QAAQ,KAAKD,SAASC;YACxB,EAAA;UACF,CAAA;QACF,WAAW,KAAKN,kBAAmBQ,cAAc,MAAM;AACrD,gBAAM,KAAKc,iBAAgB;QAC7B;MACF,SAASxB,KAAK;AACZ,YAAIA,eAAeN,gBAAgB;AACjC;QACF;AACA,cAAMM;MACR;IACF,CAAA;;EAEA,IACYyB,gBAAgB;AAC1B,WAAO;MACLf,WAAW,KAAKR,mBAAmBQ;MACnCgB,aAAa,KAAKxB,mBAAmBwB;MACrCC,cAAc,KAAKzB,mBAAmByB;MACtCZ,OAAOC,MAAMC,KAAK,KAAKb,OAAOwB,KAAI,CAAA;IACpC;EACF;EAEAC,WAAWC,SAAmC;AAC5C,SAAKvB,WAAWuB;AAChBvC,QAAI,cAAc;MAAEuC;IAAQ,GAAA;;;;;;AAC5B,QAAI,KAAK5B,mBAAmB;AAC1B,WAAKO,YAAYsB,SAAQ;IAC3B;AACA,WAAO;EACT;EAEAC,QAAQZ,MAAwB;AAC9B,SAAKhB,OAAO6B,IAAIb,KAAKE,KAAKF,IAAAA;AAC1B7B,QAAI,WAAW;MAAE8B,SAASD,KAAKE;IAAI,GAAA;;;;;;AACnC,QAAI,KAAKpB,mBAAmB;AAC1B,WAAKO,YAAYsB,SAAQ;IAC3B;EACF;EAEA,MAAMG,OAAOC,SAA2B;AACtC,SAAKjC,oBAAoBiC;AACzB5C,QAAI,QAAA,QAAA;;;;;;AAEJ,SAAKoB,OAAOhB,mBAAiD;MAC3DyC,WAAW;QACTvB,mBAAmBpB,OAAO4C,WAAW,iDAAA;MACvC;MACAC,SAAS;QACPzB,mBAAmBpB,OAAO4C,WAAW,iDAAA;MACvC;MACAE,UAAU;QACR1B,mBAAmB;UACjBC,aAAa,OAAO,EAAEC,MAAK,MAAE;AAC3BxB,gBAAI,sBAAsB;cAAEwB;YAAM,GAAA;;;;;;AAClC1B,sBAAU,KAAKa,kBAAmBQ,cAAc,MAAM,gBAAA;;;;;;;;;AACtD,iBAAKD,YAAYsB,SAAQ;UAC3B;UACAS,kBAAkB,OAAO,EAAEC,KAAI,MAAE;AAC/BlD,gBAAI,2BAA2B;cAAEkD;YAAK,GAAA;;;;;;AACtCpD,sBAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAMgC,YAAY,MAAM,KAAKC,mBAAmBF,IAAAA;AAChD,mBAAO;cACLC;YACF;UACF;UACAE,iBAAiB,OAAO,EAAEH,KAAI,MAAE;AAC9BlD,gBAAI,2BAA2B;cAAEkD;YAAK,GAAA;;;;;;AAEtCpD,sBAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAM,KAAKmC,iBAAiBJ,KAAKpB,OAAO;UAC1C;QACF;MACF;MACAyB,MAAM,MAAMX,QAAQY,WAAW,OAAO;QACpCC,aAAa;MACf,CAAA;IACF,CAAA;AACA,UAAM,KAAKrC,KAAKsC,KAAI;AAEpB,SAAKxC,YAAYsB,SAAQ;EAC3B;EAEA,MAAMmB,QAAQlD,KAAyB;AACrCT,QAAI,SAAS;MAAES;IAAI,GAAA;;;;;;AACnB,UAAM,KAAKF,KAAKqD,QAAO;AACvB,UAAM,KAAKxC,MAAMR,MAAAA;AACjB,eAAWkB,WAAW,KAAKf,SAASsB,KAAI,GAAI;AAC1C,YAAM,KAAKiB,iBAAiBxB,OAAAA;IAC9B;EACF;EAEA,MACcG,mBAAmB;AAC/BjC,QAAI,oBAAA,QAAA;;;;;;AACJ,eAAW8B,WAAW,KAAKjB,OAAOwB,KAAI,GAAI;AACxC,UAAI,KAAK9B,KAAKsD,UAAU;AACtB;MACF;AACA,UAAI,KAAK9C,SAAS+C,IAAIhC,OAAAA,KAAY,KAAKd,SAASC,WAAW,KAAKF,SAASgD,IAAIjC,OAAAA,GAAUoB,KAAKjC,QAAQ;AAClG,YAAI;AACF,gBAAMxB,aAAa,KAAK6D,iBAAiBxB,OAAAA,GAAU,GAAA;QACrD,SAASrB,KAAK;AACZT,cAAIU,MAAMD,KAAAA,QAAAA;;;;;;QACZ;MACF;AAEA,UAAI,KAAKF,KAAKsD,UAAU;AACtB;MACF;AACA,UAAI,CAAC,KAAK9C,SAAS+C,IAAIhC,OAAAA,GAAU;AAC/B,cAAM,KAAKkC,qBAAqB;UAC9BlC;UACAE,UAAU;UACVf,QAAQ,KAAKD,SAASC;QACxB,CAAA;MACF;IACF;EACF;;;;EAKA,MAAc+C,qBAAqBC,UAAoB;AACrDjE,QAAI,0BAA0B;MAAEiE;IAAS,GAAA;;;;;;AACzCnE,cAAU,KAAKa,kBAAmBQ,cAAc,MAAM,gBAAA;;;;;;;;;AACtDrB,cAAU,CAAC,KAAKiB,SAAS+C,IAAIG,SAASnC,OAAO,GAAG,6CAA6CmC,SAASnC,OAAO,IAAE;;;;;;;;;AAC/G,UAAM,EAAEqB,UAAS,IAAK,MAAM,KAAK/B,KAAMC,IAAIC,kBAAkB2B,iBAAiB;MAAEC,MAAMe;IAAS,CAAA;AAC/F,QAAI,CAACd,WAAW;AACd;IACF;AAEA,UAAM,KAAKe,eAAeD,UAAUd,SAAAA;EACtC;;;;;EAMA,MACcC,mBAAmBa,UAAiD;AAChFnE,cAAU,KAAKa,kBAAmBQ,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,QAAI,CAAC,KAAKN,OAAOiD,IAAIG,SAASnC,OAAO,KAAK,KAAKf,SAAS+C,IAAIG,SAASnC,OAAO,GAAG;AAC7E,aAAOqC;IACT;AAEA,UAAMC,MAAM,QAAQH,SAASnC,QAAQuC,MAAK,CAAA,IAAMtE,UAAUuE,OAAM,EAAGD,MAAK,EAAGE,MAAM,GAAG,CAAA,CAAA;AACpF,UAAM,KAAKL,eAAeD,UAAUG,GAAAA;AACpC,WAAOA;EACT;EAEA,MAAcF,eAAehB,MAAgBC,WAAmB;AAC9DnD,QAAI,aAAa;MAAEkD;MAAMC;IAAU,GAAA;;;;;;AACnCrD,cAAU,CAAC,KAAKiB,SAAS+C,IAAIZ,KAAKpB,OAAO,GAAG,6CAA6CoB,KAAKpB,OAAO,IAAE;;;;;;;;;AAEvG,UAAMD,OAAO,KAAKhB,OAAOkD,IAAIb,KAAKpB,OAAO,KAAKjC,cAAAA;AAC9C,UAAM2E,gBAAgB,MAAM,KAAK7D,kBAAmB8D,aAAatB,WAAW;MAC1EM,aAAa;IACf,CAAA;AAGA,UAAMiB,oBAAoB7C,KAAK8C,UAAU,MAAM;MAC7CC,MAAM;MACN3D,QAAQiC,KAAKjC;MACbe,UAAUkB,KAAKlB;MACf6C,OAAO;MACPC,WAAW;MACXC,aAAa;IACf,CAAA;AAkBAL,sBAAkBM,GAAG,SAAS,CAACvE,QAAAA;AAC7B,UACEA,KAAKwE,YAAY,wCACjBxE,KAAKwE,YAAY,kDACjB;AACA;MACF;AAEAjF,UAAIkF,KAAK,4BAA4B;QAAEzE;QAAKyC;MAAK,GAAA;;;;;;IACnD,CAAA;AAEA,SAAKnC,SAAS2B,IAAIQ,KAAKpB,SAAS;MAC9BqB;MACAqB;MACAE;MACAxB;IACF,CAAA;AAEAsB,kBAAcW,KAAKT,iBAAAA,EAA0BS,KAAKX,aAAAA;EACpD;EAEA,MAAclB,iBAAiBxB,SAAoB;AACjD,UAAMsD,SAAS,KAAKrE,SAASgD,IAAIjC,OAAAA;AACjC,QAAI,CAACsD,QAAQ;AACX;IACF;AAEAA,WAAOZ,cAAca,QAAO;AAC5B,SAAKtE,SAASuE,OAAOxD,OAAAA;EACvB;AACF;;EA9MG7B;GAvCUK,oBAAAA,WAAAA,iBAAAA,IAAAA;;EAwHVX;GAxHUW,oBAAAA,WAAAA,oBAAAA,IAAAA;;EAuKVX;GAvKUW,oBAAAA,WAAAA,sBAAAA,IAAAA;",
|
|
6
|
+
"names": ["asyncTimeout", "DeferredTask", "synchronized", "Context", "failUndefined", "invariant", "PublicKey", "log", "logInfo", "schema", "RpcClosedError", "createProtoRpcPeer", "ComplexMap", "ReplicatorExtension", "_ctx", "onError", "err", "catch", "_extensionContext", "close", "_feeds", "hash", "_streams", "_options", "upload", "_updateTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "requested", "getService", "exposed", "handlers", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "maxRequests", "on", "message", "warn", "pipe", "stream", "destroy", "delete"]
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":
|
|
1
|
+
{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":35023,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytes":520,"imports":[{"path":"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts","kind":"import-statement","original":"./replicator-extension"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-replicator/dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":15060},"packages/core/mesh/teleport-extension-replicator/dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["ReplicatorExtension"],"entryPoint":"packages/core/mesh/teleport-extension-replicator/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytesInOutput":10570},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytesInOutput":0}},"bytes":10753}}}
|
package/dist/lib/node/index.cjs
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __export = (target, all) => {
|
|
9
7
|
for (var name in all)
|
|
@@ -17,14 +15,6 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
17
15
|
}
|
|
18
16
|
return to;
|
|
19
17
|
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
19
|
|
|
30
20
|
// packages/core/mesh/teleport-extension-replicator/src/index.ts
|
|
@@ -35,10 +25,10 @@ __export(src_exports, {
|
|
|
35
25
|
module.exports = __toCommonJS(src_exports);
|
|
36
26
|
|
|
37
27
|
// packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts
|
|
38
|
-
var import_tiny_invariant = __toESM(require("tiny-invariant"));
|
|
39
28
|
var import_async = require("@dxos/async");
|
|
40
29
|
var import_context = require("@dxos/context");
|
|
41
30
|
var import_debug = require("@dxos/debug");
|
|
31
|
+
var import_invariant = require("@dxos/invariant");
|
|
42
32
|
var import_keys = require("@dxos/keys");
|
|
43
33
|
var import_log = require("@dxos/log");
|
|
44
34
|
var import_protocols = require("@dxos/protocols");
|
|
@@ -59,14 +49,13 @@ var ReplicatorExtension = class {
|
|
|
59
49
|
constructor() {
|
|
60
50
|
this._ctx = new import_context.Context({
|
|
61
51
|
onError: (err) => {
|
|
62
|
-
var _a;
|
|
63
52
|
import_log.log.catch(err, void 0, {
|
|
64
53
|
F: __dxlog_file,
|
|
65
54
|
L: 31,
|
|
66
55
|
S: this,
|
|
67
56
|
C: (f, a) => f(...a)
|
|
68
57
|
});
|
|
69
|
-
|
|
58
|
+
this._extensionContext?.close(err);
|
|
70
59
|
}
|
|
71
60
|
});
|
|
72
61
|
this._feeds = new import_util.ComplexMap(import_keys.PublicKey.hash);
|
|
@@ -96,11 +85,10 @@ var ReplicatorExtension = class {
|
|
|
96
85
|
});
|
|
97
86
|
}
|
|
98
87
|
get extensionInfo() {
|
|
99
|
-
var _a, _b, _c;
|
|
100
88
|
return {
|
|
101
|
-
initiator:
|
|
102
|
-
localPeerId:
|
|
103
|
-
remotePeerId:
|
|
89
|
+
initiator: this._extensionContext?.initiator,
|
|
90
|
+
localPeerId: this._extensionContext?.localPeerId,
|
|
91
|
+
remotePeerId: this._extensionContext?.remotePeerId,
|
|
104
92
|
feeds: Array.from(this._feeds.keys())
|
|
105
93
|
};
|
|
106
94
|
}
|
|
@@ -159,7 +147,15 @@ var ReplicatorExtension = class {
|
|
|
159
147
|
S: this,
|
|
160
148
|
C: (f, a) => f(...a)
|
|
161
149
|
});
|
|
162
|
-
(0,
|
|
150
|
+
(0, import_invariant.invariant)(this._extensionContext.initiator === true, "Invalid call", {
|
|
151
|
+
F: __dxlog_file,
|
|
152
|
+
L: 109,
|
|
153
|
+
S: this,
|
|
154
|
+
A: [
|
|
155
|
+
"this._extensionContext!.initiator === true",
|
|
156
|
+
"'Invalid call'"
|
|
157
|
+
]
|
|
158
|
+
});
|
|
163
159
|
this._updateTask.schedule();
|
|
164
160
|
},
|
|
165
161
|
startReplication: async ({ info }) => {
|
|
@@ -171,7 +167,15 @@ var ReplicatorExtension = class {
|
|
|
171
167
|
S: this,
|
|
172
168
|
C: (f, a) => f(...a)
|
|
173
169
|
});
|
|
174
|
-
(0,
|
|
170
|
+
(0, import_invariant.invariant)(this._extensionContext.initiator === false, "Invalid call", {
|
|
171
|
+
F: __dxlog_file,
|
|
172
|
+
L: 114,
|
|
173
|
+
S: this,
|
|
174
|
+
A: [
|
|
175
|
+
"this._extensionContext!.initiator === false",
|
|
176
|
+
"'Invalid call'"
|
|
177
|
+
]
|
|
178
|
+
});
|
|
175
179
|
const streamTag = await this._acceptReplication(info);
|
|
176
180
|
return {
|
|
177
181
|
streamTag
|
|
@@ -186,7 +190,15 @@ var ReplicatorExtension = class {
|
|
|
186
190
|
S: this,
|
|
187
191
|
C: (f, a) => f(...a)
|
|
188
192
|
});
|
|
189
|
-
(0,
|
|
193
|
+
(0, import_invariant.invariant)(this._extensionContext.initiator === false, "Invalid call", {
|
|
194
|
+
F: __dxlog_file,
|
|
195
|
+
L: 124,
|
|
196
|
+
S: this,
|
|
197
|
+
A: [
|
|
198
|
+
"this._extensionContext!.initiator === false",
|
|
199
|
+
"'Invalid call'"
|
|
200
|
+
]
|
|
201
|
+
});
|
|
190
202
|
await this._stopReplication(info.feedKey);
|
|
191
203
|
}
|
|
192
204
|
}
|
|
@@ -199,7 +211,6 @@ var ReplicatorExtension = class {
|
|
|
199
211
|
this._updateTask.schedule();
|
|
200
212
|
}
|
|
201
213
|
async onClose(err) {
|
|
202
|
-
var _a;
|
|
203
214
|
(0, import_log.log)("close", {
|
|
204
215
|
err
|
|
205
216
|
}, {
|
|
@@ -209,13 +220,12 @@ var ReplicatorExtension = class {
|
|
|
209
220
|
C: (f, a) => f(...a)
|
|
210
221
|
});
|
|
211
222
|
await this._ctx.dispose();
|
|
212
|
-
await
|
|
223
|
+
await this._rpc?.close();
|
|
213
224
|
for (const feedKey of this._streams.keys()) {
|
|
214
225
|
await this._stopReplication(feedKey);
|
|
215
226
|
}
|
|
216
227
|
}
|
|
217
228
|
async _reevaluateFeeds() {
|
|
218
|
-
var _a;
|
|
219
229
|
(0, import_log.log)("_reevaluateFeeds", void 0, {
|
|
220
230
|
F: __dxlog_file,
|
|
221
231
|
L: 150,
|
|
@@ -226,7 +236,7 @@ var ReplicatorExtension = class {
|
|
|
226
236
|
if (this._ctx.disposed) {
|
|
227
237
|
return;
|
|
228
238
|
}
|
|
229
|
-
if (this._streams.has(feedKey) && this._options.upload !==
|
|
239
|
+
if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {
|
|
230
240
|
try {
|
|
231
241
|
await (0, import_async.asyncTimeout)(this._stopReplication(feedKey), 1e3);
|
|
232
242
|
} catch (err) {
|
|
@@ -262,8 +272,24 @@ var ReplicatorExtension = class {
|
|
|
262
272
|
S: this,
|
|
263
273
|
C: (f, a) => f(...a)
|
|
264
274
|
});
|
|
265
|
-
(0,
|
|
266
|
-
|
|
275
|
+
(0, import_invariant.invariant)(this._extensionContext.initiator === true, "Invalid call", {
|
|
276
|
+
F: __dxlog_file,
|
|
277
|
+
L: 181,
|
|
278
|
+
S: this,
|
|
279
|
+
A: [
|
|
280
|
+
"this._extensionContext!.initiator === true",
|
|
281
|
+
"'Invalid call'"
|
|
282
|
+
]
|
|
283
|
+
});
|
|
284
|
+
(0, import_invariant.invariant)(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`, {
|
|
285
|
+
F: __dxlog_file,
|
|
286
|
+
L: 182,
|
|
287
|
+
S: this,
|
|
288
|
+
A: [
|
|
289
|
+
"!this._streams.has(feedInfo.feedKey)",
|
|
290
|
+
"`Replication already in progress for feed: ${feedInfo.feedKey}`"
|
|
291
|
+
]
|
|
292
|
+
});
|
|
267
293
|
const { streamTag } = await this._rpc.rpc.ReplicatorService.startReplication({
|
|
268
294
|
info: feedInfo
|
|
269
295
|
});
|
|
@@ -277,7 +303,15 @@ var ReplicatorExtension = class {
|
|
|
277
303
|
* @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.
|
|
278
304
|
*/
|
|
279
305
|
async _acceptReplication(feedInfo) {
|
|
280
|
-
(0,
|
|
306
|
+
(0, import_invariant.invariant)(this._extensionContext.initiator === false, "Invalid call", {
|
|
307
|
+
F: __dxlog_file,
|
|
308
|
+
L: 197,
|
|
309
|
+
S: this,
|
|
310
|
+
A: [
|
|
311
|
+
"this._extensionContext!.initiator === false",
|
|
312
|
+
"'Invalid call'"
|
|
313
|
+
]
|
|
314
|
+
});
|
|
281
315
|
if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {
|
|
282
316
|
return void 0;
|
|
283
317
|
}
|
|
@@ -286,7 +320,6 @@ var ReplicatorExtension = class {
|
|
|
286
320
|
return tag;
|
|
287
321
|
}
|
|
288
322
|
async _replicateFeed(info, streamTag) {
|
|
289
|
-
var _a;
|
|
290
323
|
(0, import_log.log)("replicate", {
|
|
291
324
|
info,
|
|
292
325
|
streamTag
|
|
@@ -296,8 +329,16 @@ var ReplicatorExtension = class {
|
|
|
296
329
|
S: this,
|
|
297
330
|
C: (f, a) => f(...a)
|
|
298
331
|
});
|
|
299
|
-
(0,
|
|
300
|
-
|
|
332
|
+
(0, import_invariant.invariant)(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`, {
|
|
333
|
+
F: __dxlog_file,
|
|
334
|
+
L: 210,
|
|
335
|
+
S: this,
|
|
336
|
+
A: [
|
|
337
|
+
"!this._streams.has(info.feedKey)",
|
|
338
|
+
"`Replication already in progress for feed: ${info.feedKey}`"
|
|
339
|
+
]
|
|
340
|
+
});
|
|
341
|
+
const feed = this._feeds.get(info.feedKey) ?? (0, import_debug.failUndefined)();
|
|
301
342
|
const networkStream = await this._extensionContext.createStream(streamTag, {
|
|
302
343
|
contentType: "application/x-hypercore"
|
|
303
344
|
});
|
|
@@ -306,10 +347,11 @@ var ReplicatorExtension = class {
|
|
|
306
347
|
upload: info.upload,
|
|
307
348
|
download: info.download,
|
|
308
349
|
noise: false,
|
|
309
|
-
encrypted: false
|
|
350
|
+
encrypted: false,
|
|
351
|
+
maxRequests: 1024
|
|
310
352
|
});
|
|
311
353
|
replicationStream.on("error", (err) => {
|
|
312
|
-
if (
|
|
354
|
+
if (err?.message === "Writable stream closed prematurely" || err?.message === "Cannot call write after a stream was destroyed") {
|
|
313
355
|
return;
|
|
314
356
|
}
|
|
315
357
|
import_log.log.warn("replication stream error", {
|
|
@@ -317,7 +359,7 @@ var ReplicatorExtension = class {
|
|
|
317
359
|
info
|
|
318
360
|
}, {
|
|
319
361
|
F: __dxlog_file,
|
|
320
|
-
L:
|
|
362
|
+
L: 251,
|
|
321
363
|
S: this,
|
|
322
364
|
C: (f, a) => f(...a)
|
|
323
365
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/index.ts", "../../../src/replicator-extension.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './replicator-extension';\n", "//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\
|
|
5
|
-
"mappings": "
|
|
6
|
-
"names": ["ReplicatorExtension", "_ctx", "Context", "onError", "err", "log", "catch", "_extensionContext", "close", "_feeds", "ComplexMap", "PublicKey", "hash", "_streams", "_options", "upload", "_updateTask", "DeferredTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "RpcClosedError", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "createProtoRpcPeer", "requested", "schema", "getService", "exposed", "handlers", "invariant", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "asyncTimeout", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "failUndefined", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "on", "message", "warn", "pipe", "stream", "destroy", "delete", "logInfo", "synchronized"]
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './replicator-extension';\n", "//\n// Copyright 2022 DXOS.org\n//\n\nimport type { ProtocolStream } from 'hypercore-protocol';\nimport { Duplex } from 'node:stream';\n\nimport { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { failUndefined } from '@dxos/debug';\nimport { FeedWrapper } from '@dxos/feed-store';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log, logInfo } from '@dxos/log';\nimport { schema, RpcClosedError } from '@dxos/protocols';\nimport { FeedInfo, ReplicatorService } from '@dxos/protocols/proto/dxos/mesh/teleport/replicator';\nimport { createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';\nimport { ExtensionContext, TeleportExtension } from '@dxos/teleport';\nimport { ComplexMap } from '@dxos/util';\n\nexport type ReplicationOptions = {\n upload: boolean;\n};\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class ReplicatorExtension implements TeleportExtension {\n private readonly _ctx = new Context({\n onError: (err) => {\n log.catch(err);\n this._extensionContext?.close(err);\n },\n });\n\n private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);\n private readonly _streams = new ComplexMap<PublicKey, ActiveStream>(PublicKey.hash);\n\n private _rpc?: ProtoRpcPeer<ServiceBundle>;\n private _extensionContext?: ExtensionContext;\n\n private _options: ReplicationOptions = {\n upload: false,\n };\n\n private readonly _updateTask = new DeferredTask(this._ctx, async () => {\n try {\n if (this._extensionContext!.initiator === false) {\n await this._rpc!.rpc.ReplicatorService.updateFeeds({\n feeds: Array.from(this._feeds.values()).map((feed) => ({\n feedKey: feed.key,\n download: true,\n upload: this._options.upload,\n })),\n });\n } else if (this._extensionContext!.initiator === true) {\n await this._reevaluateFeeds();\n }\n } catch (err) {\n if (err instanceof RpcClosedError) {\n return; // Some RPC requests might be pending while closing.\n }\n throw err;\n }\n });\n\n @logInfo\n private get extensionInfo() {\n return {\n initiator: this._extensionContext?.initiator,\n localPeerId: this._extensionContext?.localPeerId,\n remotePeerId: this._extensionContext?.remotePeerId,\n feeds: Array.from(this._feeds.keys()),\n };\n }\n\n setOptions(options: ReplicationOptions): this {\n this._options = options;\n log('setOptions', { options });\n if (this._extensionContext) {\n this._updateTask.schedule();\n }\n return this;\n }\n\n addFeed(feed: FeedWrapper<any>) {\n this._feeds.set(feed.key, feed);\n log('addFeed', { feedKey: feed.key });\n if (this._extensionContext) {\n this._updateTask.schedule();\n }\n }\n\n async onOpen(context: ExtensionContext) {\n this._extensionContext = context;\n log('open');\n\n this._rpc = createProtoRpcPeer<ServiceBundle, ServiceBundle>({\n requested: {\n ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n },\n exposed: {\n ReplicatorService: schema.getService('dxos.mesh.teleport.replicator.ReplicatorService'),\n },\n handlers: {\n ReplicatorService: {\n updateFeeds: async ({ feeds }) => {\n log('received feed info', { feeds });\n invariant(this._extensionContext!.initiator === true, 'Invalid call');\n this._updateTask.schedule();\n },\n startReplication: async ({ info }) => {\n log('starting replication...', { info });\n invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n const streamTag = await this._acceptReplication(info);\n return {\n streamTag,\n };\n },\n stopReplication: async ({ info }) => {\n log('stopping replication...', { info });\n // TODO(dmaretskyi): Make sure any peer can stop replication.\n invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n await this._stopReplication(info.feedKey);\n },\n },\n },\n port: await context.createPort('rpc', {\n contentType: 'application/x-protobuf; messageType=\"dxos.rpc.Message\"',\n }),\n });\n await this._rpc.open();\n\n this._updateTask.schedule();\n }\n\n async onClose(err?: Error | undefined) {\n log('close', { err });\n await this._ctx.dispose();\n await this._rpc?.close();\n for (const feedKey of this._streams.keys()) {\n await this._stopReplication(feedKey);\n }\n }\n\n @synchronized\n private async _reevaluateFeeds() {\n log('_reevaluateFeeds');\n for (const feedKey of this._feeds.keys()) {\n if (this._ctx.disposed) {\n return;\n }\n if (this._streams.has(feedKey) && this._options.upload !== this._streams.get(feedKey)?.info.upload) {\n try {\n await asyncTimeout(this._stopReplication(feedKey), 1000);\n } catch (err) {\n log.catch(err);\n }\n }\n\n if (this._ctx.disposed) {\n return;\n }\n if (!this._streams.has(feedKey)) {\n await this._initiateReplication({\n feedKey,\n download: true,\n upload: this._options.upload,\n });\n }\n }\n }\n\n /**\n * Try to initiate feed replication.\n */\n private async _initiateReplication(feedInfo: FeedInfo) {\n log('initiating replication', { feedInfo });\n invariant(this._extensionContext!.initiator === true, 'Invalid call');\n invariant(!this._streams.has(feedInfo.feedKey), `Replication already in progress for feed: ${feedInfo.feedKey}`);\n const { streamTag } = await this._rpc!.rpc.ReplicatorService.startReplication({ info: feedInfo });\n if (!streamTag) {\n return;\n }\n\n await this._replicateFeed(feedInfo, streamTag);\n }\n\n /**\n * Respond to a remote request to replicate a feed.\n * @returns A stream tag for the replication stream or `undefined` if we don't want to replicate.\n */\n @synchronized\n private async _acceptReplication(feedInfo: FeedInfo): Promise<string | undefined> {\n invariant(this._extensionContext!.initiator === false, 'Invalid call');\n\n if (!this._feeds.has(feedInfo.feedKey) || this._streams.has(feedInfo.feedKey)) {\n return undefined; // We don't have the feed or we are already replicating it.\n }\n\n const tag = `feed-${feedInfo.feedKey.toHex()}-${PublicKey.random().toHex().slice(0, 8)}`; // Generate a unique tag for the stream.\n await this._replicateFeed(feedInfo, tag);\n return tag;\n }\n\n private async _replicateFeed(info: FeedInfo, streamTag: string) {\n log('replicate', { info, streamTag });\n invariant(!this._streams.has(info.feedKey), `Replication already in progress for feed: ${info.feedKey}`);\n\n const feed = this._feeds.get(info.feedKey) ?? failUndefined();\n const networkStream = await this._extensionContext!.createStream(streamTag, {\n contentType: 'application/x-hypercore',\n });\n\n // https://github.com/holepunchto/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options\n const replicationStream = feed.replicate(true, {\n live: true,\n upload: info.upload,\n download: info.download,\n noise: false,\n encrypted: false,\n maxRequests: 1024,\n });\n\n // Left for testing.\n // const debug = true;\n // if (debug) {\n // feed.on('sync', () => {\n // log.info('sync', { key: feed.key, length: feed.length });\n // });\n // feed.on('download', (index: number, data: any) => {\n // log.info('download', {\n // key: feed.key,\n // index,\n // length: feed.length,\n // data: data.length,\n // });\n // });\n // }\n\n replicationStream.on('error', (err) => {\n if (\n err?.message === 'Writable stream closed prematurely' ||\n err?.message === 'Cannot call write after a stream was destroyed'\n ) {\n return;\n }\n\n log.warn('replication stream error', { err, info });\n });\n\n this._streams.set(info.feedKey, {\n streamTag,\n networkStream,\n replicationStream,\n info,\n });\n\n networkStream.pipe(replicationStream as any).pipe(networkStream);\n }\n\n private async _stopReplication(feedKey: PublicKey) {\n const stream = this._streams.get(feedKey);\n if (!stream) {\n return;\n }\n\n stream.networkStream.destroy();\n this._streams.delete(feedKey);\n }\n}\n\ntype ServiceBundle = {\n ReplicatorService: ReplicatorService;\n};\n\ntype ActiveStream = {\n streamTag: string;\n networkStream: Duplex;\n replicationStream: ProtocolStream;\n info: FeedInfo;\n};\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACOA,mBAAyD;AACzD,qBAAwB;AACxB,mBAA8B;AAE9B,uBAA0B;AAC1B,kBAA0B;AAC1B,iBAA6B;AAC7B,uBAAuC;AAEvC,iBAAiD;AAEjD,kBAA2B;;;;;;;;;;;;AASpB,IAAMA,sBAAN,MAAMA;EAAN;AACYC,gBAAO,IAAIC,uBAAQ;MAClCC,SAAS,CAACC,QAAAA;AACRC,uBAAIC,MAAMF,KAAAA,QAAAA;;;;;;AACV,aAAKG,mBAAmBC,MAAMJ,GAAAA;MAChC;IACF,CAAA;AAEiBK,kBAAS,IAAIC,uBAAwCC,sBAAUC,IAAI;AACnEC,oBAAW,IAAIH,uBAAoCC,sBAAUC,IAAI;AAK1EE,oBAA+B;MACrCC,QAAQ;IACV;AAEiBC,uBAAc,IAAIC,0BAAa,KAAKhB,MAAM,YAAA;AACzD,UAAI;AACF,YAAI,KAAKM,kBAAmBW,cAAc,OAAO;AAC/C,gBAAM,KAAKC,KAAMC,IAAIC,kBAAkBC,YAAY;YACjDC,OAAOC,MAAMC,KAAK,KAAKhB,OAAOiB,OAAM,CAAA,EAAIC,IAAI,CAACC,UAAU;cACrDC,SAASD,KAAKE;cACdC,UAAU;cACVhB,QAAQ,KAAKD,SAASC;YACxB,EAAA;UACF,CAAA;QACF,WAAW,KAAKR,kBAAmBW,cAAc,MAAM;AACrD,gBAAM,KAAKc,iBAAgB;QAC7B;MACF,SAAS5B,KAAK;AACZ,YAAIA,eAAe6B,iCAAgB;AACjC;QACF;AACA,cAAM7B;MACR;IACF,CAAA;;EAEA,IACY8B,gBAAgB;AAC1B,WAAO;MACLhB,WAAW,KAAKX,mBAAmBW;MACnCiB,aAAa,KAAK5B,mBAAmB4B;MACrCC,cAAc,KAAK7B,mBAAmB6B;MACtCb,OAAOC,MAAMC,KAAK,KAAKhB,OAAO4B,KAAI,CAAA;IACpC;EACF;EAEAC,WAAWC,SAAmC;AAC5C,SAAKzB,WAAWyB;AAChBlC,wBAAI,cAAc;MAAEkC;IAAQ,GAAA;;;;;;AAC5B,QAAI,KAAKhC,mBAAmB;AAC1B,WAAKS,YAAYwB,SAAQ;IAC3B;AACA,WAAO;EACT;EAEAC,QAAQb,MAAwB;AAC9B,SAAKnB,OAAOiC,IAAId,KAAKE,KAAKF,IAAAA;AAC1BvB,wBAAI,WAAW;MAAEwB,SAASD,KAAKE;IAAI,GAAA;;;;;;AACnC,QAAI,KAAKvB,mBAAmB;AAC1B,WAAKS,YAAYwB,SAAQ;IAC3B;EACF;EAEA,MAAMG,OAAOC,SAA2B;AACtC,SAAKrC,oBAAoBqC;AACzBvC,wBAAI,QAAA,QAAA;;;;;;AAEJ,SAAKc,WAAO0B,+BAAiD;MAC3DC,WAAW;QACTzB,mBAAmB0B,wBAAOC,WAAW,iDAAA;MACvC;MACAC,SAAS;QACP5B,mBAAmB0B,wBAAOC,WAAW,iDAAA;MACvC;MACAE,UAAU;QACR7B,mBAAmB;UACjBC,aAAa,OAAO,EAAEC,MAAK,MAAE;AAC3BlB,gCAAI,sBAAsB;cAAEkB;YAAM,GAAA;;;;;;AAClC4B,4CAAU,KAAK5C,kBAAmBW,cAAc,MAAM,gBAAA;;;;;;;;;AACtD,iBAAKF,YAAYwB,SAAQ;UAC3B;UACAY,kBAAkB,OAAO,EAAEC,KAAI,MAAE;AAC/BhD,gCAAI,2BAA2B;cAAEgD;YAAK,GAAA;;;;;;AACtCF,4CAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAMoC,YAAY,MAAM,KAAKC,mBAAmBF,IAAAA;AAChD,mBAAO;cACLC;YACF;UACF;UACAE,iBAAiB,OAAO,EAAEH,KAAI,MAAE;AAC9BhD,gCAAI,2BAA2B;cAAEgD;YAAK,GAAA;;;;;;AAEtCF,4CAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,kBAAM,KAAKuC,iBAAiBJ,KAAKxB,OAAO;UAC1C;QACF;MACF;MACA6B,MAAM,MAAMd,QAAQe,WAAW,OAAO;QACpCC,aAAa;MACf,CAAA;IACF,CAAA;AACA,UAAM,KAAKzC,KAAK0C,KAAI;AAEpB,SAAK7C,YAAYwB,SAAQ;EAC3B;EAEA,MAAMsB,QAAQ1D,KAAyB;AACrCC,wBAAI,SAAS;MAAED;IAAI,GAAA;;;;;;AACnB,UAAM,KAAKH,KAAK8D,QAAO;AACvB,UAAM,KAAK5C,MAAMX,MAAAA;AACjB,eAAWqB,WAAW,KAAKhB,SAASwB,KAAI,GAAI;AAC1C,YAAM,KAAKoB,iBAAiB5B,OAAAA;IAC9B;EACF;EAEA,MACcG,mBAAmB;AAC/B3B,wBAAI,oBAAA,QAAA;;;;;;AACJ,eAAWwB,WAAW,KAAKpB,OAAO4B,KAAI,GAAI;AACxC,UAAI,KAAKpC,KAAK+D,UAAU;AACtB;MACF;AACA,UAAI,KAAKnD,SAASoD,IAAIpC,OAAAA,KAAY,KAAKf,SAASC,WAAW,KAAKF,SAASqD,IAAIrC,OAAAA,GAAUwB,KAAKtC,QAAQ;AAClG,YAAI;AACF,oBAAMoD,2BAAa,KAAKV,iBAAiB5B,OAAAA,GAAU,GAAA;QACrD,SAASzB,KAAK;AACZC,yBAAIC,MAAMF,KAAAA,QAAAA;;;;;;QACZ;MACF;AAEA,UAAI,KAAKH,KAAK+D,UAAU;AACtB;MACF;AACA,UAAI,CAAC,KAAKnD,SAASoD,IAAIpC,OAAAA,GAAU;AAC/B,cAAM,KAAKuC,qBAAqB;UAC9BvC;UACAE,UAAU;UACVhB,QAAQ,KAAKD,SAASC;QACxB,CAAA;MACF;IACF;EACF;;;;EAKA,MAAcqD,qBAAqBC,UAAoB;AACrDhE,wBAAI,0BAA0B;MAAEgE;IAAS,GAAA;;;;;;AACzClB,oCAAU,KAAK5C,kBAAmBW,cAAc,MAAM,gBAAA;;;;;;;;;AACtDiC,oCAAU,CAAC,KAAKtC,SAASoD,IAAII,SAASxC,OAAO,GAAG,6CAA6CwC,SAASxC,OAAO,IAAE;;;;;;;;;AAC/G,UAAM,EAAEyB,UAAS,IAAK,MAAM,KAAKnC,KAAMC,IAAIC,kBAAkB+B,iBAAiB;MAAEC,MAAMgB;IAAS,CAAA;AAC/F,QAAI,CAACf,WAAW;AACd;IACF;AAEA,UAAM,KAAKgB,eAAeD,UAAUf,SAAAA;EACtC;;;;;EAMA,MACcC,mBAAmBc,UAAiD;AAChFlB,oCAAU,KAAK5C,kBAAmBW,cAAc,OAAO,gBAAA;;;;;;;;;AAEvD,QAAI,CAAC,KAAKT,OAAOwD,IAAII,SAASxC,OAAO,KAAK,KAAKhB,SAASoD,IAAII,SAASxC,OAAO,GAAG;AAC7E,aAAO0C;IACT;AAEA,UAAMC,MAAM,QAAQH,SAASxC,QAAQ4C,MAAK,CAAA,IAAM9D,sBAAU+D,OAAM,EAAGD,MAAK,EAAGE,MAAM,GAAG,CAAA,CAAA;AACpF,UAAM,KAAKL,eAAeD,UAAUG,GAAAA;AACpC,WAAOA;EACT;EAEA,MAAcF,eAAejB,MAAgBC,WAAmB;AAC9DjD,wBAAI,aAAa;MAAEgD;MAAMC;IAAU,GAAA;;;;;;AACnCH,oCAAU,CAAC,KAAKtC,SAASoD,IAAIZ,KAAKxB,OAAO,GAAG,6CAA6CwB,KAAKxB,OAAO,IAAE;;;;;;;;;AAEvG,UAAMD,OAAO,KAAKnB,OAAOyD,IAAIb,KAAKxB,OAAO,SAAK+C,4BAAAA;AAC9C,UAAMC,gBAAgB,MAAM,KAAKtE,kBAAmBuE,aAAaxB,WAAW;MAC1EM,aAAa;IACf,CAAA;AAGA,UAAMmB,oBAAoBnD,KAAKoD,UAAU,MAAM;MAC7CC,MAAM;MACNlE,QAAQsC,KAAKtC;MACbgB,UAAUsB,KAAKtB;MACfmD,OAAO;MACPC,WAAW;MACXC,aAAa;IACf,CAAA;AAkBAL,sBAAkBM,GAAG,SAAS,CAACjF,QAAAA;AAC7B,UACEA,KAAKkF,YAAY,wCACjBlF,KAAKkF,YAAY,kDACjB;AACA;MACF;AAEAjF,qBAAIkF,KAAK,4BAA4B;QAAEnF;QAAKiD;MAAK,GAAA;;;;;;IACnD,CAAA;AAEA,SAAKxC,SAAS6B,IAAIW,KAAKxB,SAAS;MAC9ByB;MACAuB;MACAE;MACA1B;IACF,CAAA;AAEAwB,kBAAcW,KAAKT,iBAAAA,EAA0BS,KAAKX,aAAAA;EACpD;EAEA,MAAcpB,iBAAiB5B,SAAoB;AACjD,UAAM4D,SAAS,KAAK5E,SAASqD,IAAIrC,OAAAA;AACjC,QAAI,CAAC4D,QAAQ;AACX;IACF;AAEAA,WAAOZ,cAAca,QAAO;AAC5B,SAAK7E,SAAS8E,OAAO9D,OAAAA;EACvB;AACF;;EA9MG+D;GAvCU5F,oBAAAA,WAAAA,iBAAAA,IAAAA;;EAwHV6F;GAxHU7F,oBAAAA,WAAAA,oBAAAA,IAAAA;;EAuKV6F;GAvKU7F,oBAAAA,WAAAA,sBAAAA,IAAAA;",
|
|
6
|
+
"names": ["ReplicatorExtension", "_ctx", "Context", "onError", "err", "log", "catch", "_extensionContext", "close", "_feeds", "ComplexMap", "PublicKey", "hash", "_streams", "_options", "upload", "_updateTask", "DeferredTask", "initiator", "_rpc", "rpc", "ReplicatorService", "updateFeeds", "feeds", "Array", "from", "values", "map", "feed", "feedKey", "key", "download", "_reevaluateFeeds", "RpcClosedError", "extensionInfo", "localPeerId", "remotePeerId", "keys", "setOptions", "options", "schedule", "addFeed", "set", "onOpen", "context", "createProtoRpcPeer", "requested", "schema", "getService", "exposed", "handlers", "invariant", "startReplication", "info", "streamTag", "_acceptReplication", "stopReplication", "_stopReplication", "port", "createPort", "contentType", "open", "onClose", "dispose", "disposed", "has", "get", "asyncTimeout", "_initiateReplication", "feedInfo", "_replicateFeed", "undefined", "tag", "toHex", "random", "slice", "failUndefined", "networkStream", "createStream", "replicationStream", "replicate", "live", "noise", "encrypted", "maxRequests", "on", "message", "warn", "pipe", "stream", "destroy", "delete", "logInfo", "synchronized"]
|
|
7
7
|
}
|
package/dist/lib/node/meta.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":
|
|
1
|
+
{"inputs":{"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytes":35023,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/debug","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/rpc","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytes":520,"imports":[{"path":"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts","kind":"import-statement","original":"./replicator-extension"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-replicator/dist/lib/node/index.cjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":15161},"packages/core/mesh/teleport-extension-replicator/dist/lib/node/index.cjs":{"imports":[{"path":"@dxos/async","kind":"require-call","external":true},{"path":"@dxos/context","kind":"require-call","external":true},{"path":"@dxos/debug","kind":"require-call","external":true},{"path":"@dxos/invariant","kind":"require-call","external":true},{"path":"@dxos/keys","kind":"require-call","external":true},{"path":"@dxos/log","kind":"require-call","external":true},{"path":"@dxos/protocols","kind":"require-call","external":true},{"path":"@dxos/rpc","kind":"require-call","external":true},{"path":"@dxos/util","kind":"require-call","external":true}],"exports":[],"entryPoint":"packages/core/mesh/teleport-extension-replicator/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-replicator/src/index.ts":{"bytesInOutput":143},"packages/core/mesh/teleport-extension-replicator/src/replicator-extension.ts":{"bytesInOutput":11108}},"bytes":12345}}}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator-extension.d.ts","sourceRoot":"","sources":["../../../src/replicator-extension.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replicator-extension.d.ts","sourceRoot":"","sources":["../../../src/replicator-extension.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAO/C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGrE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,qBAAa,mBAAoB,YAAW,iBAAiB;IAC3D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAKlB;IAEH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+D;IACtF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2D;IAEpF,OAAO,CAAC,IAAI,CAAC,CAA8B;IAC3C,OAAO,CAAC,iBAAiB,CAAC,CAAmB;IAE7C,OAAO,CAAC,QAAQ,CAEd;IAEF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAmBzB;IAGH,OAAO,KAAK,aAAa,GAOxB;IAED,UAAU,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI;IAS7C,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC;IAQxB,MAAM,CAAC,OAAO,EAAE,gBAAgB;IA6ChC,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS;YAUvB,gBAAgB;IA2B9B;;OAEG;YACW,oBAAoB;IAYlC;;;OAGG;YAEW,kBAAkB;YAYlB,cAAc;YAwDd,gBAAgB;CAS/B"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { FeedStore } from '@dxos/feed-store';
|
|
1
|
+
import { FeedOptions, FeedStore } from '@dxos/feed-store';
|
|
2
2
|
import { Keyring } from '@dxos/keyring';
|
|
3
3
|
import { PublicKey } from '@dxos/keys';
|
|
4
4
|
import { Teleport } from '@dxos/teleport';
|
|
@@ -11,7 +11,7 @@ export declare class TestAgent {
|
|
|
11
11
|
keyring: Keyring;
|
|
12
12
|
feedStore: FeedStore<{}>;
|
|
13
13
|
createWriteFeed(numBlocks?: number): Promise<import("@dxos/feed-store").FeedWrapper<{}>>;
|
|
14
|
-
createReadFeed(key: PublicKey): Promise<import("@dxos/feed-store").FeedWrapper<{}>>;
|
|
14
|
+
createReadFeed(key: PublicKey, opts?: FeedOptions): Promise<import("@dxos/feed-store").FeedWrapper<{}>>;
|
|
15
15
|
}
|
|
16
16
|
/**
|
|
17
17
|
* Simulates two peers connected via P2P network.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../../src/testing.ts"],"names":[],"mappings":"AAMA,OAAO,EAAe,SAAS,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../../src/testing.ts"],"names":[],"mappings":"AAMA,OAAO,EAAe,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAI1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,qBAAa,WAAW;IACtB,WAAW,IAAI,SAAS;CAGzB;AAGD,qBAAa,SAAS;IACb,OAAO,gDAA4C;IACnD,OAAO,UAAwD;IAC/D,SAAS,gBAEb;IAEG,eAAe,CAAC,SAAS,SAAI;IAUnC,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,WAAW;CAGlD;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;EAuB5B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,oBAAoB;;;EAUhC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/teleport-extension-replicator",
|
|
3
|
-
"version": "0.1.56-main.
|
|
3
|
+
"version": "0.1.56-main.e79d64a",
|
|
4
4
|
"description": "Replicator extension for teleport.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -16,25 +16,25 @@
|
|
|
16
16
|
"src"
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"
|
|
20
|
-
"@dxos/
|
|
21
|
-
"@dxos/
|
|
22
|
-
"@dxos/
|
|
23
|
-
"@dxos/
|
|
24
|
-
"@dxos/keys": "0.1.56-main.
|
|
25
|
-
"@dxos/log": "0.1.56-main.
|
|
26
|
-
"@dxos/node-std": "0.1.56-main.
|
|
27
|
-
"@dxos/protocols": "0.1.56-main.
|
|
28
|
-
"@dxos/rpc": "0.1.56-main.
|
|
29
|
-
"@dxos/teleport": "0.1.56-main.
|
|
30
|
-
"@dxos/util": "0.1.56-main.
|
|
19
|
+
"@dxos/async": "0.1.56-main.e79d64a",
|
|
20
|
+
"@dxos/context": "0.1.56-main.e79d64a",
|
|
21
|
+
"@dxos/debug": "0.1.56-main.e79d64a",
|
|
22
|
+
"@dxos/feed-store": "0.1.56-main.e79d64a",
|
|
23
|
+
"@dxos/invariant": "0.1.56-main.e79d64a",
|
|
24
|
+
"@dxos/keys": "0.1.56-main.e79d64a",
|
|
25
|
+
"@dxos/log": "0.1.56-main.e79d64a",
|
|
26
|
+
"@dxos/node-std": "0.1.56-main.e79d64a",
|
|
27
|
+
"@dxos/protocols": "0.1.56-main.e79d64a",
|
|
28
|
+
"@dxos/rpc": "0.1.56-main.e79d64a",
|
|
29
|
+
"@dxos/teleport": "0.1.56-main.e79d64a",
|
|
30
|
+
"@dxos/util": "0.1.56-main.e79d64a"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"add": "^2.0.6",
|
|
34
34
|
"fast-check": "~3.3.0",
|
|
35
35
|
"typescript": "^5.0.4",
|
|
36
|
-
"@dxos/keyring": "0.1.56-main.
|
|
37
|
-
"@dxos/random-access-storage": "0.1.56-main.
|
|
36
|
+
"@dxos/keyring": "0.1.56-main.e79d64a",
|
|
37
|
+
"@dxos/random-access-storage": "0.1.56-main.e79d64a"
|
|
38
38
|
},
|
|
39
39
|
"publishConfig": {
|
|
40
40
|
"access": "public"
|
|
@@ -6,6 +6,7 @@ import expect from 'expect';
|
|
|
6
6
|
|
|
7
7
|
import { Event, sleep } from '@dxos/async';
|
|
8
8
|
import { describe, test } from '@dxos/test';
|
|
9
|
+
import { range } from '@dxos/util';
|
|
9
10
|
|
|
10
11
|
import { createReplicatorPair, TestBuilder } from './testing';
|
|
11
12
|
|
|
@@ -103,6 +104,33 @@ describe('ReplicatorExtension', () => {
|
|
|
103
104
|
await Event.wrap(feed1B, 'download').waitForCondition(() => feed1B.length === 10);
|
|
104
105
|
});
|
|
105
106
|
|
|
107
|
+
test('sparse replication', async () => {
|
|
108
|
+
const builder = new TestBuilder();
|
|
109
|
+
const agent1 = builder.createAgent();
|
|
110
|
+
const agent2 = builder.createAgent();
|
|
111
|
+
|
|
112
|
+
const { replicator1, replicator2 } = await createReplicatorPair();
|
|
113
|
+
|
|
114
|
+
replicator1.setOptions({ upload: true });
|
|
115
|
+
replicator2.setOptions({ upload: true });
|
|
116
|
+
|
|
117
|
+
const feed1 = await agent1.createWriteFeed(10);
|
|
118
|
+
const feed2 = await agent2.createReadFeed(feed1.key, { sparse: true });
|
|
119
|
+
|
|
120
|
+
replicator1.addFeed(feed1);
|
|
121
|
+
replicator2.addFeed(feed2);
|
|
122
|
+
|
|
123
|
+
void feed2.download({ start: 10, linear: true });
|
|
124
|
+
|
|
125
|
+
await sleep(50);
|
|
126
|
+
|
|
127
|
+
expect(range(10).every((idx) => feed2.has(idx))).toEqual(false);
|
|
128
|
+
|
|
129
|
+
void feed2.download({ start: 0, linear: true });
|
|
130
|
+
|
|
131
|
+
await Event.wrap(feed2, 'download').waitForCondition(() => range(10).every((idx) => feed2.has(idx)));
|
|
132
|
+
});
|
|
133
|
+
|
|
106
134
|
// TODO: not working yet.
|
|
107
135
|
// eslint-disable-next-line mocha/no-skipped-tests
|
|
108
136
|
test.skip('enabling upload mid replication', async () => {
|
|
@@ -4,12 +4,12 @@
|
|
|
4
4
|
|
|
5
5
|
import type { ProtocolStream } from 'hypercore-protocol';
|
|
6
6
|
import { Duplex } from 'node:stream';
|
|
7
|
-
import invariant from 'tiny-invariant';
|
|
8
7
|
|
|
9
8
|
import { asyncTimeout, DeferredTask, synchronized } from '@dxos/async';
|
|
10
9
|
import { Context } from '@dxos/context';
|
|
11
10
|
import { failUndefined } from '@dxos/debug';
|
|
12
11
|
import { FeedWrapper } from '@dxos/feed-store';
|
|
12
|
+
import { invariant } from '@dxos/invariant';
|
|
13
13
|
import { PublicKey } from '@dxos/keys';
|
|
14
14
|
import { log, logInfo } from '@dxos/log';
|
|
15
15
|
import { schema, RpcClosedError } from '@dxos/protocols';
|
|
@@ -213,21 +213,32 @@ export class ReplicatorExtension implements TeleportExtension {
|
|
|
213
213
|
const networkStream = await this._extensionContext!.createStream(streamTag, {
|
|
214
214
|
contentType: 'application/x-hypercore',
|
|
215
215
|
});
|
|
216
|
+
|
|
217
|
+
// https://github.com/holepunchto/hypercore/tree/v9.12.0#var-stream--feedreplicateisinitiator-options
|
|
216
218
|
const replicationStream = feed.replicate(true, {
|
|
217
219
|
live: true,
|
|
218
220
|
upload: info.upload,
|
|
219
221
|
download: info.download,
|
|
220
222
|
noise: false,
|
|
221
223
|
encrypted: false,
|
|
224
|
+
maxRequests: 1024,
|
|
222
225
|
});
|
|
223
226
|
|
|
224
|
-
//
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
//
|
|
227
|
+
// Left for testing.
|
|
228
|
+
// const debug = true;
|
|
229
|
+
// if (debug) {
|
|
230
|
+
// feed.on('sync', () => {
|
|
231
|
+
// log.info('sync', { key: feed.key, length: feed.length });
|
|
232
|
+
// });
|
|
233
|
+
// feed.on('download', (index: number, data: any) => {
|
|
234
|
+
// log.info('download', {
|
|
235
|
+
// key: feed.key,
|
|
236
|
+
// index,
|
|
237
|
+
// length: feed.length,
|
|
238
|
+
// data: data.length,
|
|
239
|
+
// });
|
|
229
240
|
// });
|
|
230
|
-
// }
|
|
241
|
+
// }
|
|
231
242
|
|
|
232
243
|
replicationStream.on('error', (err) => {
|
|
233
244
|
if (
|
package/src/stress.test.ts
CHANGED
|
@@ -209,8 +209,8 @@ describe('stress-tests', () => {
|
|
|
209
209
|
|
|
210
210
|
await fc.assert(
|
|
211
211
|
fc.asyncProperty(fc.commands(allCommands, { maxCommands: 100 }), async (cmds) => {
|
|
212
|
-
console.log('\n=====');
|
|
213
|
-
console.log([...cmds].map((c) => c.toString()).join('\n'));
|
|
212
|
+
// console.log('\n=====');
|
|
213
|
+
// console.log([...cmds].map((c) => c.toString()).join('\n'));
|
|
214
214
|
|
|
215
215
|
const system = await factory(keyring)();
|
|
216
216
|
try {
|
package/src/testing.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { pipeline } from 'stream';
|
|
6
6
|
|
|
7
|
-
import { FeedFactory, FeedStore } from '@dxos/feed-store';
|
|
7
|
+
import { FeedFactory, FeedOptions, FeedStore } from '@dxos/feed-store';
|
|
8
8
|
import { Keyring } from '@dxos/keyring';
|
|
9
9
|
import { PublicKey } from '@dxos/keys';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
@@ -39,8 +39,8 @@ export class TestAgent {
|
|
|
39
39
|
return feed;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
createReadFeed(key: PublicKey) {
|
|
43
|
-
return this.feedStore.openFeed(key);
|
|
42
|
+
createReadFeed(key: PublicKey, opts?: FeedOptions) {
|
|
43
|
+
return this.feedStore.openFeed(key, opts);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
|