@dxos/teleport-extension-object-sync 0.8.3 → 0.8.4-main.1da679c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import "@dxos/node-std/globals";
2
2
 
3
- // packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts
3
+ // src/blob-sync-extension.ts
4
4
  import { DeferredTask, sleep, synchronized } from "@dxos/async";
5
5
  import { Context } from "@dxos/context";
6
6
  import { invariant } from "@dxos/invariant";
@@ -9,91 +9,29 @@ import { RpcClosedError } from "@dxos/protocols";
9
9
  import { schema } from "@dxos/protocols/proto";
10
10
  import { RpcExtension } from "@dxos/teleport";
11
11
  import { BitField } from "@dxos/util";
12
+ function _define_property(obj, key, value) {
13
+ if (key in obj) {
14
+ Object.defineProperty(obj, key, {
15
+ value,
16
+ enumerable: true,
17
+ configurable: true,
18
+ writable: true
19
+ });
20
+ } else {
21
+ obj[key] = value;
22
+ }
23
+ return obj;
24
+ }
12
25
  function _ts_decorate(decorators, target, key, desc) {
13
26
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
14
27
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
15
28
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
16
29
  return c > 3 && r && Object.defineProperty(target, key, r), r;
17
30
  }
18
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
31
+ var __dxlog_file = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
19
32
  var MIN_WANT_LIST_UPDATE_INTERVAL = false ? 5 : 500;
20
33
  var MAX_CONCURRENT_UPLOADS = 20;
21
34
  var BlobSyncExtension = class extends RpcExtension {
22
- constructor(_params) {
23
- super({
24
- exposed: {
25
- BlobSyncService: schema.getService("dxos.mesh.teleport.blobsync.BlobSyncService")
26
- },
27
- requested: {
28
- BlobSyncService: schema.getService("dxos.mesh.teleport.blobsync.BlobSyncService")
29
- },
30
- timeout: 2e4,
31
- encodingOptions: {
32
- preserveAny: true
33
- }
34
- }), this._params = _params, this._ctx = new Context({
35
- onError: (err) => log.catch(err, void 0, {
36
- F: __dxlog_file,
37
- L: 35,
38
- S: this,
39
- C: (f, a) => f(...a)
40
- })
41
- }, {
42
- F: __dxlog_file,
43
- L: 35
44
- }), this._lastWantListUpdate = 0, this._localWantList = {
45
- blobs: []
46
- }, this._updateWantList = new DeferredTask(this._ctx, async () => {
47
- if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {
48
- await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());
49
- if (this._ctx.disposed) {
50
- return;
51
- }
52
- }
53
- log("want", {
54
- list: this._localWantList
55
- }, {
56
- F: __dxlog_file,
57
- L: 49,
58
- S: this,
59
- C: (f, a) => f(...a)
60
- });
61
- await this.rpc.BlobSyncService.want(this._localWantList);
62
- this._lastWantListUpdate = Date.now();
63
- }), this._currentUploads = 0, this._upload = new DeferredTask(this._ctx, async () => {
64
- if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {
65
- return;
66
- }
67
- const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);
68
- if (!blobChunks) {
69
- return;
70
- }
71
- for (const blobChunk of blobChunks) {
72
- if (this._ctx.disposed) {
73
- break;
74
- }
75
- this._currentUploads++;
76
- this.push(blobChunk).catch((err) => {
77
- if (err instanceof RpcClosedError) {
78
- return;
79
- }
80
- log.warn("push failed", {
81
- err
82
- }, {
83
- F: __dxlog_file,
84
- L: 76,
85
- S: this,
86
- C: (f, a) => f(...a)
87
- });
88
- }).finally(() => {
89
- this._currentUploads--;
90
- this.reconcileUploads();
91
- });
92
- }
93
- }), this.remoteWantList = {
94
- blobs: []
95
- };
96
- }
97
35
  async onOpen(context) {
98
36
  log("open", void 0, {
99
37
  F: __dxlog_file,
@@ -259,37 +197,118 @@ var BlobSyncExtension = class extends RpcExtension {
259
197
  }
260
198
  return chunks;
261
199
  }
200
+ constructor(_params) {
201
+ super({
202
+ exposed: {
203
+ BlobSyncService: schema.getService("dxos.mesh.teleport.blobsync.BlobSyncService")
204
+ },
205
+ requested: {
206
+ BlobSyncService: schema.getService("dxos.mesh.teleport.blobsync.BlobSyncService")
207
+ },
208
+ timeout: 2e4,
209
+ encodingOptions: {
210
+ preserveAny: true
211
+ }
212
+ }), _define_property(this, "_params", void 0), _define_property(this, "_ctx", void 0), _define_property(this, "_lastWantListUpdate", void 0), _define_property(this, "_localWantList", void 0), _define_property(this, "_updateWantList", void 0), _define_property(this, "_currentUploads", void 0), _define_property(this, "_upload", void 0), /**
213
+ * Set of id's remote peer wants.
214
+ */
215
+ _define_property(this, "remoteWantList", void 0), this._params = _params, this._ctx = new Context({
216
+ onError: (err) => log.catch(err, void 0, {
217
+ F: __dxlog_file,
218
+ L: 35,
219
+ S: this,
220
+ C: (f, a) => f(...a)
221
+ })
222
+ }, {
223
+ F: __dxlog_file,
224
+ L: 35
225
+ }), this._lastWantListUpdate = 0, this._localWantList = {
226
+ blobs: []
227
+ }, this._updateWantList = new DeferredTask(this._ctx, async () => {
228
+ if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {
229
+ await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());
230
+ if (this._ctx.disposed) {
231
+ return;
232
+ }
233
+ }
234
+ log("want", {
235
+ list: this._localWantList
236
+ }, {
237
+ F: __dxlog_file,
238
+ L: 49,
239
+ S: this,
240
+ C: (f, a) => f(...a)
241
+ });
242
+ await this.rpc.BlobSyncService.want(this._localWantList);
243
+ this._lastWantListUpdate = Date.now();
244
+ }), this._currentUploads = 0, this._upload = new DeferredTask(this._ctx, async () => {
245
+ if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {
246
+ return;
247
+ }
248
+ const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);
249
+ if (!blobChunks) {
250
+ return;
251
+ }
252
+ for (const blobChunk of blobChunks) {
253
+ if (this._ctx.disposed) {
254
+ break;
255
+ }
256
+ this._currentUploads++;
257
+ this.push(blobChunk).catch((err) => {
258
+ if (err instanceof RpcClosedError) {
259
+ return;
260
+ }
261
+ log.warn("push failed", {
262
+ err
263
+ }, {
264
+ F: __dxlog_file,
265
+ L: 76,
266
+ S: this,
267
+ C: (f, a) => f(...a)
268
+ });
269
+ }).finally(() => {
270
+ this._currentUploads--;
271
+ this.reconcileUploads();
272
+ });
273
+ }
274
+ }), this.remoteWantList = {
275
+ blobs: []
276
+ };
277
+ }
262
278
  };
263
279
  _ts_decorate([
264
280
  synchronized
265
281
  ], BlobSyncExtension.prototype, "push", null);
266
282
 
267
- // packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts
268
- import { trackLeaks, Trigger, Mutex } from "@dxos/async";
269
- import { cancelWithContext, Context as Context2 } from "@dxos/context";
283
+ // src/blob-sync.ts
284
+ import { Mutex, Trigger, trackLeaks } from "@dxos/async";
285
+ import { Context as Context2, cancelWithContext } from "@dxos/context";
270
286
  import { invariant as invariant2 } from "@dxos/invariant";
271
287
  import { PublicKey } from "@dxos/keys";
272
288
  import { log as log2 } from "@dxos/log";
273
289
  import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
274
290
  import { BitField as BitField2, ComplexMap } from "@dxos/util";
291
+ function _define_property2(obj, key, value) {
292
+ if (key in obj) {
293
+ Object.defineProperty(obj, key, {
294
+ value,
295
+ enumerable: true,
296
+ configurable: true,
297
+ writable: true
298
+ });
299
+ } else {
300
+ obj[key] = value;
301
+ }
302
+ return obj;
303
+ }
275
304
  function _ts_decorate2(decorators, target, key, desc) {
276
305
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
277
306
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
278
307
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
279
308
  return c > 3 && r && Object.defineProperty(target, key, r), r;
280
309
  }
281
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
310
+ var __dxlog_file2 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
282
311
  var BlobSync = class {
283
- constructor(_params) {
284
- this._params = _params;
285
- this._ctx = new Context2(void 0, {
286
- F: __dxlog_file2,
287
- L: 30
288
- });
289
- this._mutex = new Mutex();
290
- this._downloadRequests = new ComplexMap((key) => PublicKey.from(key).toHex());
291
- this._extensions = /* @__PURE__ */ new Set();
292
- }
293
312
  async open() {
294
313
  }
295
314
  async close() {
@@ -431,12 +450,27 @@ var BlobSync = class {
431
450
  extension.updateWantList(this._getWantList());
432
451
  }
433
452
  }
453
+ constructor(_params) {
454
+ _define_property2(this, "_params", void 0);
455
+ _define_property2(this, "_ctx", void 0);
456
+ _define_property2(this, "_mutex", void 0);
457
+ _define_property2(this, "_downloadRequests", void 0);
458
+ _define_property2(this, "_extensions", void 0);
459
+ this._params = _params;
460
+ this._ctx = new Context2(void 0, {
461
+ F: __dxlog_file2,
462
+ L: 30
463
+ });
464
+ this._mutex = new Mutex();
465
+ this._downloadRequests = new ComplexMap((key) => PublicKey.from(key).toHex());
466
+ this._extensions = /* @__PURE__ */ new Set();
467
+ }
434
468
  };
435
469
  BlobSync = _ts_decorate2([
436
470
  trackLeaks("open", "close")
437
471
  ], BlobSync);
438
472
 
439
- // packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts
473
+ // src/blob-store.ts
440
474
  import path from "@dxos/node-std/path";
441
475
  import { synchronized as synchronized2 } from "@dxos/async";
442
476
  import { subtleCrypto } from "@dxos/crypto";
@@ -445,19 +479,29 @@ import { PublicKey as PublicKey2 } from "@dxos/keys";
445
479
  import { schema as schema2 } from "@dxos/protocols/proto";
446
480
  import { BlobMeta as BlobMeta2 } from "@dxos/protocols/proto/dxos/echo/blob";
447
481
  import { BitField as BitField3, arrayToBuffer } from "@dxos/util";
482
+ function _define_property3(obj, key, value) {
483
+ if (key in obj) {
484
+ Object.defineProperty(obj, key, {
485
+ value,
486
+ enumerable: true,
487
+ configurable: true,
488
+ writable: true
489
+ });
490
+ } else {
491
+ obj[key] = value;
492
+ }
493
+ return obj;
494
+ }
448
495
  function _ts_decorate3(decorators, target, key, desc) {
449
496
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
450
497
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
451
498
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
452
499
  return c > 3 && r && Object.defineProperty(target, key, r), r;
453
500
  }
454
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
501
+ var __dxlog_file3 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
455
502
  var DEFAULT_CHUNK_SIZE = 4096;
456
503
  var BlobMetaCodec = schema2.getCodecForType("dxos.echo.blob.BlobMeta");
457
504
  var BlobStore = class {
458
- constructor(_directory) {
459
- this._directory = _directory;
460
- }
461
505
  async getMeta(id) {
462
506
  return this._getMeta(id);
463
507
  }
@@ -609,6 +653,10 @@ var BlobStore = class {
609
653
  _getDataFile(id) {
610
654
  return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString("hex"), "data"));
611
655
  }
656
+ constructor(_directory) {
657
+ _define_property3(this, "_directory", void 0);
658
+ this._directory = _directory;
659
+ }
612
660
  };
613
661
  _ts_decorate3([
614
662
  synchronized2
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../../src/blob-sync-extension.ts", "../../../src/blob-sync.ts", "../../../src/blob-store.ts"],
4
- "sourcesContent": ["//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { RpcClosedError } from '@dxos/protocols';\nimport { schema } from '@dxos/protocols/proto';\nimport { type BlobChunk, type BlobSyncService, type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onAbort: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = process.env.NODE_ENV === 'test' ? 5 : 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n override async onAbort(err?: Error | undefined): Promise<void> {\n log('abort');\n await this._ctx.dispose();\n await this._params.onAbort();\n await super.onAbort(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk): Promise<void> {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList): void {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads(): void {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { trackLeaks, Trigger, Mutex } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _mutex = new Mutex();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open(): Promise<void> {}\n\n async close(): Promise<void> {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._mutex.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._mutex.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension(): BlobSyncExtension {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onAbort: async () => {\n log('extension aborted');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array): Promise<void> {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads(): void {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList(): void {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { schema } from '@dxos/protocols/proto';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n constructor(private readonly _directory: Directory) {}\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
5
- "mappings": ";;;AAMA,SAASA,cAAcC,OAAOC,oBAAoB;AAClD,SAASC,eAAe;AACxB,SAASC,iBAAiB;AAC1B,SAASC,WAAW;AACpB,SAASC,sBAAsB;AAC/B,SAASC,cAAc;AAEvB,SAAgCC,oBAAoB;AACpD,SAASC,gBAAgB;;;;;;;;AAYzB,IAAMC,gCAAgCC,QAAkC,IAAI;AAE5E,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCL,aAAAA;EAwDrC,YACmBM,SACjB;AACA,UAAM;MACJC,SAAS;QACPC,iBAAiBT,OAAOU,WAAW,6CAAA;MACrC;MACAC,WAAW;QACTF,iBAAiBT,OAAOU,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA,GAAA,KAbiBP,UAAAA,SAAAA,KAxDFQ,OAAO,IAAInB,QAAQ;MAAEoB,SAAS,CAACC,QAAQnB,IAAIoB,MAAMD,KAAAA,QAAAA;;;;;;IAAK,GAAA;;;aAE/DE,sBAAsB,GAAA,KACtBC,iBAA2B;MAAEC,OAAO,CAAA;IAAG,GAAA,KAE9BC,kBAAkB,IAAI7B,aAAa,KAAKsB,MAAM,YAAA;AAE7D,UAAI,KAAKI,sBAAsBhB,gCAAgCoB,KAAKC,IAAG,GAAI;AACzE,cAAM9B,MAAM,KAAKyB,sBAAsBhB,gCAAgCoB,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAKT,KAAKU,UAAU;AACtB;QACF;MACF;AAEA3B,UAAI,QAAQ;QAAE4B,MAAM,KAAKN;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKO,IAAIlB,gBAAgBmB,KAAK,KAAKR,cAAc;AACvD,WAAKD,sBAAsBI,KAAKC,IAAG;IACrC,CAAA,GAAA,KAEQK,kBAAkB,GAAA,KAETC,UAAU,IAAIrC,aAAa,KAAKsB,MAAM,YAAA;AACrD,UAAI,KAAKc,mBAAmBxB,wBAAwB;AAClD;MACF;AACA,YAAM0B,aAAa,MAAM,KAAKC,gBAAgB3B,yBAAyB,KAAKwB,eAAe;AAC3F,UAAI,CAACE,YAAY;AACf;MACF;AACA,iBAAWE,aAAaF,YAAY;AAClC,YAAI,KAAKhB,KAAKU,UAAU;AACtB;QACF;AAEA,aAAKI;AAEL,aAAKK,KAAKD,SAAAA,EACPf,MAAM,CAACD,QAAAA;AACN,cAAIA,eAAelB,gBAAgB;AACjC;UACF;AACAD,cAAIqC,KAAK,eAAe;YAAElB;UAAI,GAAA;;;;;;QAChC,CAAA,EACCmB,QAAQ,MAAA;AACP,eAAKP;AACL,eAAKQ,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA,GAAA,KAKOC,iBAA2B;MAAEjB,OAAO,CAAA;IAAG;EAiB9C;EAEA,MAAekB,OAAOC,SAA0C;AAC9D1C,QAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMyC,OAAOC,OAAAA;AACnB,UAAM,KAAKjC,QAAQgC,OAAM;EAC3B;EAEA,MAAeE,QAAQxB,KAAwC;AAC7DnB,QAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKiB,KAAK2B,QAAO;AACvB,UAAM,KAAKnC,QAAQkC,QAAO;AAC1B,UAAM,MAAMA,QAAQxB,GAAAA;EACtB;EAEA,MAAe0B,QAAQ1B,KAAwC;AAC7DnB,QAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKiB,KAAK2B,QAAO;AACvB,UAAM,KAAKnC,QAAQoC,QAAO;AAC1B,UAAM,MAAMA,QAAQ1B,GAAAA;EACtB;EAEA,MAAgB2B,cAAsC;AACpD,WAAO;MACLnC,iBAAiB;QACfmB,MAAM,OAAOiB,aAAAA;AACX/C,cAAI,eAAe;YAAEwC,gBAAgBO;UAAS,GAAA;;;;;;AAC9C,eAAKP,iBAAiBO;AACtB,eAAKR,iBAAgB;QACvB;QACAH,MAAM,OAAOY,SAAAA;AACXhD,cAAI,YAAY;YAAEgD;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAKvC,QAAQwC,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMZ,KAAKY,MAAgC;AACzC,QAAI,KAAK/B,KAAKU,UAAU;AACtB;IACF;AACA3B,QAAI,QAAQ;MAAEgD;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKnB,IAAIlB,gBAAgByB,KAAKY,IAAAA;EACtC;EAEAE,eAAeH,UAA0B;AACvC,QAAI,KAAK9B,KAAKU,UAAU;AACtB;IACF;AACA,SAAKL,iBAAiByB;AACtB,SAAKvB,gBAAgB2B,SAAQ;EAC/B;EAEAZ,mBAAyB;AACvB,QAAI,KAAKtB,KAAKU,UAAU;AACtB;IACF;AACA,SAAKK,QAAQmB,SAAQ;EACvB;EAEA,MAAcjB,gBAAgBkB,SAAS,GAAgC;AACrE,QAAI,KAAKnC,KAAKU,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKa,eAAejB,SAAS,KAAKiB,eAAejB,OAAO8B,WAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKd,eAAejB;MAAOgC,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKnD,QAAQoD,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACA7D,gBAAU6D,KAAKI,UAAQ,QAAA;;;;;;;;;AACvBjE,gBAAU6D,KAAKK,WAAS,QAAA;;;;;;;;;AACxBlE,gBAAU6D,KAAKP,QAAM,QAAA;;;;;;;;;AAErB,UAAIM,OAAOM,aAAaN,OAAOM,cAAcL,KAAKK,WAAW;AAC3DjE,YAAIqC,KAAK,sBAAsB;UAAEsB;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMM,kBAAkBP,OAAOK,YAAY5D,SAAS+D,KAAKP,KAAKP,SAASO,KAAKK,SAAS;AAErF,YAAMG,cAAchE,SAASiE,IAAIH,iBAAiBN,KAAKI,QAAQ;AAC/D,YAAMM,eAAelE,SAASmE,YAAYH,WAAAA,EAAab,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWe,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAKhE,QAAQoD,UAAUa,IAAIf,OAAOI,IAAI;UAC5DY,QAAQH,MAAMZ,KAAKK;UACnBZ,QAAQG,KAAKoB,IAAIhB,KAAKK,WAAWL,KAAKP,SAASmB,MAAMZ,KAAKK,SAAS;QACrE,CAAA;AACAP,eAAOtB,KAAK;UACV2B,IAAIJ,OAAOI;UACXc,aAAajB,KAAKP;UAClBY,WAAWL,KAAKK;UAChBa,aAAaN,MAAMZ,KAAKK;UACxBc,SAASN;QACX,CAAA;AAEA,YAAIf,OAAOL,UAAUD,QAAQ;AAC3B,iBAAOM;QACT;MACF;IACF;AAEA,WAAOA;EACT;AACF;;;;;;ACzNA,SAASsB,YAAYC,SAASC,aAAa;AAC3C,SAASC,mBAAmBC,WAAAA,gBAAe;AAC3C,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,gBAAgB;AAEzB,SAASC,YAAAA,WAAUC,kBAAkB;;;;;;;;AAiB9B,IAAMC,WAAN,MAAMA;EAUX,YAA6BC,SAAyB;SAAzBA,UAAAA;SATZC,OAAO,IAAIC,SAAAA,QAAAA;;;;SACXC,SAAS,IAAIC,MAAAA;SAEbC,oBAAoB,IAAIC,WAAwC,CAACC,QAChFC,UAAUC,KAAKF,GAAAA,EAAKG,MAAK,CAAA;SAGVC,cAAc,oBAAIC,IAAAA;EAEoB;EAEvD,MAAMC,OAAsB;EAAC;EAE7B,MAAMC,QAAuB;AAC3B,UAAM,KAAKb,KAAKc,QAAO;EACzB;;;;;;EAOA,MAAMC,SAASC,KAAcC,IAA+B;AAC1DC,IAAAA,KAAI,YAAY;MAAED;IAAG,GAAA;;;;;;AACrB,UAAME,UAAU,MAAM,KAAKjB,OAAOkB,oBAAoB,YAAA;AACpD,YAAMC,kBAAkB,KAAKjB,kBAAkBkB,IAAIL,EAAAA;AAEnD,UAAII,iBAAiB;AACnBA,wBAAgBE;AAChB,eAAOF;MACT;AAEA,YAAMG,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUC,QAAQT,EAAAA;AAClD,YAAME,WAA2B;QAC/BQ,SAAS,IAAIC,QAAAA;QACbL,SAAS;QACTM,MAAM;UACJZ;UACAa,WAAWN,MAAMM;UACjBC,UAAUP,MAAMO,YAAYC,WAAWxB,KAAKyB,UAASC,OAAOV,KAAKO,QAAQ,CAAA;QAC3E;MACF;AAGA,UAAIP,MAAMW,UAAUC,SAASC,MAAMC,eAAe;AAChDnB,QAAAA,SAAQQ,QAAQY,KAAI;MACtB,OAAO;AACL,aAAKnC,kBAAkBoC,IAAIvB,IAAIE,QAAAA;AAC/B,aAAKsB,0BAAyB;MAChC;AAEA,aAAOtB;IACT,CAAA;AAEAH,SAAK0B,UAAU,MACb,KAAKxC,OAAOkB,oBAAoB,YAAA;AAE9B,YAAMD,WAAU,KAAKf,kBAAkBkB,IAAIL,EAAAA;AAC3C,UAAI,CAACE,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQI,YAAY,GAAG;AAC3B,aAAKnB,kBAAkBuC,OAAO1B,EAAAA;MAChC;AACA,WAAKwB,0BAAyB;IAChC,CAAA,CAAA;AAGF,WAAOzB,MAAM4B,kBAAkB5B,KAAKG,QAAQQ,QAAQkB,KAAI,CAAA,IAAM1B,QAAQQ,QAAQkB,KAAI;EACpF;EAEAC,kBAAqC;AACnC,UAAMC,YAAY,IAAIC,kBAAkB;MACtCvB,WAAW,KAAK1B,QAAQ0B;MACxBwB,QAAQ,YAAA;AACN/B,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYwC,IAAIH,SAAAA;AACrBA,kBAAUI,eAAe,KAAKC,aAAY,CAAA;MAC5C;MACAC,SAAS,YAAA;AACPnC,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYiC,OAAOI,SAAAA;MAC1B;MACAO,SAAS,YAAA;AACPpC,QAAAA,KAAI,qBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYiC,OAAOI,SAAAA;MAC1B;MACAQ,QAAQ,OAAOC,cAAAA;AACb,YAAI,CAAC,KAAKpD,kBAAkBqD,IAAID,UAAUvC,EAAE,GAAG;AAC7C;QACF;AACAC,QAAAA,KAAI,YAAY;UAAEsC;QAAU,GAAA;;;;;;AAC5B,cAAMhC,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUiC,SAASF,SAAAA;AACnD,YAAIhC,KAAKW,UAAUC,SAASC,MAAMC,eAAe;AAC/C,eAAKlC,kBAAkBkB,IAAIkC,UAAUvC,EAAE,GAAGU,QAAQY,KAAAA;AAClD,eAAKnC,kBAAkBuC,OAAOa,UAAUvC,EAAE;QAC5C,OAAO;AACL0C,UAAAA,WAAUnC,KAAKO,UAAQ,QAAA;;;;;;;;;AACvB,eAAK3B,kBAAkBkB,IAAIkC,UAAUvC,EAAE,EAAGY,KAAKE,WAAWE,UAASC,OAAOV,KAAKO,QAAQ;QACzF;AAEA,aAAKU,0BAAyB;AAC9B,aAAKmB,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOb;EACT;;;;EAKA,MAAMc,gBAAgBC,KAAgC;AACpD,SAAKF,kBAAiB;EACxB;EAEQR,eAAyB;AAC/B,WAAO;MACLW,OAAOC,MAAMxD,KAAK,KAAKJ,kBAAkB6D,OAAM,CAAA,EAAIC,IAAI,CAAC/C,YAAYA,QAAQU,IAAI;IAClF;EACF;EAEQ+B,oBAA0B;AAChC,eAAWb,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUoB,iBAAgB;IAC5B;EACF;EAEQ1B,4BAAkC;AACxC,eAAWM,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUI,eAAe,KAAKC,aAAY,CAAA;IAC5C;EACF;AACF;;;;;;AC7JA,OAAOgB,UAAU;AAEjB,SAASC,gBAAAA,qBAAoB;AAC7B,SAASC,oBAAoB;AAC7B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,UAAAA,eAAc;AACvB,SAASC,YAAAA,iBAAgB;AAGzB,SAASC,YAAAA,WAAUC,qBAAqB;;;;;;;;AAOjC,IAAMC,qBAAqB;AAElC,IAAMC,gBAAgBL,QAAOM,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;EACX,YAA6BC,YAAuB;SAAvBA,aAAAA;EAAwB;EAErD,MACMC,QAAQC,IAA+C;AAC3D,WAAO,KAAKC,SAASD,EAAAA;EACvB;;;;EAKA,MACME,IAAIF,IAAgBG,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKH,SAASD,EAAAA;AAErC,QAAI,CAACI,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAEC,SAAS,GAAGC,SAASH,SAASG,OAAM,IAAKJ;AAEjD,QAAIG,SAASC,SAASH,SAASG,QAAQ;AACrC,YAAM,IAAIF,MAAM,eAAA;IAClB;AAEA,QAAID,SAASI,UAAUjB,UAASkB,MAAMC,eAAe;AACnD,YAAMC,QAAO,KAAKC,aAAaZ,EAAAA;AAC/B,aAAOW,MAAKE,KAAKP,QAAQC,MAAAA;IAC3B,WAAWJ,QAAQG,WAAWQ,UAAaX,QAAQI,WAAWO,QAAW;AACvE,YAAM,IAAIT,MAAM,oBAAA;IAClB;AAEA,UAAMU,aAAaC,KAAKC,MAAMX,SAASF,SAASc,SAAS;AACzD,UAAMC,WAAWH,KAAKI,MAAMd,SAASC,UAAUH,SAASc,SAAS;AAEjE9B,IAAAA,WAAUgB,SAASiB,UAAU,wBAAA;;;;;;;;;AAC7BjC,IAAAA,WAAUgB,SAASiB,SAASd,SAAS,KAAKY,UAAU,2BAAA;;;;;;;;;AAEpD,UAAMG,UAAU9B,UAAS+B,MAAMnB,SAASiB,UAAUN,YAAYI,QAAAA,MAAcA,WAAWJ;AAEvF,QAAI,CAACO,SAAS;AACZ,YAAM,IAAIjB,MAAM,oBAAA;IAClB;AAEA,UAAMM,OAAO,KAAKC,aAAaZ,EAAAA;AAC/B,WAAOW,KAAKE,KAAKP,QAAQC,MAAAA;EAC3B;EAEA,MACMiB,OAA4B;AAQhC,UAAMC,QAAQ,IAAIC,KAAK,MAAM,KAAK5B,WAAW0B,KAAI,GAAIG,IAAI,CAACC,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWnB,QAAQc,OAAO;AACxB,YAAMzB,KAAKX,WAAU0C,KAAKpB,IAAAA,EAAMqB,aAAY;AAC5C,YAAMC,OAAO,MAAM,KAAKhC,SAASD,EAAAA;AACjC,UAAIiC,MAAM;AACRH,YAAII,KAAKD,IAAAA;MACX;IACF;AAEA,WAAOH;EACT;EAEA,MACMK,IAAIC,MAAqC;AAC7C,UAAMpC,KAAK,IAAIqC,WAAW,MAAMlD,aAAamD,OAAO,WAAWF,IAAAA,CAAAA;AAC/D,UAAMf,WAAW7B,UAAS+C,KAAKH,KAAK7B,SAASb,kBAAAA;AAE7C,UAAMuC,OAAiB;MACrBjC;MACAQ,OAAOjB,UAASkB,MAAMC;MACtBH,QAAQ6B,KAAK7B;MACbW,WAAWxB;MACX2B;MACAmB,SAAS,oBAAIC,KAAAA;MACbC,SAAS,oBAAID,KAAAA;IACf;AAEA,UAAM,KAAK7B,aAAaZ,EAAAA,EAAI2C,MAAM,GAAGlD,cAAc2C,IAAAA,CAAAA;AACnD,UAAM,KAAKQ,WAAW5C,IAAIiC,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACMY,SAASC,OAAqC;AAElD,QAAIb,OAAO,MAAM,KAAKhC,SAAS6C,MAAM9C,EAAE;AACvC,QAAI,CAACiC,MAAM;AACT7C,MAAAA,WAAU0D,MAAMC,aAAa,8BAAA;;;;;;;;;AAC7Bd,aAAO;QACLjC,IAAI8C,MAAM9C;QACVQ,OAAOjB,UAASkB,MAAMuC;QACtBzC,QAAQuC,MAAMC;QACd7B,WAAW4B,MAAM5B,aAAaxB;QAC9B8C,SAAS,oBAAIC,KAAAA;MACf;AACAR,WAAKZ,WAAW7B,UAASyD,MAAMhB,KAAK1B,SAAS0B,KAAKf,SAAS;IAC7D;AAEA,QAAI4B,MAAM5B,aAAa4B,MAAM5B,cAAce,KAAKf,WAAW;AACzD,YAAM,IAAIb,MAAM,oBAAA;IAClB;AAEAjB,IAAAA,WAAU6C,KAAKZ,UAAU,wBAAA;;;;;;;;;AACzBjC,IAAAA,WAAU0D,MAAMI,gBAAgBpC,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAakC,MAAM9C,EAAE,EAAE2C,MAAMG,MAAMI,aAAazD,cAAcqD,MAAMK,OAAO,CAAA;AAGtF3D,IAAAA,UAAS2C,IAAIF,KAAKZ,UAAUL,KAAKC,MAAM6B,MAAMI,cAAcjB,KAAKf,SAAS,GAAG,IAAA;AAG5E,QAAI1B,UAAS+B,MAAMU,KAAKZ,UAAU,GAAGY,KAAK1B,MAAM,IAAI0B,KAAKf,aAAae,KAAK1B,QAAQ;AACjF0B,WAAKzB,QAAQjB,UAASkB,MAAMC;IAC9B;AACAuB,SAAKS,UAAU,oBAAID,KAAAA;AAEnB,UAAM,KAAKG,WAAWE,MAAM9C,IAAIiC,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAcW,WAAW5C,IAAgBiC,MAA+B;AACtE,UAAMmB,UAAU3D,cAAcE,cAAc0D,OAAOpB,IAAAA,CAAAA;AACnD,UAAMG,OAAOkB,OAAOC,MAAMH,QAAQ7C,SAAS,CAAA;AAC3C6B,SAAKoB,cAAcJ,QAAQ7C,QAAQ,CAAA;AACnC6C,YAAQK,KAAKrB,MAAM,CAAA;AAGnB,UAAM,KAAKsB,aAAa1D,EAAAA,EAAI2C,MAAM,GAAGP,IAAAA;EACvC;EAEA,MAAcnC,SAASD,IAA+C;AACpE,UAAMW,OAAO,KAAK+C,aAAa1D,EAAAA;AAC/B,UAAM2D,QAAQ,MAAMhD,KAAKiD,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAMvB,OAAO,MAAMzB,KAAKE,KAAK,GAAG8C,IAAAA;AAChC,UAAME,YAAYzB,KAAK0B,aAAa,CAAA;AACpC,WAAOnE,cAAcoE,OAAO3B,KAAK4B,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAa1D,IAAgB;AACnC,WAAO,KAAKF,WAAWmE,gBAAgBhF,KAAKiF,KAAKzE,cAAcO,EAAAA,EAAImE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQvD,aAAaZ,IAAgB;AACnC,WAAO,KAAKF,WAAWmE,gBAAgBhF,KAAKiF,KAAKzE,cAAcO,EAAAA,EAAImE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;AACF;;;;;;;;;;;;;;;;",
6
- "names": ["DeferredTask", "sleep", "synchronized", "Context", "invariant", "log", "RpcClosedError", "schema", "RpcExtension", "BitField", "MIN_WANT_LIST_UPDATE_INTERVAL", "process", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "_params", "exposed", "BlobSyncService", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_ctx", "onError", "err", "catch", "_lastWantListUpdate", "_localWantList", "blobs", "_updateWantList", "Date", "now", "disposed", "list", "rpc", "want", "_currentUploads", "_upload", "blobChunks", "_pickBlobChunks", "blobChunk", "push", "warn", "finally", "reconcileUploads", "remoteWantList", "onOpen", "context", "onClose", "dispose", "onAbort", "getHandlers", "wantList", "data", "onPush", "updateWantList", "schedule", "amount", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "bitfield", "chunkSize", "requestBitfield", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "trackLeaks", "Trigger", "Mutex", "cancelWithContext", "Context", "invariant", "PublicKey", "log", "BlobMeta", "BitField", "ComplexMap", "BlobSync", "_params", "_ctx", "Context", "_mutex", "Mutex", "_downloadRequests", "ComplexMap", "key", "PublicKey", "from", "toHex", "_extensions", "Set", "open", "close", "dispose", "download", "ctx", "id", "log", "request", "executeSynchronized", "existingRequest", "get", "counter", "meta", "blobStore", "getMeta", "trigger", "Trigger", "want", "chunkSize", "bitfield", "Uint8Array", "BitField", "invert", "state", "BlobMeta", "State", "FULLY_PRESENT", "wake", "set", "_updateExtensionsWantList", "onDispose", "delete", "cancelWithContext", "wait", "createExtension", "extension", "BlobSyncExtension", "onOpen", "add", "updateWantList", "_getWantList", "onClose", "onAbort", "onPush", "blobChunk", "has", "setChunk", "invariant", "_reconcileUploads", "notifyBlobAdded", "_id", "blobs", "Array", "values", "map", "reconcileUploads", "path", "synchronized", "subtleCrypto", "invariant", "PublicKey", "schema", "BlobMeta", "BitField", "arrayToBuffer", "DEFAULT_CHUNK_SIZE", "BlobMetaCodec", "getCodecForType", "BlobStore", "_directory", "getMeta", "id", "_getMeta", "get", "options", "metadata", "Error", "offset", "length", "state", "State", "FULLY_PRESENT", "file", "_getDataFile", "read", "undefined", "beginChunk", "Math", "floor", "chunkSize", "endChunk", "ceil", "bitfield", "present", "count", "list", "files", "Set", "map", "f", "split", "res", "from", "asUint8Array", "meta", "push", "set", "data", "Uint8Array", "digest", "ones", "created", "Date", "updated", "write", "_writeMeta", "setChunk", "chunk", "totalLength", "PARTIALLY_PRESENT", "zeros", "chunkOffset", "payload", "encoded", "encode", "Buffer", "alloc", "writeUInt32LE", "copy", "_getMetaFile", "size", "stat", "protoSize", "readUInt32LE", "decode", "subarray", "getOrCreateFile", "join", "toString"]
4
+ "sourcesContent": ["//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { RpcClosedError } from '@dxos/protocols';\nimport { schema } from '@dxos/protocols/proto';\nimport { type BlobChunk, type BlobSyncService, type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onAbort: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = process.env.NODE_ENV === 'test' ? 5 : 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n override async onAbort(err?: Error | undefined): Promise<void> {\n log('abort');\n await this._ctx.dispose();\n await this._params.onAbort();\n await super.onAbort(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk): Promise<void> {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList): void {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads(): void {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { Mutex, Trigger, trackLeaks } from '@dxos/async';\nimport { Context, cancelWithContext } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _mutex = new Mutex();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open(): Promise<void> {}\n\n async close(): Promise<void> {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._mutex.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._mutex.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension(): BlobSyncExtension {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onAbort: async () => {\n log('extension aborted');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array): Promise<void> {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads(): void {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList(): void {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { schema } from '@dxos/protocols/proto';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n constructor(private readonly _directory: Directory) {}\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data as Uint8Array<ArrayBuffer>));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
5
+ "mappings": ";;;AAMA,SAASA,cAAcC,OAAOC,oBAAoB;AAClD,SAASC,eAAe;AACxB,SAASC,iBAAiB;AAC1B,SAASC,WAAW;AACpB,SAASC,sBAAsB;AAC/B,SAASC,cAAc;AAEvB,SAAgCC,oBAAoB;AACpD,SAASC,gBAAgB;;;;;;;;;;;;;;;;;;;;;AAYzB,IAAMC,gCAAgCC,QAAkC,IAAI;AAE5E,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCL,aAAAA;EAyErC,MAAeM,OAAOC,SAA0C;AAC9DV,QAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMS,OAAOC,OAAAA;AACnB,UAAM,KAAKC,QAAQF,OAAM;EAC3B;EAEA,MAAeG,QAAQC,KAAwC;AAC7Db,QAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKc,KAAKC,QAAO;AACvB,UAAM,KAAKJ,QAAQC,QAAO;AAC1B,UAAM,MAAMA,QAAQC,GAAAA;EACtB;EAEA,MAAeG,QAAQH,KAAwC;AAC7Db,QAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKc,KAAKC,QAAO;AACvB,UAAM,KAAKJ,QAAQK,QAAO;AAC1B,UAAM,MAAMA,QAAQH,GAAAA;EACtB;EAEA,MAAgBI,cAAsC;AACpD,WAAO;MACLC,iBAAiB;QACfC,MAAM,OAAOC,aAAAA;AACXpB,cAAI,eAAe;YAAEqB,gBAAgBD;UAAS,GAAA;;;;;;AAC9C,eAAKC,iBAAiBD;AACtB,eAAKE,iBAAgB;QACvB;QACAC,MAAM,OAAOC,SAAAA;AACXxB,cAAI,YAAY;YAAEwB;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAKb,QAAQc,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMD,KAAKC,MAAgC;AACzC,QAAI,KAAKV,KAAKY,UAAU;AACtB;IACF;AACA1B,QAAI,QAAQ;MAAEwB;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKG,IAAIT,gBAAgBK,KAAKC,IAAAA;EACtC;EAEAI,eAAeR,UAA0B;AACvC,QAAI,KAAKN,KAAKY,UAAU;AACtB;IACF;AACA,SAAKG,iBAAiBT;AACtB,SAAKU,gBAAgBC,SAAQ;EAC/B;EAEAT,mBAAyB;AACvB,QAAI,KAAKR,KAAKY,UAAU;AACtB;IACF;AACA,SAAKM,QAAQD,SAAQ;EACvB;EAEA,MAAcE,gBAAgBC,SAAS,GAAgC;AACrE,QAAI,KAAKpB,KAAKY,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKL,eAAec,SAAS,KAAKd,eAAec,OAAOC,WAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKhB,eAAec;MAAOG,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKhC,QAAQiC,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACA5C,gBAAU4C,KAAKI,UAAQ,QAAA;;;;;;;;;AACvBhD,gBAAU4C,KAAKK,WAAS,QAAA;;;;;;;;;AACxBjD,gBAAU4C,KAAKP,QAAM,QAAA;;;;;;;;;AAErB,UAAIM,OAAOM,aAAaN,OAAOM,cAAcL,KAAKK,WAAW;AAC3DhD,YAAIiD,KAAK,sBAAsB;UAAEP;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMO,kBAAkBR,OAAOK,YAAY3C,SAAS+C,KAAKR,KAAKP,SAASO,KAAKK,SAAS;AAErF,YAAMI,cAAchD,SAASiD,IAAIH,iBAAiBP,KAAKI,QAAQ;AAC/D,YAAMO,eAAelD,SAASmD,YAAYH,WAAAA,EAAad,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWgB,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAK9C,QAAQiC,UAAUc,IAAIhB,OAAOI,IAAI;UAC5Da,QAAQH,MAAMb,KAAKK;UACnBZ,QAAQG,KAAKqB,IAAIjB,KAAKK,WAAWL,KAAKP,SAASoB,MAAMb,KAAKK,SAAS;QACrE,CAAA;AACAP,eAAOlB,KAAK;UACVuB,IAAIJ,OAAOI;UACXe,aAAalB,KAAKP;UAClBY,WAAWL,KAAKK;UAChBc,aAAaN,MAAMb,KAAKK;UACxBe,SAASN;QACX,CAAA;AAEA,YAAIhB,OAAOL,UAAUF,QAAQ;AAC3B,iBAAOO;QACT;MACF;IACF;AAEA,WAAOA;EACT;EAnIA,YACmB9B,SACjB;AACA,UAAM;MACJqD,SAAS;QACP9C,iBAAiBhB,OAAO+D,WAAW,6CAAA;MACrC;MACAC,WAAW;QACThD,iBAAiBhB,OAAO+D,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA,GAAA,iBAAA,MAAA,WAAA,MAAA,GArEF,iBAAA,MAAiBvD,QAAjB,MAAA,GAEA,iBAAA,MAAQwD,uBAAR,MAAA,GACA,iBAAA,MAAQzC,kBAAR,MAAA,GAEA,iBAAA,MAAiBC,mBAAjB,MAAA,GAcA,iBAAA,MAAQyC,mBAAR,MAAA,GAEA,iBAAA,MAAiBvC,WAAjB,MAAA;;;IAgCA,iBAAA,MAAOX,kBAAP,MAAA,GAAA,KAGmBV,UAAAA,SAAAA,KAxDFG,OAAO,IAAIhB,QAAQ;MAAE0E,SAAS,CAAC3D,QAAQb,IAAIyE,MAAM5D,KAAAA,QAAAA;;;;;;IAAK,GAAA;;;aAE/DyD,sBAAsB,GAAA,KACtBzC,iBAA2B;MAAEM,OAAO,CAAA;IAAG,GAAA,KAE9BL,kBAAkB,IAAInC,aAAa,KAAKmB,MAAM,YAAA;AAE7D,UAAI,KAAKwD,sBAAsBjE,gCAAgCqE,KAAKC,IAAG,GAAI;AACzE,cAAM/E,MAAM,KAAK0E,sBAAsBjE,gCAAgCqE,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAK7D,KAAKY,UAAU;AACtB;QACF;MACF;AAEA1B,UAAI,QAAQ;QAAE4E,MAAM,KAAK/C;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKF,IAAIT,gBAAgBC,KAAK,KAAKU,cAAc;AACvD,WAAKyC,sBAAsBI,KAAKC,IAAG;IACrC,CAAA,GAAA,KAEQJ,kBAAkB,GAAA,KAETvC,UAAU,IAAIrC,aAAa,KAAKmB,MAAM,YAAA;AACrD,UAAI,KAAKyD,mBAAmBhE,wBAAwB;AAClD;MACF;AACA,YAAMsE,aAAa,MAAM,KAAK5C,gBAAgB1B,yBAAyB,KAAKgE,eAAe;AAC3F,UAAI,CAACM,YAAY;AACf;MACF;AACA,iBAAWC,aAAaD,YAAY;AAClC,YAAI,KAAK/D,KAAKY,UAAU;AACtB;QACF;AAEA,aAAK6C;AAEL,aAAKhD,KAAKuD,SAAAA,EACPL,MAAM,CAAC5D,QAAAA;AACN,cAAIA,eAAeZ,gBAAgB;AACjC;UACF;AACAD,cAAIiD,KAAK,eAAe;YAAEpC;UAAI,GAAA;;;;;;QAChC,CAAA,EACCkE,QAAQ,MAAA;AACP,eAAKR;AACL,eAAKjD,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA,GAAA,KAKOD,iBAA2B;MAAEc,OAAO,CAAA;IAAG;EAiB9C;AAqHF;;;;;;ACzNA,SAAS6C,OAAOC,SAASC,kBAAkB;AAC3C,SAASC,WAAAA,UAASC,yBAAyB;AAC3C,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,gBAAgB;AAEzB,SAASC,YAAAA,WAAUC,kBAAkB;;;;;;;;;;;;;;;;;;;;;AAiB9B,IAAMC,WAAN,MAAMA;EAYX,MAAMC,OAAsB;EAAC;EAE7B,MAAMC,QAAuB;AAC3B,UAAM,KAAKC,KAAKC,QAAO;EACzB;;;;;;EAOA,MAAMC,SAASC,KAAcC,IAA+B;AAC1DC,IAAAA,KAAI,YAAY;MAAED;IAAG,GAAA;;;;;;AACrB,UAAME,UAAU,MAAM,KAAKC,OAAOC,oBAAoB,YAAA;AACpD,YAAMC,kBAAkB,KAAKC,kBAAkBC,IAAIP,EAAAA;AAEnD,UAAIK,iBAAiB;AACnBA,wBAAgBG;AAChB,eAAOH;MACT;AAEA,YAAMI,OAAO,MAAM,KAAKC,QAAQC,UAAUC,QAAQZ,EAAAA;AAClD,YAAME,WAA2B;QAC/BW,SAAS,IAAIC,QAAAA;QACbN,SAAS;QACTO,MAAM;UACJf;UACAgB,WAAWP,MAAMO;UACjBC,UAAUR,MAAMQ,YAAYC,WAAWC,KAAKC,UAASC,OAAOZ,KAAKQ,QAAQ,CAAA;QAC3E;MACF;AAGA,UAAIR,MAAMa,UAAUC,SAASC,MAAMC,eAAe;AAChDvB,QAAAA,SAAQW,QAAQa,KAAI;MACtB,OAAO;AACL,aAAKpB,kBAAkBqB,IAAI3B,IAAIE,QAAAA;AAC/B,aAAK0B,0BAAyB;MAChC;AAEA,aAAO1B;IACT,CAAA;AAEAH,SAAK8B,UAAU,MACb,KAAK1B,OAAOC,oBAAoB,YAAA;AAE9B,YAAMF,WAAU,KAAKI,kBAAkBC,IAAIP,EAAAA;AAC3C,UAAI,CAACE,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQM,YAAY,GAAG;AAC3B,aAAKF,kBAAkBwB,OAAO9B,EAAAA;MAChC;AACA,WAAK4B,0BAAyB;IAChC,CAAA,CAAA;AAGF,WAAO7B,MAAMgC,kBAAkBhC,KAAKG,QAAQW,QAAQmB,KAAI,CAAA,IAAM9B,QAAQW,QAAQmB,KAAI;EACpF;EAEAC,kBAAqC;AACnC,UAAMC,YAAY,IAAIC,kBAAkB;MACtCxB,WAAW,KAAKD,QAAQC;MACxByB,QAAQ,YAAA;AACNnC,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKoC,YAAYC,IAAIJ,SAAAA;AACrBA,kBAAUK,eAAe,KAAKC,aAAY,CAAA;MAC5C;MACAC,SAAS,YAAA;AACPxC,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKoC,YAAYP,OAAOI,SAAAA;MAC1B;MACAQ,SAAS,YAAA;AACPzC,QAAAA,KAAI,qBAAA,QAAA;;;;;;AACJ,aAAKoC,YAAYP,OAAOI,SAAAA;MAC1B;MACAS,QAAQ,OAAOC,cAAAA;AACb,YAAI,CAAC,KAAKtC,kBAAkBuC,IAAID,UAAU5C,EAAE,GAAG;AAC7C;QACF;AACAC,QAAAA,KAAI,YAAY;UAAE2C;QAAU,GAAA;;;;;;AAC5B,cAAMnC,OAAO,MAAM,KAAKC,QAAQC,UAAUmC,SAASF,SAAAA;AACnD,YAAInC,KAAKa,UAAUC,SAASC,MAAMC,eAAe;AAC/C,eAAKnB,kBAAkBC,IAAIqC,UAAU5C,EAAE,GAAGa,QAAQa,KAAAA;AAClD,eAAKpB,kBAAkBwB,OAAOc,UAAU5C,EAAE;QAC5C,OAAO;AACL+C,UAAAA,WAAUtC,KAAKQ,UAAQ,QAAA;;;;;;;;;AACvB,eAAKX,kBAAkBC,IAAIqC,UAAU5C,EAAE,EAAGe,KAAKE,WAAWG,UAASC,OAAOZ,KAAKQ,QAAQ;QACzF;AAEA,aAAKW,0BAAyB;AAC9B,aAAKoB,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOd;EACT;;;;EAKA,MAAMe,gBAAgBC,KAAgC;AACpD,SAAKF,kBAAiB;EACxB;EAEQR,eAAyB;AAC/B,WAAO;MACLW,OAAOC,MAAMjC,KAAK,KAAKb,kBAAkB+C,OAAM,CAAA,EAAIC,IAAI,CAACpD,YAAYA,QAAQa,IAAI;IAClF;EACF;EAEQiC,oBAA0B;AAChC,eAAWd,aAAa,KAAKG,aAAa;AACxCH,gBAAUqB,iBAAgB;IAC5B;EACF;EAEQ3B,4BAAkC;AACxC,eAAWM,aAAa,KAAKG,aAAa;AACxCH,gBAAUK,eAAe,KAAKC,aAAY,CAAA;IAC5C;EACF;EA1HA,YAA6B9B,SAAyB;;AATtD,IAAA8C,kBAAA,MAAiB5D,QAAjB,MAAA;AACA,IAAA4D,kBAAA,MAAiBrD,UAAjB,MAAA;AAEA,IAAAqD,kBAAA,MAAiBlD,qBAAjB,MAAA;AAIA,IAAAkD,kBAAA,MAAiBnB,eAAjB,MAAA;SAE6B3B,UAAAA;SATZd,OAAO,IAAI6D,SAAAA,QAAAA;;;;SACXtD,SAAS,IAAIuD,MAAAA;SAEbpD,oBAAoB,IAAIqD,WAAwC,CAACC,QAChFC,UAAU1C,KAAKyC,GAAAA,EAAKE,MAAK,CAAA;SAGVzB,cAAc,oBAAI0B,IAAAA;EAEoB;AA2HzD;;;;;;AC7JA,OAAOC,UAAU;AAEjB,SAASC,gBAAAA,qBAAoB;AAC7B,SAASC,oBAAoB;AAC7B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,UAAAA,eAAc;AACvB,SAASC,YAAAA,iBAAgB;AAGzB,SAASC,YAAAA,WAAUC,qBAAqB;;;;;;;;;;;;;;;;;;;;;AAOjC,IAAMC,qBAAqB;AAElC,IAAMC,gBAAgBL,QAAOM,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;EAGX,MACMC,QAAQC,IAA+C;AAC3D,WAAO,KAAKC,SAASD,EAAAA;EACvB;;;;EAKA,MACME,IAAIF,IAAgBG,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKH,SAASD,EAAAA;AAErC,QAAI,CAACI,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAEC,SAAS,GAAGC,SAASH,SAASG,OAAM,IAAKJ;AAEjD,QAAIG,SAASC,SAASH,SAASG,QAAQ;AACrC,YAAM,IAAIF,MAAM,eAAA;IAClB;AAEA,QAAID,SAASI,UAAUhB,UAASiB,MAAMC,eAAe;AACnD,YAAMC,QAAO,KAAKC,aAAaZ,EAAAA;AAC/B,aAAOW,MAAKE,KAAKP,QAAQC,MAAAA;IAC3B,WAAWJ,QAAQG,WAAWQ,UAAaX,QAAQI,WAAWO,QAAW;AACvE,YAAM,IAAIT,MAAM,oBAAA;IAClB;AAEA,UAAMU,aAAaC,KAAKC,MAAMX,SAASF,SAASc,SAAS;AACzD,UAAMC,WAAWH,KAAKI,MAAMd,SAASC,UAAUH,SAASc,SAAS;AAEjE7B,IAAAA,WAAUe,SAASiB,UAAU,wBAAA;;;;;;;;;AAC7BhC,IAAAA,WAAUe,SAASiB,SAASd,SAAS,KAAKY,UAAU,2BAAA;;;;;;;;;AAEpD,UAAMG,UAAU7B,UAAS8B,MAAMnB,SAASiB,UAAUN,YAAYI,QAAAA,MAAcA,WAAWJ;AAEvF,QAAI,CAACO,SAAS;AACZ,YAAM,IAAIjB,MAAM,oBAAA;IAClB;AAEA,UAAMM,OAAO,KAAKC,aAAaZ,EAAAA;AAC/B,WAAOW,KAAKE,KAAKP,QAAQC,MAAAA;EAC3B;EAEA,MACMiB,OAA4B;AAQhC,UAAMC,QAAQ,IAAIC,KAAK,MAAM,KAAKC,WAAWH,KAAI,GAAII,IAAI,CAACC,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWpB,QAAQc,OAAO;AACxB,YAAMzB,KAAKV,WAAU0C,KAAKrB,IAAAA,EAAMsB,aAAY;AAC5C,YAAMC,OAAO,MAAM,KAAKjC,SAASD,EAAAA;AACjC,UAAIkC,MAAM;AACRH,YAAII,KAAKD,IAAAA;MACX;IACF;AAEA,WAAOH;EACT;EAEA,MACMK,IAAIC,MAAqC;AAC7C,UAAMrC,KAAK,IAAIsC,WAAW,MAAMlD,aAAamD,OAAO,WAAWF,IAAAA,CAAAA;AAC/D,UAAMhB,WAAW5B,UAAS+C,KAAKH,KAAK9B,SAASZ,kBAAAA;AAE7C,UAAMuC,OAAiB;MACrBlC;MACAQ,OAAOhB,UAASiB,MAAMC;MACtBH,QAAQ8B,KAAK9B;MACbW,WAAWvB;MACX0B;MACAoB,SAAS,oBAAIC,KAAAA;MACbC,SAAS,oBAAID,KAAAA;IACf;AAEA,UAAM,KAAK9B,aAAaZ,EAAAA,EAAI4C,MAAM,GAAGlD,cAAc2C,IAAAA,CAAAA;AACnD,UAAM,KAAKQ,WAAW7C,IAAIkC,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACMY,SAASC,OAAqC;AAElD,QAAIb,OAAO,MAAM,KAAKjC,SAAS8C,MAAM/C,EAAE;AACvC,QAAI,CAACkC,MAAM;AACT7C,MAAAA,WAAU0D,MAAMC,aAAa,8BAAA;;;;;;;;;AAC7Bd,aAAO;QACLlC,IAAI+C,MAAM/C;QACVQ,OAAOhB,UAASiB,MAAMwC;QACtB1C,QAAQwC,MAAMC;QACd9B,WAAW6B,MAAM7B,aAAavB;QAC9B8C,SAAS,oBAAIC,KAAAA;MACf;AACAR,WAAKb,WAAW5B,UAASyD,MAAMhB,KAAK3B,SAAS2B,KAAKhB,SAAS;IAC7D;AAEA,QAAI6B,MAAM7B,aAAa6B,MAAM7B,cAAcgB,KAAKhB,WAAW;AACzD,YAAM,IAAIb,MAAM,oBAAA;IAClB;AAEAhB,IAAAA,WAAU6C,KAAKb,UAAU,wBAAA;;;;;;;;;AACzBhC,IAAAA,WAAU0D,MAAMI,gBAAgBrC,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAamC,MAAM/C,EAAE,EAAE4C,MAAMG,MAAMI,aAAazD,cAAcqD,MAAMK,OAAO,CAAA;AAGtF3D,IAAAA,UAAS2C,IAAIF,KAAKb,UAAUL,KAAKC,MAAM8B,MAAMI,cAAcjB,KAAKhB,SAAS,GAAG,IAAA;AAG5E,QAAIzB,UAAS8B,MAAMW,KAAKb,UAAU,GAAGa,KAAK3B,MAAM,IAAI2B,KAAKhB,aAAagB,KAAK3B,QAAQ;AACjF2B,WAAK1B,QAAQhB,UAASiB,MAAMC;IAC9B;AACAwB,SAAKS,UAAU,oBAAID,KAAAA;AAEnB,UAAM,KAAKG,WAAWE,MAAM/C,IAAIkC,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAcW,WAAW7C,IAAgBkC,MAA+B;AACtE,UAAMmB,UAAU3D,cAAcE,cAAc0D,OAAOpB,IAAAA,CAAAA;AACnD,UAAMG,OAAOkB,OAAOC,MAAMH,QAAQ9C,SAAS,CAAA;AAC3C8B,SAAKoB,cAAcJ,QAAQ9C,QAAQ,CAAA;AACnC8C,YAAQK,KAAKrB,MAAM,CAAA;AAGnB,UAAM,KAAKsB,aAAa3D,EAAAA,EAAI4C,MAAM,GAAGP,IAAAA;EACvC;EAEA,MAAcpC,SAASD,IAA+C;AACpE,UAAMW,OAAO,KAAKgD,aAAa3D,EAAAA;AAC/B,UAAM4D,QAAQ,MAAMjD,KAAKkD,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAMvB,OAAO,MAAM1B,KAAKE,KAAK,GAAG+C,IAAAA;AAChC,UAAME,YAAYzB,KAAK0B,aAAa,CAAA;AACpC,WAAOnE,cAAcoE,OAAO3B,KAAK4B,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAa3D,IAAgB;AACnC,WAAO,KAAK2B,WAAWuC,gBAAgBhF,KAAKiF,KAAKzE,cAAcM,EAAAA,EAAIoE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQxD,aAAaZ,IAAgB;AACnC,WAAO,KAAK2B,WAAWuC,gBAAgBhF,KAAKiF,KAAKzE,cAAcM,EAAAA,EAAIoE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EA/JA,YAA6BzC,YAAuB;;SAAvBA,aAAAA;EAAwB;AAgKvD;;;;;;;;;;;;;;;;",
6
+ "names": ["DeferredTask", "sleep", "synchronized", "Context", "invariant", "log", "RpcClosedError", "schema", "RpcExtension", "BitField", "MIN_WANT_LIST_UPDATE_INTERVAL", "process", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "onOpen", "context", "_params", "onClose", "err", "_ctx", "dispose", "onAbort", "getHandlers", "BlobSyncService", "want", "wantList", "remoteWantList", "reconcileUploads", "push", "data", "onPush", "disposed", "rpc", "updateWantList", "_localWantList", "_updateWantList", "schedule", "_upload", "_pickBlobChunks", "amount", "blobs", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "bitfield", "chunkSize", "warn", "requestBitfield", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "exposed", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_lastWantListUpdate", "_currentUploads", "onError", "catch", "Date", "now", "list", "blobChunks", "blobChunk", "finally", "Mutex", "Trigger", "trackLeaks", "Context", "cancelWithContext", "invariant", "PublicKey", "log", "BlobMeta", "BitField", "ComplexMap", "BlobSync", "open", "close", "_ctx", "dispose", "download", "ctx", "id", "log", "request", "_mutex", "executeSynchronized", "existingRequest", "_downloadRequests", "get", "counter", "meta", "_params", "blobStore", "getMeta", "trigger", "Trigger", "want", "chunkSize", "bitfield", "Uint8Array", "from", "BitField", "invert", "state", "BlobMeta", "State", "FULLY_PRESENT", "wake", "set", "_updateExtensionsWantList", "onDispose", "delete", "cancelWithContext", "wait", "createExtension", "extension", "BlobSyncExtension", "onOpen", "_extensions", "add", "updateWantList", "_getWantList", "onClose", "onAbort", "onPush", "blobChunk", "has", "setChunk", "invariant", "_reconcileUploads", "notifyBlobAdded", "_id", "blobs", "Array", "values", "map", "reconcileUploads", "_define_property", "Context", "Mutex", "ComplexMap", "key", "PublicKey", "toHex", "Set", "path", "synchronized", "subtleCrypto", "invariant", "PublicKey", "schema", "BlobMeta", "BitField", "arrayToBuffer", "DEFAULT_CHUNK_SIZE", "BlobMetaCodec", "getCodecForType", "BlobStore", "getMeta", "id", "_getMeta", "get", "options", "metadata", "Error", "offset", "length", "state", "State", "FULLY_PRESENT", "file", "_getDataFile", "read", "undefined", "beginChunk", "Math", "floor", "chunkSize", "endChunk", "ceil", "bitfield", "present", "count", "list", "files", "Set", "_directory", "map", "f", "split", "res", "from", "asUint8Array", "meta", "push", "set", "data", "Uint8Array", "digest", "ones", "created", "Date", "updated", "write", "_writeMeta", "setChunk", "chunk", "totalLength", "PARTIALLY_PRESENT", "zeros", "chunkOffset", "payload", "encoded", "encode", "Buffer", "alloc", "writeUInt32LE", "copy", "_getMetaFile", "size", "stat", "protoSize", "readUInt32LE", "decode", "subarray", "getOrCreateFile", "join", "toString"]
7
7
  }
@@ -1 +1 @@
1
- {"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":25326,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytes":18525,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytes":22187,"imports":[{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytes":703,"imports":[{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts","kind":"import-statement","original":"./blob-sync"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts","kind":"import-statement","original":"./blob-store"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-object-sync/dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":30394},"packages/core/mesh/teleport-extension-object-sync/dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["BlobStore","BlobSync","BlobSyncExtension","DEFAULT_CHUNK_SIZE"],"entryPoint":"packages/core/mesh/teleport-extension-object-sync/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytesInOutput":7387},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytesInOutput":0},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytesInOutput":5434},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytesInOutput":6531}},"bytes":19723}}}
1
+ {"inputs":{"src/blob-sync-extension.ts":{"bytes":25935,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"src/blob-sync.ts":{"bytes":19069,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"}],"format":"esm"},"src/blob-store.ts":{"bytes":22575,"imports":[{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"src/index.ts":{"bytes":690,"imports":[{"path":"src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"},{"path":"src/blob-sync.ts","kind":"import-statement","original":"./blob-sync"},{"path":"src/blob-store.ts","kind":"import-statement","original":"./blob-store"}],"format":"esm"}},"outputs":{"dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":30847},"dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["BlobStore","BlobSync","BlobSyncExtension","DEFAULT_CHUNK_SIZE"],"entryPoint":"src/index.ts","inputs":{"src/blob-sync-extension.ts":{"bytesInOutput":8057},"src/index.ts":{"bytesInOutput":0},"src/blob-sync.ts":{"bytesInOutput":5920},"src/blob-store.ts":{"bytesInOutput":6818}},"bytes":21016}}}