@dxos/teleport-extension-object-sync 0.1.56-main.83e9b8a → 0.1.56-main.a25bb93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/index.mjs +52 -20
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/node/index.cjs +57 -25
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/types/src/blob-sync-extension.d.ts.map +1 -1
- package/dist/types/src/blob-sync.d.ts.map +1 -1
- package/package.json +14 -14
- package/src/blob-store.ts +1 -1
- package/src/blob-sync-extension.ts +1 -2
- package/src/blob-sync.ts +1 -2
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import "@dxos/node-std/globals";
|
|
2
2
|
|
|
3
3
|
// packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts
|
|
4
|
-
import invariant from "tiny-invariant";
|
|
5
4
|
import { DeferredTask, sleep, synchronized } from "@dxos/async";
|
|
6
5
|
import { Context } from "@dxos/context";
|
|
6
|
+
import { invariant } from "@dxos/invariant";
|
|
7
7
|
import { log } from "@dxos/log";
|
|
8
8
|
import { RpcClosedError, schema } from "@dxos/protocols";
|
|
9
9
|
import { RpcExtension } from "@dxos/teleport";
|
|
@@ -39,7 +39,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
39
39
|
this._ctx = new Context({
|
|
40
40
|
onError: (err) => log.catch(err, void 0, {
|
|
41
41
|
F: __dxlog_file,
|
|
42
|
-
L:
|
|
42
|
+
L: 33,
|
|
43
43
|
S: this,
|
|
44
44
|
C: (f, a) => f(...a)
|
|
45
45
|
})
|
|
@@ -59,7 +59,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
59
59
|
list: this._localWantList
|
|
60
60
|
}, {
|
|
61
61
|
F: __dxlog_file,
|
|
62
|
-
L:
|
|
62
|
+
L: 47,
|
|
63
63
|
S: this,
|
|
64
64
|
C: (f, a) => f(...a)
|
|
65
65
|
});
|
|
@@ -88,7 +88,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
88
88
|
err
|
|
89
89
|
}, {
|
|
90
90
|
F: __dxlog_file,
|
|
91
|
-
L:
|
|
91
|
+
L: 74,
|
|
92
92
|
S: this,
|
|
93
93
|
C: (f, a) => f(...a)
|
|
94
94
|
});
|
|
@@ -105,7 +105,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
105
105
|
async onOpen(context) {
|
|
106
106
|
log("open", void 0, {
|
|
107
107
|
F: __dxlog_file,
|
|
108
|
-
L:
|
|
108
|
+
L: 106,
|
|
109
109
|
S: this,
|
|
110
110
|
C: (f, a) => f(...a)
|
|
111
111
|
});
|
|
@@ -115,7 +115,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
115
115
|
async onClose(err) {
|
|
116
116
|
log("close", void 0, {
|
|
117
117
|
F: __dxlog_file,
|
|
118
|
-
L:
|
|
118
|
+
L: 112,
|
|
119
119
|
S: this,
|
|
120
120
|
C: (f, a) => f(...a)
|
|
121
121
|
});
|
|
@@ -131,7 +131,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
131
131
|
remoteWantList: wantList
|
|
132
132
|
}, {
|
|
133
133
|
F: __dxlog_file,
|
|
134
|
-
L:
|
|
134
|
+
L: 122,
|
|
135
135
|
S: this,
|
|
136
136
|
C: (f, a) => f(...a)
|
|
137
137
|
});
|
|
@@ -143,7 +143,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
143
143
|
data
|
|
144
144
|
}, {
|
|
145
145
|
F: __dxlog_file,
|
|
146
|
-
L:
|
|
146
|
+
L: 127,
|
|
147
147
|
S: this,
|
|
148
148
|
C: (f, a) => f(...a)
|
|
149
149
|
});
|
|
@@ -160,7 +160,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
160
160
|
data
|
|
161
161
|
}, {
|
|
162
162
|
F: __dxlog_file,
|
|
163
|
-
L:
|
|
163
|
+
L: 139,
|
|
164
164
|
S: this,
|
|
165
165
|
C: (f, a) => f(...a)
|
|
166
166
|
});
|
|
@@ -196,16 +196,40 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
196
196
|
if (!meta) {
|
|
197
197
|
continue;
|
|
198
198
|
}
|
|
199
|
-
invariant(meta.bitfield
|
|
200
|
-
|
|
201
|
-
|
|
199
|
+
invariant(meta.bitfield, void 0, {
|
|
200
|
+
F: __dxlog_file,
|
|
201
|
+
L: 178,
|
|
202
|
+
S: this,
|
|
203
|
+
A: [
|
|
204
|
+
"meta.bitfield",
|
|
205
|
+
""
|
|
206
|
+
]
|
|
207
|
+
});
|
|
208
|
+
invariant(meta.chunkSize, void 0, {
|
|
209
|
+
F: __dxlog_file,
|
|
210
|
+
L: 179,
|
|
211
|
+
S: this,
|
|
212
|
+
A: [
|
|
213
|
+
"meta.chunkSize",
|
|
214
|
+
""
|
|
215
|
+
]
|
|
216
|
+
});
|
|
217
|
+
invariant(meta.length, void 0, {
|
|
218
|
+
F: __dxlog_file,
|
|
219
|
+
L: 180,
|
|
220
|
+
S: this,
|
|
221
|
+
A: [
|
|
222
|
+
"meta.length",
|
|
223
|
+
""
|
|
224
|
+
]
|
|
225
|
+
});
|
|
202
226
|
if (header.chunkSize && header.chunkSize !== meta.chunkSize) {
|
|
203
227
|
log.warn("Invalid chunk size", {
|
|
204
228
|
header,
|
|
205
229
|
meta
|
|
206
230
|
}, {
|
|
207
231
|
F: __dxlog_file,
|
|
208
|
-
L:
|
|
232
|
+
L: 183,
|
|
209
233
|
S: this,
|
|
210
234
|
C: (f, a) => f(...a)
|
|
211
235
|
});
|
|
@@ -239,9 +263,9 @@ _ts_decorate([
|
|
|
239
263
|
], BlobSyncExtension.prototype, "push", null);
|
|
240
264
|
|
|
241
265
|
// packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts
|
|
242
|
-
import invariant2 from "tiny-invariant";
|
|
243
266
|
import { trackLeaks, Trigger, Lock } from "@dxos/async";
|
|
244
267
|
import { cancelWithContext, Context as Context2 } from "@dxos/context";
|
|
268
|
+
import { invariant as invariant2 } from "@dxos/invariant";
|
|
245
269
|
import { PublicKey } from "@dxos/keys";
|
|
246
270
|
import { log as log2 } from "@dxos/log";
|
|
247
271
|
import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
@@ -280,7 +304,7 @@ var BlobSync = class BlobSync2 {
|
|
|
280
304
|
id
|
|
281
305
|
}, {
|
|
282
306
|
F: __dxlog_file2,
|
|
283
|
-
L:
|
|
307
|
+
L: 53,
|
|
284
308
|
S: this,
|
|
285
309
|
C: (f, a) => f(...a)
|
|
286
310
|
});
|
|
@@ -326,7 +350,7 @@ var BlobSync = class BlobSync2 {
|
|
|
326
350
|
onOpen: async () => {
|
|
327
351
|
log2("extension opened", void 0, {
|
|
328
352
|
F: __dxlog_file2,
|
|
329
|
-
L:
|
|
353
|
+
L: 105,
|
|
330
354
|
S: this,
|
|
331
355
|
C: (f, a) => f(...a)
|
|
332
356
|
});
|
|
@@ -336,7 +360,7 @@ var BlobSync = class BlobSync2 {
|
|
|
336
360
|
onClose: async () => {
|
|
337
361
|
log2("extension closed", void 0, {
|
|
338
362
|
F: __dxlog_file2,
|
|
339
|
-
L:
|
|
363
|
+
L: 110,
|
|
340
364
|
S: this,
|
|
341
365
|
C: (f, a) => f(...a)
|
|
342
366
|
});
|
|
@@ -351,7 +375,7 @@ var BlobSync = class BlobSync2 {
|
|
|
351
375
|
blobChunk
|
|
352
376
|
}, {
|
|
353
377
|
F: __dxlog_file2,
|
|
354
|
-
L:
|
|
378
|
+
L: 117,
|
|
355
379
|
S: this,
|
|
356
380
|
C: (f, a) => f(...a)
|
|
357
381
|
});
|
|
@@ -360,7 +384,15 @@ var BlobSync = class BlobSync2 {
|
|
|
360
384
|
(_a = this._downloadRequests.get(blobChunk.id)) == null ? void 0 : _a.trigger.wake();
|
|
361
385
|
this._downloadRequests.delete(blobChunk.id);
|
|
362
386
|
} else {
|
|
363
|
-
invariant2(meta.bitfield
|
|
387
|
+
invariant2(meta.bitfield, void 0, {
|
|
388
|
+
F: __dxlog_file2,
|
|
389
|
+
L: 123,
|
|
390
|
+
S: this,
|
|
391
|
+
A: [
|
|
392
|
+
"meta.bitfield",
|
|
393
|
+
""
|
|
394
|
+
]
|
|
395
|
+
});
|
|
364
396
|
this._downloadRequests.get(blobChunk.id).want.bitfield = BitField2.invert(meta.bitfield);
|
|
365
397
|
}
|
|
366
398
|
this._updateExtensionsWantList();
|
|
@@ -399,8 +431,8 @@ BlobSync = _ts_decorate2([
|
|
|
399
431
|
import path from "@dxos/node-std/path";
|
|
400
432
|
import { synchronized as synchronized2 } from "@dxos/async";
|
|
401
433
|
import { subtleCrypto } from "@dxos/crypto";
|
|
434
|
+
import { invariant as invariant3 } from "@dxos/invariant";
|
|
402
435
|
import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
403
|
-
import { invariant as invariant3 } from "@dxos/log";
|
|
404
436
|
import { schema as schema2 } from "@dxos/protocols";
|
|
405
437
|
import { BlobMeta as BlobMeta2 } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
406
438
|
import { BitField as BitField3, arrayToBuffer } from "@dxos/util";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/blob-sync-extension.ts", "../../../src/blob-sync.ts", "../../../src/blob-store.ts"],
|
|
4
|
-
"sourcesContent": ["//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport invariant from 'tiny-invariant';\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { log } from '@dxos/log';\nimport { RpcClosedError, schema } from '@dxos/protocols';\nimport { BlobChunk, BlobSyncService, WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk) {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList) {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads() {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport invariant from 'tiny-invariant';\n\nimport { trackLeaks, Trigger, Lock } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _lock = new Lock();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open() {}\n\n async close() {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._lock.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._lock.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension() {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array) {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads() {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList() {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { PublicKey } from '@dxos/keys';\nimport { invariant } from '@dxos/log';\nimport { schema } from '@dxos/protocols';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n // prettier-ignore\n constructor(\n private readonly _directory: Directory\n ) { }\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
|
|
5
|
-
"mappings": ";;;AAMA,OAAOA,eAAe;AAEtB,SAASC,cAAcC,OAAOC,oBAAoB;AAClD,SAASC,eAAe;AACxB,SAASC,WAAW;AACpB,SAASC,gBAAgBC,cAAc;AAEvC,SAA2BC,oBAAoB;AAC/C,SAASC,gBAAgB;;;;;;;;;;;;AAWzB,IAAMC,gCAAgC;AAEtC,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCJ,aAAAA;EAwDrCK,YACmBC,SACjB;AACA,UAAM;MACJC,SAAS;QACPC,iBAAiBT,OAAOU,WAAW,6CAAA;MACrC;MACAC,WAAW;QACTF,iBAAiBT,OAAOU,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA;mBAbiBP;SAxDFQ,OAAO,IAAIlB,QAAQ;MAAEmB,SAAS,CAACC,QAAQnB,IAAIoB,MAAMD,KAAAA,QAAAA;;;;;;IAAK,CAAA;SAE/DE,sBAAsB;SACtBC,iBAA2B;MAAEC,OAAO,CAAA;IAAG;SAE9BC,kBAAkB,IAAI5B,aAAa,KAAKqB,MAAM,YAAA;AAE7D,UAAI,KAAKI,sBAAsBhB,gCAAgCoB,KAAKC,IAAG,GAAI;AACzE,cAAM7B,MAAM,KAAKwB,sBAAsBhB,gCAAgCoB,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAKT,KAAKU,UAAU;AACtB;QACF;MACF;AAEA3B,UAAI,QAAQ;QAAE4B,MAAM,KAAKN;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKO,IAAIlB,gBAAgBmB,KAAK,KAAKR,cAAc;AACvD,WAAKD,sBAAsBI,KAAKC,IAAG;IACrC,CAAA;SAEQK,kBAAkB;SAETC,UAAU,IAAIpC,aAAa,KAAKqB,MAAM,YAAA;AACrD,UAAI,KAAKc,mBAAmBzB,wBAAwB;AAClD;MACF;AACA,YAAM2B,aAAa,MAAM,KAAKC,gBAAgB5B,yBAAyB,KAAKyB,eAAe;AAC3F,UAAI,CAACE,YAAY;AACf;MACF;AACA,iBAAWE,aAAaF,YAAY;AAClC,YAAI,KAAKhB,KAAKU,UAAU;AACtB;QACF;AAEA,aAAKI;AAEL,aAAKK,KAAKD,SAAAA,EACPf,MAAM,CAACD,QAAAA;AACN,cAAIA,eAAelB,gBAAgB;AACjC;UACF;AACAD,cAAIqC,KAAK,eAAe;YAAElB;UAAI,GAAA;;;;;;QAChC,CAAA,EACCmB,QAAQ,MAAA;AACP,eAAKP;AACL,eAAKQ,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA;SAKOC,iBAA2B;MAAEjB,OAAO,CAAA;IAAG;EAiB9C;EAEA,MAAekB,OAAOC,SAA0C;AAC9D1C,QAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMyC,OAAOC,OAAAA;AACnB,UAAM,KAAKjC,QAAQgC,OAAM;EAC3B;EAEA,MAAeE,QAAQxB,KAAwC;AAC7DnB,QAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKiB,KAAK2B,QAAO;AACvB,UAAM,KAAKnC,QAAQkC,QAAO;AAC1B,UAAM,MAAMA,QAAQxB,GAAAA;EACtB;EAEA,MAAgB0B,cAAsC;AACpD,WAAO;MACLlC,iBAAiB;QACfmB,MAAM,OAAOgB,aAAAA;AACX9C,cAAI,eAAe;YAAEwC,gBAAgBM;UAAS,GAAA;;;;;;AAC9C,eAAKN,iBAAiBM;AACtB,eAAKP,iBAAgB;QACvB;QACAH,MAAM,OAAOW,SAAAA;AACX/C,cAAI,YAAY;YAAE+C;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAKtC,QAAQuC,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMX,KAAKW,MAAiB;AAC1B,QAAI,KAAK9B,KAAKU,UAAU;AACtB;IACF;AACA3B,QAAI,QAAQ;MAAE+C;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKlB,IAAIlB,gBAAgByB,KAAKW,IAAAA;EACtC;EAEAE,eAAeH,UAAoB;AACjC,QAAI,KAAK7B,KAAKU,UAAU;AACtB;IACF;AACA,SAAKL,iBAAiBwB;AACtB,SAAKtB,gBAAgB0B,SAAQ;EAC/B;EAEAX,mBAAmB;AACjB,QAAI,KAAKtB,KAAKU,UAAU;AACtB;IACF;AACA,SAAKK,QAAQkB,SAAQ;EACvB;EAEA,MAAchB,gBAAgBiB,SAAS,GAAgC;AA9JzE;AA+JI,QAAI,KAAKlC,KAAKU,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKa,eAAejB,WAAS,UAAKiB,eAAejB,UAApB,mBAA2B6B,YAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKb,eAAejB;MAAO+B,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKlD,QAAQmD,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACAhE,gBAAUgE,KAAKI,QAAQ;AACvBpE,gBAAUgE,KAAKK,SAAS;AACxBrE,gBAAUgE,KAAKP,MAAM;AAErB,UAAIM,OAAOM,aAAaN,OAAOM,cAAcL,KAAKK,WAAW;AAC3DhE,YAAIqC,KAAK,sBAAsB;UAAEqB;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMM,mBAAkBP,YAAOK,aAAPL,YAAmBtD,SAAS8D,KAAKP,KAAKP,SAASO,KAAKK,SAAS;AAErF,YAAMG,cAAc/D,SAASgE,IAAIH,iBAAiBN,KAAKI,QAAQ;AAC/D,YAAMM,eAAejE,SAASkE,YAAYH,WAAAA,EAAab,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWe,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAK/D,QAAQmD,UAAUa,IAAIf,OAAOI,IAAI;UAC5DY,QAAQH,MAAMZ,KAAKK;UACnBZ,QAAQG,KAAKoB,IAAIhB,KAAKK,WAAWL,KAAKP,SAASmB,MAAMZ,KAAKK,SAAS;QACrE,CAAA;AACAP,eAAOrB,KAAK;UACV0B,IAAIJ,OAAOI;UACXc,aAAajB,KAAKP;UAClBY,WAAWL,KAAKK;UAChBa,aAAaN,MAAMZ,KAAKK;UACxBc,SAASN;QACX,CAAA;AAEA,YAAIf,OAAOL,UAAUD,QAAQ;AAC3B,iBAAOM;QACT;MACF;IACF;AAEA,WAAOA;EACT;AACF;;EA/EG3D;GAtGUS,kBAAAA,WAAAA,QAAAA,IAAAA;;;AC5Bb,OAAOwE,gBAAe;AAEtB,SAASC,YAAYC,SAASC,YAAY;AAC1C,SAASC,mBAAmBC,WAAAA,gBAAe;AAC3C,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,gBAAgB;AAEzB,SAASC,YAAAA,WAAUC,kBAAkB;;;;;;;;;;;;AAiBrC,IAAaC,WAAN,MAAAA,UAAA;EAULC,YAA6BC,SAAyB;mBAAzBA;SATZC,OAAO,IAAIC,SAAAA;SACXC,QAAQ,IAAIC,KAAAA;SAEZC,oBAAoB,IAAIC,WAAwC,CAACC,QAChFC,UAAUC,KAAKF,GAAAA,EAAKG,MAAK,CAAA;SAGVC,cAAc,oBAAIC,IAAAA;EAEoB;EAEvD,MAAMC,OAAO;EAAC;EAEd,MAAMC,QAAQ;AACZ,UAAM,KAAKb,KAAKc,QAAO;EACzB;;;;;;EAOA,MAAMC,SAASC,KAAcC,IAA+B;AAC1DC,IAAAA,KAAI,YAAY;MAAED;IAAG,GAAA;;;;;;AACrB,UAAME,UAAU,MAAM,KAAKjB,MAAMkB,oBAAoB,YAAA;AACnD,YAAMC,kBAAkB,KAAKjB,kBAAkBkB,IAAIL,EAAAA;AAEnD,UAAII,iBAAiB;AACnBA,wBAAgBE;AAChB,eAAOF;MACT;AAEA,YAAMG,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUC,QAAQT,EAAAA;AAClD,YAAME,WAA2B;QAC/BQ,SAAS,IAAIC,QAAAA;QACbL,SAAS;QACTM,MAAM;UACJZ;UACAa,WAAWN,6BAAMM;UACjBC,WAAUP,6BAAMO,aAAYC,WAAWxB,KAAKyB,UAASC,OAAOV,KAAKO,QAAQ,CAAA;QAC3E;MACF;AAGA,WAAIP,6BAAMW,WAAUC,SAASC,MAAMC,eAAe;AAChDnB,QAAAA,SAAQQ,QAAQY,KAAI;MACtB,OAAO;AACL,aAAKnC,kBAAkBoC,IAAIvB,IAAIE,QAAAA;AAC/B,aAAKsB,0BAAyB;MAChC;AAEA,aAAOtB;IACT,CAAA;AAEAH,+BAAK0B,UAAU,MACb,KAAKxC,MAAMkB,oBAAoB,YAAA;AAE7B,YAAMD,WAAU,KAAKf,kBAAkBkB,IAAIL,EAAAA;AAC3C,UAAI,CAACE,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQI,YAAY,GAAG;AAC3B,aAAKnB,kBAAkBuC,OAAO1B,EAAAA;MAChC;AACA,WAAKwB,0BAAyB;IAChC,CAAA;AAGF,WAAOzB,MAAM4B,kBAAkB5B,KAAKG,QAAQQ,QAAQkB,KAAI,CAAA,IAAM1B,QAAQQ,QAAQkB,KAAI;EACpF;EAEAC,kBAAkB;AAChB,UAAMC,YAAY,IAAIC,kBAAkB;MACtCvB,WAAW,KAAK1B,QAAQ0B;MACxBwB,QAAQ,YAAA;AACN/B,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYwC,IAAIH,SAAAA;AACrBA,kBAAUI,eAAe,KAAKC,aAAY,CAAA;MAC5C;MACAC,SAAS,YAAA;AACPnC,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYiC,OAAOI,SAAAA;MAC1B;MACAO,QAAQ,OAAOC,cAAAA;AAjHrB;AAkHQ,YAAI,CAAC,KAAKnD,kBAAkBoD,IAAID,UAAUtC,EAAE,GAAG;AAC7C;QACF;AACAC,QAAAA,KAAI,YAAY;UAAEqC;QAAU,GAAA;;;;;;AAC5B,cAAM/B,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUgC,SAASF,SAAAA;AACnD,YAAI/B,KAAKW,UAAUC,SAASC,MAAMC,eAAe;AAC/C,qBAAKlC,kBAAkBkB,IAAIiC,UAAUtC,EAAE,MAAvC,mBAA0CU,QAAQY;AAClD,eAAKnC,kBAAkBuC,OAAOY,UAAUtC,EAAE;QAC5C,OAAO;AACLyC,UAAAA,WAAUlC,KAAKO,QAAQ;AACvB,eAAK3B,kBAAkBkB,IAAIiC,UAAUtC,EAAE,EAAGY,KAAKE,WAAWE,UAASC,OAAOV,KAAKO,QAAQ;QACzF;AAEA,aAAKU,0BAAyB;AAC9B,aAAKkB,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOZ;EACT;;;;EAKA,MAAMa,gBAAgBC,KAAiB;AACrC,SAAKF,kBAAiB;EACxB;EAEQP,eAAyB;AAC/B,WAAO;MACLU,OAAOC,MAAMvD,KAAK,KAAKJ,kBAAkB4D,OAAM,CAAA,EAAIC,IAAI,CAAC9C,YAAYA,QAAQU,IAAI;IAClF;EACF;EAEQ8B,oBAAoB;AAC1B,eAAWZ,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUmB,iBAAgB;IAC5B;EACF;EAEQzB,4BAA4B;AAClC,eAAWM,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUI,eAAe,KAAKC,aAAY,CAAA;IAC5C;EACF;AACF;AAjIavD,WAAAA,cAAAA;EADZsE,WAAW,QAAQ,OAAA;GACPtE,QAAAA;;;ACzBb,OAAOuE,UAAU;AAEjB,SAASC,gBAAAA,qBAAoB;AAC7B,SAASC,oBAAoB;AAC7B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,UAAAA,eAAc;AACvB,SAASC,YAAAA,iBAAgB;AAGzB,SAASC,YAAAA,WAAUC,qBAAqB;;;;;;;;;;;;AAOjC,IAAMC,qBAAqB;AAElC,IAAMC,gBAAgBL,QAAOM,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;;EAEXC,YACmBC,YACjB;sBADiBA;EACf;EAEJ,MACMC,QAAQC,IAA+C;AAC3D,WAAO,KAAKC,SAASD,EAAAA;EACvB;;;;EAKA,MACME,IAAIF,IAAgBG,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKH,SAASD,EAAAA;AAErC,QAAI,CAACI,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAEC,SAAS,GAAGC,SAASH,SAASG,OAAM,IAAKJ;AAEjD,QAAIG,SAASC,SAASH,SAASG,QAAQ;AACrC,YAAM,IAAIF,MAAM,eAAA;IAClB;AAEA,QAAID,SAASI,UAAUlB,UAASmB,MAAMC,eAAe;AACnD,YAAMC,QAAO,KAAKC,aAAaZ,EAAAA;AAC/B,aAAOW,MAAKE,KAAKP,QAAQC,MAAAA;IAC3B,WAAWJ,QAAQG,WAAWQ,UAAaX,QAAQI,WAAWO,QAAW;AACvE,YAAM,IAAIT,MAAM,oBAAA;IAClB;AAEA,UAAMU,aAAaC,KAAKC,MAAMX,SAASF,SAASc,SAAS;AACzD,UAAMC,WAAWH,KAAKI,MAAMd,SAASC,UAAUH,SAASc,SAAS;AAEjE9B,IAAAA,WAAUgB,SAASiB,UAAU,wBAAA;;;;;;;;;AAC7BjC,IAAAA,WAAUgB,SAASiB,SAASd,SAAS,KAAKY,UAAU,2BAAA;;;;;;;;;AAEpD,UAAMG,UAAU/B,UAASgC,MAAMnB,SAASiB,UAAUN,YAAYI,QAAAA,MAAcA,WAAWJ;AAEvF,QAAI,CAACO,SAAS;AACZ,YAAM,IAAIjB,MAAM,oBAAA;IAClB;AAEA,UAAMM,OAAO,KAAKC,aAAaZ,EAAAA;AAC/B,WAAOW,KAAKE,KAAKP,QAAQC,MAAAA;EAC3B;EAEA,MACMiB,OAA4B;AAQhC,UAAMC,QAAQ,IAAIC,KAAK,MAAM,KAAK5B,WAAW0B,KAAI,GAAIG,IAAI,CAACC,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWnB,QAAQc,OAAO;AACxB,YAAMzB,KAAKb,WAAU4C,KAAKpB,IAAAA,EAAMqB,aAAY;AAC5C,YAAMC,OAAO,MAAM,KAAKhC,SAASD,EAAAA;AACjC,UAAIiC,MAAM;AACRH,YAAII,KAAKD,IAAAA;MACX;IACF;AAEA,WAAOH;EACT;EAEA,MACMK,IAAIC,MAAqC;AAC7C,UAAMpC,KAAK,IAAIqC,WAAW,MAAMnD,aAAaoD,OAAO,WAAWF,IAAAA,CAAAA;AAC/D,UAAMf,WAAW9B,UAASgD,KAAKH,KAAK7B,SAASd,kBAAAA;AAE7C,UAAMwC,OAAiB;MACrBjC;MACAQ,OAAOlB,UAASmB,MAAMC;MACtBH,QAAQ6B,KAAK7B;MACbW,WAAWzB;MACX4B;MACAmB,SAAS,IAAIC,KAAAA;MACbC,SAAS,IAAID,KAAAA;IACf;AAEA,UAAM,KAAK7B,aAAaZ,EAAAA,EAAI2C,MAAM,GAAGnD,cAAc4C,IAAAA,CAAAA;AACnD,UAAM,KAAKQ,WAAW5C,IAAIiC,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACMY,SAASC,OAAqC;AA1HtD;AA4HI,QAAIb,OAAO,MAAM,KAAKhC,SAAS6C,MAAM9C,EAAE;AACvC,QAAI,CAACiC,MAAM;AACT7C,MAAAA,WAAU0D,MAAMC,aAAa,8BAAA;;;;;;;;;AAC7Bd,aAAO;QACLjC,IAAI8C,MAAM9C;QACVQ,OAAOlB,UAASmB,MAAMuC;QACtBzC,QAAQuC,MAAMC;QACd7B,YAAW4B,WAAM5B,cAAN4B,YAAmBrD;QAC9B+C,SAAS,IAAIC,KAAAA;MACf;AACAR,WAAKZ,WAAW9B,UAAS0D,MAAMhB,KAAK1B,SAAS0B,KAAKf,SAAS;IAC7D;AAEA,QAAI4B,MAAM5B,aAAa4B,MAAM5B,cAAce,KAAKf,WAAW;AACzD,YAAM,IAAIb,MAAM,oBAAA;IAClB;AAEAjB,IAAAA,WAAU6C,KAAKZ,UAAU,wBAAA;;;;;;;;;AACzBjC,IAAAA,WAAU0D,MAAMI,gBAAgBpC,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAakC,MAAM9C,EAAE,EAAE2C,MAAMG,MAAMI,aAAa1D,cAAcsD,MAAMK,OAAO,CAAA;AAGtF5D,IAAAA,UAAS4C,IAAIF,KAAKZ,UAAUL,KAAKC,MAAM6B,MAAMI,cAAcjB,KAAKf,SAAS,GAAG,IAAA;AAG5E,QAAI3B,UAASgC,MAAMU,KAAKZ,UAAU,GAAGY,KAAK1B,MAAM,IAAI0B,KAAKf,aAAae,KAAK1B,QAAQ;AACjF0B,WAAKzB,QAAQlB,UAASmB,MAAMC;IAC9B;AACAuB,SAAKS,UAAU,IAAID,KAAAA;AAEnB,UAAM,KAAKG,WAAWE,MAAM9C,IAAIiC,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAcW,WAAW5C,IAAgBiC,MAA+B;AACtE,UAAMmB,UAAU5D,cAAcE,cAAc2D,OAAOpB,IAAAA,CAAAA;AACnD,UAAMG,OAAOkB,OAAOC,MAAMH,QAAQ7C,SAAS,CAAA;AAC3C6B,SAAKoB,cAAcJ,QAAQ7C,QAAQ,CAAA;AACnC6C,YAAQK,KAAKrB,MAAM,CAAA;AAGnB,UAAM,KAAKsB,aAAa1D,EAAAA,EAAI2C,MAAM,GAAGP,IAAAA;EACvC;EAEA,MAAcnC,SAASD,IAA+C;AACpE,UAAMW,OAAO,KAAK+C,aAAa1D,EAAAA;AAC/B,UAAM2D,QAAQ,MAAMhD,KAAKiD,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAMvB,OAAO,MAAMzB,KAAKE,KAAK,GAAG8C,IAAAA;AAChC,UAAME,YAAYzB,KAAK0B,aAAa,CAAA;AACpC,WAAOpE,cAAcqE,OAAO3B,KAAK4B,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAa1D,IAAgB;AACnC,WAAO,KAAKF,WAAWmE,gBAAgBjF,KAAKkF,KAAK1E,cAAcQ,EAAAA,EAAImE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQvD,aAAaZ,IAAgB;AACnC,WAAO,KAAKF,WAAWmE,gBAAgBjF,KAAKkF,KAAK1E,cAAcQ,EAAAA,EAAImE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;AACF;;EA9JGlF;GANUW,UAAAA,WAAAA,WAAAA,IAAAA;;EAcVX;GAdUW,UAAAA,WAAAA,OAAAA,IAAAA;;EAmDVX;GAnDUW,UAAAA,WAAAA,QAAAA,IAAAA;;EA2EVX;GA3EUW,UAAAA,WAAAA,OAAAA,IAAAA;;EAgGVX;GAhGUW,UAAAA,WAAAA,YAAAA,IAAAA;",
|
|
6
|
-
"names": ["
|
|
4
|
+
"sourcesContent": ["//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { RpcClosedError, schema } from '@dxos/protocols';\nimport { BlobChunk, BlobSyncService, WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk) {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList) {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads() {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { trackLeaks, Trigger, Lock } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _lock = new Lock();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open() {}\n\n async close() {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._lock.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._lock.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension() {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array) {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads() {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList() {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { schema } from '@dxos/protocols';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n // prettier-ignore\n constructor(\n private readonly _directory: Directory\n ) { }\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;AAMA,SAASA,cAAcC,OAAOC,oBAAoB;AAClD,SAASC,eAAe;AACxB,SAASC,iBAAiB;AAC1B,SAASC,WAAW;AACpB,SAASC,gBAAgBC,cAAc;AAEvC,SAA2BC,oBAAoB;AAC/C,SAASC,gBAAgB;;;;;;;;;;;;AAWzB,IAAMC,gCAAgC;AAEtC,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCJ,aAAAA;EAwDrCK,YACmBC,SACjB;AACA,UAAM;MACJC,SAAS;QACPC,iBAAiBT,OAAOU,WAAW,6CAAA;MACrC;MACAC,WAAW;QACTF,iBAAiBT,OAAOU,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA;mBAbiBP;SAxDFQ,OAAO,IAAInB,QAAQ;MAAEoB,SAAS,CAACC,QAAQnB,IAAIoB,MAAMD,KAAAA,QAAAA;;;;;;IAAK,CAAA;SAE/DE,sBAAsB;SACtBC,iBAA2B;MAAEC,OAAO,CAAA;IAAG;SAE9BC,kBAAkB,IAAI7B,aAAa,KAAKsB,MAAM,YAAA;AAE7D,UAAI,KAAKI,sBAAsBhB,gCAAgCoB,KAAKC,IAAG,GAAI;AACzE,cAAM9B,MAAM,KAAKyB,sBAAsBhB,gCAAgCoB,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAKT,KAAKU,UAAU;AACtB;QACF;MACF;AAEA3B,UAAI,QAAQ;QAAE4B,MAAM,KAAKN;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKO,IAAIlB,gBAAgBmB,KAAK,KAAKR,cAAc;AACvD,WAAKD,sBAAsBI,KAAKC,IAAG;IACrC,CAAA;SAEQK,kBAAkB;SAETC,UAAU,IAAIrC,aAAa,KAAKsB,MAAM,YAAA;AACrD,UAAI,KAAKc,mBAAmBzB,wBAAwB;AAClD;MACF;AACA,YAAM2B,aAAa,MAAM,KAAKC,gBAAgB5B,yBAAyB,KAAKyB,eAAe;AAC3F,UAAI,CAACE,YAAY;AACf;MACF;AACA,iBAAWE,aAAaF,YAAY;AAClC,YAAI,KAAKhB,KAAKU,UAAU;AACtB;QACF;AAEA,aAAKI;AAEL,aAAKK,KAAKD,SAAAA,EACPf,MAAM,CAACD,QAAAA;AACN,cAAIA,eAAelB,gBAAgB;AACjC;UACF;AACAD,cAAIqC,KAAK,eAAe;YAAElB;UAAI,GAAA;;;;;;QAChC,CAAA,EACCmB,QAAQ,MAAA;AACP,eAAKP;AACL,eAAKQ,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA;SAKOC,iBAA2B;MAAEjB,OAAO,CAAA;IAAG;EAiB9C;EAEA,MAAekB,OAAOC,SAA0C;AAC9D1C,QAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMyC,OAAOC,OAAAA;AACnB,UAAM,KAAKjC,QAAQgC,OAAM;EAC3B;EAEA,MAAeE,QAAQxB,KAAwC;AAC7DnB,QAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKiB,KAAK2B,QAAO;AACvB,UAAM,KAAKnC,QAAQkC,QAAO;AAC1B,UAAM,MAAMA,QAAQxB,GAAAA;EACtB;EAEA,MAAgB0B,cAAsC;AACpD,WAAO;MACLlC,iBAAiB;QACfmB,MAAM,OAAOgB,aAAAA;AACX9C,cAAI,eAAe;YAAEwC,gBAAgBM;UAAS,GAAA;;;;;;AAC9C,eAAKN,iBAAiBM;AACtB,eAAKP,iBAAgB;QACvB;QACAH,MAAM,OAAOW,SAAAA;AACX/C,cAAI,YAAY;YAAE+C;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAKtC,QAAQuC,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMX,KAAKW,MAAiB;AAC1B,QAAI,KAAK9B,KAAKU,UAAU;AACtB;IACF;AACA3B,QAAI,QAAQ;MAAE+C;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKlB,IAAIlB,gBAAgByB,KAAKW,IAAAA;EACtC;EAEAE,eAAeH,UAAoB;AACjC,QAAI,KAAK7B,KAAKU,UAAU;AACtB;IACF;AACA,SAAKL,iBAAiBwB;AACtB,SAAKtB,gBAAgB0B,SAAQ;EAC/B;EAEAX,mBAAmB;AACjB,QAAI,KAAKtB,KAAKU,UAAU;AACtB;IACF;AACA,SAAKK,QAAQkB,SAAQ;EACvB;EAEA,MAAchB,gBAAgBiB,SAAS,GAAgC;AA7JzE;AA8JI,QAAI,KAAKlC,KAAKU,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKa,eAAejB,WAAS,UAAKiB,eAAejB,UAApB,mBAA2B6B,YAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKb,eAAejB;MAAO+B,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKlD,QAAQmD,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACA5D,gBAAU4D,KAAKI,UAAQ,QAAA;;;;;;;;;AACvBhE,gBAAU4D,KAAKK,WAAS,QAAA;;;;;;;;;AACxBjE,gBAAU4D,KAAKP,QAAM,QAAA;;;;;;;;;AAErB,UAAIM,OAAOM,aAAaN,OAAOM,cAAcL,KAAKK,WAAW;AAC3DhE,YAAIqC,KAAK,sBAAsB;UAAEqB;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMM,mBAAkBP,YAAOK,aAAPL,YAAmBtD,SAAS8D,KAAKP,KAAKP,SAASO,KAAKK,SAAS;AAErF,YAAMG,cAAc/D,SAASgE,IAAIH,iBAAiBN,KAAKI,QAAQ;AAC/D,YAAMM,eAAejE,SAASkE,YAAYH,WAAAA,EAAab,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWe,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAK/D,QAAQmD,UAAUa,IAAIf,OAAOI,IAAI;UAC5DY,QAAQH,MAAMZ,KAAKK;UACnBZ,QAAQG,KAAKoB,IAAIhB,KAAKK,WAAWL,KAAKP,SAASmB,MAAMZ,KAAKK,SAAS;QACrE,CAAA;AACAP,eAAOrB,KAAK;UACV0B,IAAIJ,OAAOI;UACXc,aAAajB,KAAKP;UAClBY,WAAWL,KAAKK;UAChBa,aAAaN,MAAMZ,KAAKK;UACxBc,SAASN;QACX,CAAA;AAEA,YAAIf,OAAOL,UAAUD,QAAQ;AAC3B,iBAAOM;QACT;MACF;IACF;AAEA,WAAOA;EACT;AACF;;EA/EG5D;GAtGUU,kBAAAA,WAAAA,QAAAA,IAAAA;;;AC3Bb,SAASwE,YAAYC,SAASC,YAAY;AAC1C,SAASC,mBAAmBC,WAAAA,gBAAe;AAC3C,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,gBAAgB;AAEzB,SAASC,YAAAA,WAAUC,kBAAkB;;;;;;;;;;;;AAiBrC,IAAaC,WAAN,MAAAA,UAAA;EAULC,YAA6BC,SAAyB;mBAAzBA;SATZC,OAAO,IAAIC,SAAAA;SACXC,QAAQ,IAAIC,KAAAA;SAEZC,oBAAoB,IAAIC,WAAwC,CAACC,QAChFC,UAAUC,KAAKF,GAAAA,EAAKG,MAAK,CAAA;SAGVC,cAAc,oBAAIC,IAAAA;EAEoB;EAEvD,MAAMC,OAAO;EAAC;EAEd,MAAMC,QAAQ;AACZ,UAAM,KAAKb,KAAKc,QAAO;EACzB;;;;;;EAOA,MAAMC,SAASC,KAAcC,IAA+B;AAC1DC,IAAAA,KAAI,YAAY;MAAED;IAAG,GAAA;;;;;;AACrB,UAAME,UAAU,MAAM,KAAKjB,MAAMkB,oBAAoB,YAAA;AACnD,YAAMC,kBAAkB,KAAKjB,kBAAkBkB,IAAIL,EAAAA;AAEnD,UAAII,iBAAiB;AACnBA,wBAAgBE;AAChB,eAAOF;MACT;AAEA,YAAMG,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUC,QAAQT,EAAAA;AAClD,YAAME,WAA2B;QAC/BQ,SAAS,IAAIC,QAAAA;QACbL,SAAS;QACTM,MAAM;UACJZ;UACAa,WAAWN,6BAAMM;UACjBC,WAAUP,6BAAMO,aAAYC,WAAWxB,KAAKyB,UAASC,OAAOV,KAAKO,QAAQ,CAAA;QAC3E;MACF;AAGA,WAAIP,6BAAMW,WAAUC,SAASC,MAAMC,eAAe;AAChDnB,QAAAA,SAAQQ,QAAQY,KAAI;MACtB,OAAO;AACL,aAAKnC,kBAAkBoC,IAAIvB,IAAIE,QAAAA;AAC/B,aAAKsB,0BAAyB;MAChC;AAEA,aAAOtB;IACT,CAAA;AAEAH,+BAAK0B,UAAU,MACb,KAAKxC,MAAMkB,oBAAoB,YAAA;AAE7B,YAAMD,WAAU,KAAKf,kBAAkBkB,IAAIL,EAAAA;AAC3C,UAAI,CAACE,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQI,YAAY,GAAG;AAC3B,aAAKnB,kBAAkBuC,OAAO1B,EAAAA;MAChC;AACA,WAAKwB,0BAAyB;IAChC,CAAA;AAGF,WAAOzB,MAAM4B,kBAAkB5B,KAAKG,QAAQQ,QAAQkB,KAAI,CAAA,IAAM1B,QAAQQ,QAAQkB,KAAI;EACpF;EAEAC,kBAAkB;AAChB,UAAMC,YAAY,IAAIC,kBAAkB;MACtCvB,WAAW,KAAK1B,QAAQ0B;MACxBwB,QAAQ,YAAA;AACN/B,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYwC,IAAIH,SAAAA;AACrBA,kBAAUI,eAAe,KAAKC,aAAY,CAAA;MAC5C;MACAC,SAAS,YAAA;AACPnC,QAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYiC,OAAOI,SAAAA;MAC1B;MACAO,QAAQ,OAAOC,cAAAA;AAhHrB;AAiHQ,YAAI,CAAC,KAAKnD,kBAAkBoD,IAAID,UAAUtC,EAAE,GAAG;AAC7C;QACF;AACAC,QAAAA,KAAI,YAAY;UAAEqC;QAAU,GAAA;;;;;;AAC5B,cAAM/B,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUgC,SAASF,SAAAA;AACnD,YAAI/B,KAAKW,UAAUC,SAASC,MAAMC,eAAe;AAC/C,qBAAKlC,kBAAkBkB,IAAIiC,UAAUtC,EAAE,MAAvC,mBAA0CU,QAAQY;AAClD,eAAKnC,kBAAkBuC,OAAOY,UAAUtC,EAAE;QAC5C,OAAO;AACLyC,UAAAA,WAAUlC,KAAKO,UAAQ,QAAA;;;;;;;;;AACvB,eAAK3B,kBAAkBkB,IAAIiC,UAAUtC,EAAE,EAAGY,KAAKE,WAAWE,UAASC,OAAOV,KAAKO,QAAQ;QACzF;AAEA,aAAKU,0BAAyB;AAC9B,aAAKkB,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOZ;EACT;;;;EAKA,MAAMa,gBAAgBC,KAAiB;AACrC,SAAKF,kBAAiB;EACxB;EAEQP,eAAyB;AAC/B,WAAO;MACLU,OAAOC,MAAMvD,KAAK,KAAKJ,kBAAkB4D,OAAM,CAAA,EAAIC,IAAI,CAAC9C,YAAYA,QAAQU,IAAI;IAClF;EACF;EAEQ8B,oBAAoB;AAC1B,eAAWZ,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUmB,iBAAgB;IAC5B;EACF;EAEQzB,4BAA4B;AAClC,eAAWM,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUI,eAAe,KAAKC,aAAY,CAAA;IAC5C;EACF;AACF;AAjIavD,WAAAA,cAAAA;EADZsE,WAAW,QAAQ,OAAA;GACPtE,QAAAA;;;ACxBb,OAAOuE,UAAU;AAEjB,SAASC,gBAAAA,qBAAoB;AAC7B,SAASC,oBAAoB;AAC7B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,aAAAA,kBAAiB;AAC1B,SAASC,UAAAA,eAAc;AACvB,SAASC,YAAAA,iBAAgB;AAGzB,SAASC,YAAAA,WAAUC,qBAAqB;;;;;;;;;;;;AAOjC,IAAMC,qBAAqB;AAElC,IAAMC,gBAAgBL,QAAOM,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;;EAEXC,YACmBC,YACjB;sBADiBA;EACf;EAEJ,MACMC,QAAQC,IAA+C;AAC3D,WAAO,KAAKC,SAASD,EAAAA;EACvB;;;;EAKA,MACME,IAAIF,IAAgBG,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKH,SAASD,EAAAA;AAErC,QAAI,CAACI,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAEC,SAAS,GAAGC,SAASH,SAASG,OAAM,IAAKJ;AAEjD,QAAIG,SAASC,SAASH,SAASG,QAAQ;AACrC,YAAM,IAAIF,MAAM,eAAA;IAClB;AAEA,QAAID,SAASI,UAAUlB,UAASmB,MAAMC,eAAe;AACnD,YAAMC,QAAO,KAAKC,aAAaZ,EAAAA;AAC/B,aAAOW,MAAKE,KAAKP,QAAQC,MAAAA;IAC3B,WAAWJ,QAAQG,WAAWQ,UAAaX,QAAQI,WAAWO,QAAW;AACvE,YAAM,IAAIT,MAAM,oBAAA;IAClB;AAEA,UAAMU,aAAaC,KAAKC,MAAMX,SAASF,SAASc,SAAS;AACzD,UAAMC,WAAWH,KAAKI,MAAMd,SAASC,UAAUH,SAASc,SAAS;AAEjE/B,IAAAA,WAAUiB,SAASiB,UAAU,wBAAA;;;;;;;;;AAC7BlC,IAAAA,WAAUiB,SAASiB,SAASd,SAAS,KAAKY,UAAU,2BAAA;;;;;;;;;AAEpD,UAAMG,UAAU/B,UAASgC,MAAMnB,SAASiB,UAAUN,YAAYI,QAAAA,MAAcA,WAAWJ;AAEvF,QAAI,CAACO,SAAS;AACZ,YAAM,IAAIjB,MAAM,oBAAA;IAClB;AAEA,UAAMM,OAAO,KAAKC,aAAaZ,EAAAA;AAC/B,WAAOW,KAAKE,KAAKP,QAAQC,MAAAA;EAC3B;EAEA,MACMiB,OAA4B;AAQhC,UAAMC,QAAQ,IAAIC,KAAK,MAAM,KAAK5B,WAAW0B,KAAI,GAAIG,IAAI,CAACC,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWnB,QAAQc,OAAO;AACxB,YAAMzB,KAAKZ,WAAU2C,KAAKpB,IAAAA,EAAMqB,aAAY;AAC5C,YAAMC,OAAO,MAAM,KAAKhC,SAASD,EAAAA;AACjC,UAAIiC,MAAM;AACRH,YAAII,KAAKD,IAAAA;MACX;IACF;AAEA,WAAOH;EACT;EAEA,MACMK,IAAIC,MAAqC;AAC7C,UAAMpC,KAAK,IAAIqC,WAAW,MAAMnD,aAAaoD,OAAO,WAAWF,IAAAA,CAAAA;AAC/D,UAAMf,WAAW9B,UAASgD,KAAKH,KAAK7B,SAASd,kBAAAA;AAE7C,UAAMwC,OAAiB;MACrBjC;MACAQ,OAAOlB,UAASmB,MAAMC;MACtBH,QAAQ6B,KAAK7B;MACbW,WAAWzB;MACX4B;MACAmB,SAAS,IAAIC,KAAAA;MACbC,SAAS,IAAID,KAAAA;IACf;AAEA,UAAM,KAAK7B,aAAaZ,EAAAA,EAAI2C,MAAM,GAAGnD,cAAc4C,IAAAA,CAAAA;AACnD,UAAM,KAAKQ,WAAW5C,IAAIiC,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACMY,SAASC,OAAqC;AA1HtD;AA4HI,QAAIb,OAAO,MAAM,KAAKhC,SAAS6C,MAAM9C,EAAE;AACvC,QAAI,CAACiC,MAAM;AACT9C,MAAAA,WAAU2D,MAAMC,aAAa,8BAAA;;;;;;;;;AAC7Bd,aAAO;QACLjC,IAAI8C,MAAM9C;QACVQ,OAAOlB,UAASmB,MAAMuC;QACtBzC,QAAQuC,MAAMC;QACd7B,YAAW4B,WAAM5B,cAAN4B,YAAmBrD;QAC9B+C,SAAS,IAAIC,KAAAA;MACf;AACAR,WAAKZ,WAAW9B,UAAS0D,MAAMhB,KAAK1B,SAAS0B,KAAKf,SAAS;IAC7D;AAEA,QAAI4B,MAAM5B,aAAa4B,MAAM5B,cAAce,KAAKf,WAAW;AACzD,YAAM,IAAIb,MAAM,oBAAA;IAClB;AAEAlB,IAAAA,WAAU8C,KAAKZ,UAAU,wBAAA;;;;;;;;;AACzBlC,IAAAA,WAAU2D,MAAMI,gBAAgBpC,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAakC,MAAM9C,EAAE,EAAE2C,MAAMG,MAAMI,aAAa1D,cAAcsD,MAAMK,OAAO,CAAA;AAGtF5D,IAAAA,UAAS4C,IAAIF,KAAKZ,UAAUL,KAAKC,MAAM6B,MAAMI,cAAcjB,KAAKf,SAAS,GAAG,IAAA;AAG5E,QAAI3B,UAASgC,MAAMU,KAAKZ,UAAU,GAAGY,KAAK1B,MAAM,IAAI0B,KAAKf,aAAae,KAAK1B,QAAQ;AACjF0B,WAAKzB,QAAQlB,UAASmB,MAAMC;IAC9B;AACAuB,SAAKS,UAAU,IAAID,KAAAA;AAEnB,UAAM,KAAKG,WAAWE,MAAM9C,IAAIiC,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAcW,WAAW5C,IAAgBiC,MAA+B;AACtE,UAAMmB,UAAU5D,cAAcE,cAAc2D,OAAOpB,IAAAA,CAAAA;AACnD,UAAMG,OAAOkB,OAAOC,MAAMH,QAAQ7C,SAAS,CAAA;AAC3C6B,SAAKoB,cAAcJ,QAAQ7C,QAAQ,CAAA;AACnC6C,YAAQK,KAAKrB,MAAM,CAAA;AAGnB,UAAM,KAAKsB,aAAa1D,EAAAA,EAAI2C,MAAM,GAAGP,IAAAA;EACvC;EAEA,MAAcnC,SAASD,IAA+C;AACpE,UAAMW,OAAO,KAAK+C,aAAa1D,EAAAA;AAC/B,UAAM2D,QAAQ,MAAMhD,KAAKiD,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAMvB,OAAO,MAAMzB,KAAKE,KAAK,GAAG8C,IAAAA;AAChC,UAAME,YAAYzB,KAAK0B,aAAa,CAAA;AACpC,WAAOpE,cAAcqE,OAAO3B,KAAK4B,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAa1D,IAAgB;AACnC,WAAO,KAAKF,WAAWmE,gBAAgBjF,KAAKkF,KAAK1E,cAAcQ,EAAAA,EAAImE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQvD,aAAaZ,IAAgB;AACnC,WAAO,KAAKF,WAAWmE,gBAAgBjF,KAAKkF,KAAK1E,cAAcQ,EAAAA,EAAImE,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;AACF;;EA9JGlF;GANUW,UAAAA,WAAAA,WAAAA,IAAAA;;EAcVX;GAdUW,UAAAA,WAAAA,OAAAA,IAAAA;;EAmDVX;GAnDUW,UAAAA,WAAAA,QAAAA,IAAAA;;EA2EVX;GA3EUW,UAAAA,WAAAA,OAAAA,IAAAA;;EAgGVX;GAhGUW,UAAAA,WAAAA,YAAAA,IAAAA;",
|
|
6
|
+
"names": ["DeferredTask", "sleep", "synchronized", "Context", "invariant", "log", "RpcClosedError", "schema", "RpcExtension", "BitField", "MIN_WANT_LIST_UPDATE_INTERVAL", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "constructor", "_params", "exposed", "BlobSyncService", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_ctx", "onError", "err", "catch", "_lastWantListUpdate", "_localWantList", "blobs", "_updateWantList", "Date", "now", "disposed", "list", "rpc", "want", "_currentUploads", "_upload", "blobChunks", "_pickBlobChunks", "blobChunk", "push", "warn", "finally", "reconcileUploads", "remoteWantList", "onOpen", "context", "onClose", "dispose", "getHandlers", "wantList", "data", "onPush", "updateWantList", "schedule", "amount", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "bitfield", "chunkSize", "requestBitfield", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "trackLeaks", "Trigger", "Lock", "cancelWithContext", "Context", "invariant", "PublicKey", "log", "BlobMeta", "BitField", "ComplexMap", "BlobSync", "constructor", "_params", "_ctx", "Context", "_lock", "Lock", "_downloadRequests", "ComplexMap", "key", "PublicKey", "from", "toHex", "_extensions", "Set", "open", "close", "dispose", "download", "ctx", "id", "log", "request", "executeSynchronized", "existingRequest", "get", "counter", "meta", "blobStore", "getMeta", "trigger", "Trigger", "want", "chunkSize", "bitfield", "Uint8Array", "BitField", "invert", "state", "BlobMeta", "State", "FULLY_PRESENT", "wake", "set", "_updateExtensionsWantList", "onDispose", "delete", "cancelWithContext", "wait", "createExtension", "extension", "BlobSyncExtension", "onOpen", "add", "updateWantList", "_getWantList", "onClose", "onPush", "blobChunk", "has", "setChunk", "invariant", "_reconcileUploads", "notifyBlobAdded", "_id", "blobs", "Array", "values", "map", "reconcileUploads", "trackLeaks", "path", "synchronized", "subtleCrypto", "invariant", "PublicKey", "schema", "BlobMeta", "BitField", "arrayToBuffer", "DEFAULT_CHUNK_SIZE", "BlobMetaCodec", "getCodecForType", "BlobStore", "constructor", "_directory", "getMeta", "id", "_getMeta", "get", "options", "metadata", "Error", "offset", "length", "state", "State", "FULLY_PRESENT", "file", "_getDataFile", "read", "undefined", "beginChunk", "Math", "floor", "chunkSize", "endChunk", "ceil", "bitfield", "present", "count", "list", "files", "Set", "map", "f", "split", "res", "from", "asUint8Array", "meta", "push", "set", "data", "Uint8Array", "digest", "ones", "created", "Date", "updated", "write", "_writeMeta", "setChunk", "chunk", "totalLength", "PARTIALLY_PRESENT", "zeros", "chunkOffset", "payload", "encoded", "encode", "Buffer", "alloc", "writeUInt32LE", "copy", "_getMetaFile", "size", "stat", "protoSize", "readUInt32LE", "decode", "subarray", "getOrCreateFile", "join", "toString"]
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":
|
|
1
|
+
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":24170,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}]},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytes":17801,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"}]},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytes":22348,"imports":[{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}]},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytes":703,"imports":[{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts","kind":"import-statement","original":"./blob-sync"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts","kind":"import-statement","original":"./blob-store"}]}},"outputs":{"packages/core/mesh/teleport-extension-object-sync/dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":29874},"packages/core/mesh/teleport-extension-object-sync/dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["BlobStore","BlobSync","BlobSyncExtension","DEFAULT_CHUNK_SIZE"],"entryPoint":"packages/core/mesh/teleport-extension-object-sync/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytesInOutput":7174},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytesInOutput":0},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytesInOutput":5315},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytesInOutput":6535}},"bytes":19395}}}
|
package/dist/lib/node/index.cjs
CHANGED
|
@@ -38,9 +38,9 @@ __export(src_exports, {
|
|
|
38
38
|
module.exports = __toCommonJS(src_exports);
|
|
39
39
|
|
|
40
40
|
// packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts
|
|
41
|
-
var import_tiny_invariant = __toESM(require("tiny-invariant"));
|
|
42
41
|
var import_async = require("@dxos/async");
|
|
43
42
|
var import_context = require("@dxos/context");
|
|
43
|
+
var import_invariant = require("@dxos/invariant");
|
|
44
44
|
var import_log = require("@dxos/log");
|
|
45
45
|
var import_protocols = require("@dxos/protocols");
|
|
46
46
|
var import_teleport = require("@dxos/teleport");
|
|
@@ -76,7 +76,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
76
76
|
this._ctx = new import_context.Context({
|
|
77
77
|
onError: (err) => import_log.log.catch(err, void 0, {
|
|
78
78
|
F: __dxlog_file,
|
|
79
|
-
L:
|
|
79
|
+
L: 33,
|
|
80
80
|
S: this,
|
|
81
81
|
C: (f, a) => f(...a)
|
|
82
82
|
})
|
|
@@ -96,7 +96,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
96
96
|
list: this._localWantList
|
|
97
97
|
}, {
|
|
98
98
|
F: __dxlog_file,
|
|
99
|
-
L:
|
|
99
|
+
L: 47,
|
|
100
100
|
S: this,
|
|
101
101
|
C: (f, a) => f(...a)
|
|
102
102
|
});
|
|
@@ -125,7 +125,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
125
125
|
err
|
|
126
126
|
}, {
|
|
127
127
|
F: __dxlog_file,
|
|
128
|
-
L:
|
|
128
|
+
L: 74,
|
|
129
129
|
S: this,
|
|
130
130
|
C: (f, a) => f(...a)
|
|
131
131
|
});
|
|
@@ -142,7 +142,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
142
142
|
async onOpen(context) {
|
|
143
143
|
(0, import_log.log)("open", void 0, {
|
|
144
144
|
F: __dxlog_file,
|
|
145
|
-
L:
|
|
145
|
+
L: 106,
|
|
146
146
|
S: this,
|
|
147
147
|
C: (f, a) => f(...a)
|
|
148
148
|
});
|
|
@@ -152,7 +152,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
152
152
|
async onClose(err) {
|
|
153
153
|
(0, import_log.log)("close", void 0, {
|
|
154
154
|
F: __dxlog_file,
|
|
155
|
-
L:
|
|
155
|
+
L: 112,
|
|
156
156
|
S: this,
|
|
157
157
|
C: (f, a) => f(...a)
|
|
158
158
|
});
|
|
@@ -168,7 +168,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
168
168
|
remoteWantList: wantList
|
|
169
169
|
}, {
|
|
170
170
|
F: __dxlog_file,
|
|
171
|
-
L:
|
|
171
|
+
L: 122,
|
|
172
172
|
S: this,
|
|
173
173
|
C: (f, a) => f(...a)
|
|
174
174
|
});
|
|
@@ -180,7 +180,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
180
180
|
data
|
|
181
181
|
}, {
|
|
182
182
|
F: __dxlog_file,
|
|
183
|
-
L:
|
|
183
|
+
L: 127,
|
|
184
184
|
S: this,
|
|
185
185
|
C: (f, a) => f(...a)
|
|
186
186
|
});
|
|
@@ -197,7 +197,7 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
197
197
|
data
|
|
198
198
|
}, {
|
|
199
199
|
F: __dxlog_file,
|
|
200
|
-
L:
|
|
200
|
+
L: 139,
|
|
201
201
|
S: this,
|
|
202
202
|
C: (f, a) => f(...a)
|
|
203
203
|
});
|
|
@@ -233,16 +233,40 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
233
233
|
if (!meta) {
|
|
234
234
|
continue;
|
|
235
235
|
}
|
|
236
|
-
(0,
|
|
237
|
-
|
|
238
|
-
|
|
236
|
+
(0, import_invariant.invariant)(meta.bitfield, void 0, {
|
|
237
|
+
F: __dxlog_file,
|
|
238
|
+
L: 178,
|
|
239
|
+
S: this,
|
|
240
|
+
A: [
|
|
241
|
+
"meta.bitfield",
|
|
242
|
+
""
|
|
243
|
+
]
|
|
244
|
+
});
|
|
245
|
+
(0, import_invariant.invariant)(meta.chunkSize, void 0, {
|
|
246
|
+
F: __dxlog_file,
|
|
247
|
+
L: 179,
|
|
248
|
+
S: this,
|
|
249
|
+
A: [
|
|
250
|
+
"meta.chunkSize",
|
|
251
|
+
""
|
|
252
|
+
]
|
|
253
|
+
});
|
|
254
|
+
(0, import_invariant.invariant)(meta.length, void 0, {
|
|
255
|
+
F: __dxlog_file,
|
|
256
|
+
L: 180,
|
|
257
|
+
S: this,
|
|
258
|
+
A: [
|
|
259
|
+
"meta.length",
|
|
260
|
+
""
|
|
261
|
+
]
|
|
262
|
+
});
|
|
239
263
|
if (header.chunkSize && header.chunkSize !== meta.chunkSize) {
|
|
240
264
|
import_log.log.warn("Invalid chunk size", {
|
|
241
265
|
header,
|
|
242
266
|
meta
|
|
243
267
|
}, {
|
|
244
268
|
F: __dxlog_file,
|
|
245
|
-
L:
|
|
269
|
+
L: 183,
|
|
246
270
|
S: this,
|
|
247
271
|
C: (f, a) => f(...a)
|
|
248
272
|
});
|
|
@@ -276,9 +300,9 @@ _ts_decorate([
|
|
|
276
300
|
], BlobSyncExtension.prototype, "push", null);
|
|
277
301
|
|
|
278
302
|
// packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts
|
|
279
|
-
var import_tiny_invariant2 = __toESM(require("tiny-invariant"));
|
|
280
303
|
var import_async2 = require("@dxos/async");
|
|
281
304
|
var import_context2 = require("@dxos/context");
|
|
305
|
+
var import_invariant2 = require("@dxos/invariant");
|
|
282
306
|
var import_keys = require("@dxos/keys");
|
|
283
307
|
var import_log2 = require("@dxos/log");
|
|
284
308
|
var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
@@ -317,7 +341,7 @@ var BlobSync = class BlobSync2 {
|
|
|
317
341
|
id
|
|
318
342
|
}, {
|
|
319
343
|
F: __dxlog_file2,
|
|
320
|
-
L:
|
|
344
|
+
L: 53,
|
|
321
345
|
S: this,
|
|
322
346
|
C: (f, a) => f(...a)
|
|
323
347
|
});
|
|
@@ -363,7 +387,7 @@ var BlobSync = class BlobSync2 {
|
|
|
363
387
|
onOpen: async () => {
|
|
364
388
|
(0, import_log2.log)("extension opened", void 0, {
|
|
365
389
|
F: __dxlog_file2,
|
|
366
|
-
L:
|
|
390
|
+
L: 105,
|
|
367
391
|
S: this,
|
|
368
392
|
C: (f, a) => f(...a)
|
|
369
393
|
});
|
|
@@ -373,7 +397,7 @@ var BlobSync = class BlobSync2 {
|
|
|
373
397
|
onClose: async () => {
|
|
374
398
|
(0, import_log2.log)("extension closed", void 0, {
|
|
375
399
|
F: __dxlog_file2,
|
|
376
|
-
L:
|
|
400
|
+
L: 110,
|
|
377
401
|
S: this,
|
|
378
402
|
C: (f, a) => f(...a)
|
|
379
403
|
});
|
|
@@ -388,7 +412,7 @@ var BlobSync = class BlobSync2 {
|
|
|
388
412
|
blobChunk
|
|
389
413
|
}, {
|
|
390
414
|
F: __dxlog_file2,
|
|
391
|
-
L:
|
|
415
|
+
L: 117,
|
|
392
416
|
S: this,
|
|
393
417
|
C: (f, a) => f(...a)
|
|
394
418
|
});
|
|
@@ -397,7 +421,15 @@ var BlobSync = class BlobSync2 {
|
|
|
397
421
|
(_a = this._downloadRequests.get(blobChunk.id)) == null ? void 0 : _a.trigger.wake();
|
|
398
422
|
this._downloadRequests.delete(blobChunk.id);
|
|
399
423
|
} else {
|
|
400
|
-
(0,
|
|
424
|
+
(0, import_invariant2.invariant)(meta.bitfield, void 0, {
|
|
425
|
+
F: __dxlog_file2,
|
|
426
|
+
L: 123,
|
|
427
|
+
S: this,
|
|
428
|
+
A: [
|
|
429
|
+
"meta.bitfield",
|
|
430
|
+
""
|
|
431
|
+
]
|
|
432
|
+
});
|
|
401
433
|
this._downloadRequests.get(blobChunk.id).want.bitfield = import_util2.BitField.invert(meta.bitfield);
|
|
402
434
|
}
|
|
403
435
|
this._updateExtensionsWantList();
|
|
@@ -436,8 +468,8 @@ BlobSync = _ts_decorate2([
|
|
|
436
468
|
var import_node_path = __toESM(require("node:path"));
|
|
437
469
|
var import_async3 = require("@dxos/async");
|
|
438
470
|
var import_crypto = require("@dxos/crypto");
|
|
471
|
+
var import_invariant3 = require("@dxos/invariant");
|
|
439
472
|
var import_keys2 = require("@dxos/keys");
|
|
440
|
-
var import_log3 = require("@dxos/log");
|
|
441
473
|
var import_protocols2 = require("@dxos/protocols");
|
|
442
474
|
var import_blob2 = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
443
475
|
var import_util3 = require("@dxos/util");
|
|
@@ -482,7 +514,7 @@ var BlobStore = class {
|
|
|
482
514
|
}
|
|
483
515
|
const beginChunk = Math.floor(offset / metadata.chunkSize);
|
|
484
516
|
const endChunk = Math.ceil((offset + length) / metadata.chunkSize);
|
|
485
|
-
(0,
|
|
517
|
+
(0, import_invariant3.invariant)(metadata.bitfield, "Bitfield not present", {
|
|
486
518
|
F: __dxlog_file3,
|
|
487
519
|
L: 64,
|
|
488
520
|
S: this,
|
|
@@ -491,7 +523,7 @@ var BlobStore = class {
|
|
|
491
523
|
"'Bitfield not present'"
|
|
492
524
|
]
|
|
493
525
|
});
|
|
494
|
-
(0,
|
|
526
|
+
(0, import_invariant3.invariant)(metadata.bitfield.length * 8 >= endChunk, "Invalid bitfield length", {
|
|
495
527
|
F: __dxlog_file3,
|
|
496
528
|
L: 65,
|
|
497
529
|
S: this,
|
|
@@ -540,7 +572,7 @@ var BlobStore = class {
|
|
|
540
572
|
var _a;
|
|
541
573
|
let meta = await this._getMeta(chunk.id);
|
|
542
574
|
if (!meta) {
|
|
543
|
-
(0,
|
|
575
|
+
(0, import_invariant3.invariant)(chunk.totalLength, "totalLength is not present", {
|
|
544
576
|
F: __dxlog_file3,
|
|
545
577
|
L: 127,
|
|
546
578
|
S: this,
|
|
@@ -561,7 +593,7 @@ var BlobStore = class {
|
|
|
561
593
|
if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {
|
|
562
594
|
throw new Error("Invalid chunk size");
|
|
563
595
|
}
|
|
564
|
-
(0,
|
|
596
|
+
(0, import_invariant3.invariant)(meta.bitfield, "Bitfield not present", {
|
|
565
597
|
F: __dxlog_file3,
|
|
566
598
|
L: 142,
|
|
567
599
|
S: this,
|
|
@@ -570,7 +602,7 @@ var BlobStore = class {
|
|
|
570
602
|
"'Bitfield not present'"
|
|
571
603
|
]
|
|
572
604
|
});
|
|
573
|
-
(0,
|
|
605
|
+
(0, import_invariant3.invariant)(chunk.chunkOffset !== void 0, "chunkOffset is not present", {
|
|
574
606
|
F: __dxlog_file3,
|
|
575
607
|
L: 143,
|
|
576
608
|
S: this,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/index.ts", "../../../src/blob-sync-extension.ts", "../../../src/blob-sync.ts", "../../../src/blob-store.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './blob-sync-extension';\nexport * from './blob-sync';\nexport * from './blob-store';\n", "//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport invariant from 'tiny-invariant';\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { log } from '@dxos/log';\nimport { RpcClosedError, schema } from '@dxos/protocols';\nimport { BlobChunk, BlobSyncService, WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk) {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList) {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads() {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport invariant from 'tiny-invariant';\n\nimport { trackLeaks, Trigger, Lock } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _lock = new Lock();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open() {}\n\n async close() {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._lock.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._lock.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension() {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array) {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads() {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList() {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { PublicKey } from '@dxos/keys';\nimport { invariant } from '@dxos/log';\nimport { schema } from '@dxos/protocols';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n // prettier-ignore\n constructor(\n private readonly _directory: Directory\n ) { }\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;;;ACMA,
|
|
6
|
-
"names": ["MIN_WANT_LIST_UPDATE_INTERVAL", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "RpcExtension", "constructor", "_params", "exposed", "BlobSyncService", "schema", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_ctx", "Context", "onError", "err", "log", "catch", "_lastWantListUpdate", "_localWantList", "blobs", "_updateWantList", "DeferredTask", "Date", "now", "sleep", "disposed", "list", "rpc", "want", "_currentUploads", "_upload", "blobChunks", "_pickBlobChunks", "blobChunk", "push", "RpcClosedError", "warn", "finally", "reconcileUploads", "remoteWantList", "onOpen", "context", "onClose", "dispose", "getHandlers", "wantList", "data", "onPush", "updateWantList", "schedule", "amount", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "invariant", "bitfield", "chunkSize", "requestBitfield", "BitField", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "synchronized", "
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './blob-sync-extension';\nexport * from './blob-sync';\nexport * from './blob-store';\n", "//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { RpcClosedError, schema } from '@dxos/protocols';\nimport { BlobChunk, BlobSyncService, WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk) {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList) {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads() {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { trackLeaks, Trigger, Lock } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _lock = new Lock();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open() {}\n\n async close() {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._lock.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._lock.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension() {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array) {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads() {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList() {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { schema } from '@dxos/protocols';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n // prettier-ignore\n constructor(\n private readonly _directory: Directory\n ) { }\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;;;ACMA,mBAAkD;AAClD,qBAAwB;AACxB,uBAA0B;AAC1B,iBAAoB;AACpB,uBAAuC;AAEvC,sBAA+C;AAC/C,kBAAyB;;;;;;;;;;;;AAWzB,IAAMA,gCAAgC;AAEtC,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCC,6BAAAA;EAwDrCC,YACmBC,SACjB;AACA,UAAM;MACJC,SAAS;QACPC,iBAAiBC,wBAAOC,WAAW,6CAAA;MACrC;MACAC,WAAW;QACTH,iBAAiBC,wBAAOC,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA;mBAbiBR;SAxDFS,OAAO,IAAIC,uBAAQ;MAAEC,SAAS,CAACC,QAAQC,eAAIC,MAAMF,KAAAA,QAAAA;;;;;;IAAK,CAAA;SAE/DG,sBAAsB;SACtBC,iBAA2B;MAAEC,OAAO,CAAA;IAAG;SAE9BC,kBAAkB,IAAIC,0BAAa,KAAKV,MAAM,YAAA;AAE7D,UAAI,KAAKM,sBAAsBpB,gCAAgCyB,KAAKC,IAAG,GAAI;AACzE,kBAAMC,oBAAM,KAAKP,sBAAsBpB,gCAAgCyB,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAKZ,KAAKc,UAAU;AACtB;QACF;MACF;AAEAV,0BAAI,QAAQ;QAAEW,MAAM,KAAKR;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKS,IAAIvB,gBAAgBwB,KAAK,KAAKV,cAAc;AACvD,WAAKD,sBAAsBK,KAAKC,IAAG;IACrC,CAAA;SAEQM,kBAAkB;SAETC,UAAU,IAAIT,0BAAa,KAAKV,MAAM,YAAA;AACrD,UAAI,KAAKkB,mBAAmB/B,wBAAwB;AAClD;MACF;AACA,YAAMiC,aAAa,MAAM,KAAKC,gBAAgBlC,yBAAyB,KAAK+B,eAAe;AAC3F,UAAI,CAACE,YAAY;AACf;MACF;AACA,iBAAWE,aAAaF,YAAY;AAClC,YAAI,KAAKpB,KAAKc,UAAU;AACtB;QACF;AAEA,aAAKI;AAEL,aAAKK,KAAKD,SAAAA,EACPjB,MAAM,CAACF,QAAAA;AACN,cAAIA,eAAeqB,iCAAgB;AACjC;UACF;AACApB,yBAAIqB,KAAK,eAAe;YAAEtB;UAAI,GAAA;;;;;;QAChC,CAAA,EACCuB,QAAQ,MAAA;AACP,eAAKR;AACL,eAAKS,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA;SAKOC,iBAA2B;MAAEpB,OAAO,CAAA;IAAG;EAiB9C;EAEA,MAAeqB,OAAOC,SAA0C;AAC9D1B,wBAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMyB,OAAOC,OAAAA;AACnB,UAAM,KAAKvC,QAAQsC,OAAM;EAC3B;EAEA,MAAeE,QAAQ5B,KAAwC;AAC7DC,wBAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKJ,KAAKgC,QAAO;AACvB,UAAM,KAAKzC,QAAQwC,QAAO;AAC1B,UAAM,MAAMA,QAAQ5B,GAAAA;EACtB;EAEA,MAAgB8B,cAAsC;AACpD,WAAO;MACLxC,iBAAiB;QACfwB,MAAM,OAAOiB,aAAAA;AACX9B,8BAAI,eAAe;YAAEwB,gBAAgBM;UAAS,GAAA;;;;;;AAC9C,eAAKN,iBAAiBM;AACtB,eAAKP,iBAAgB;QACvB;QACAJ,MAAM,OAAOY,SAAAA;AACX/B,8BAAI,YAAY;YAAE+B;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAK5C,QAAQ6C,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMZ,KAAKY,MAAiB;AAC1B,QAAI,KAAKnC,KAAKc,UAAU;AACtB;IACF;AACAV,wBAAI,QAAQ;MAAE+B;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKnB,IAAIvB,gBAAgB8B,KAAKY,IAAAA;EACtC;EAEAE,eAAeH,UAAoB;AACjC,QAAI,KAAKlC,KAAKc,UAAU;AACtB;IACF;AACA,SAAKP,iBAAiB2B;AACtB,SAAKzB,gBAAgB6B,SAAQ;EAC/B;EAEAX,mBAAmB;AACjB,QAAI,KAAK3B,KAAKc,UAAU;AACtB;IACF;AACA,SAAKK,QAAQmB,SAAQ;EACvB;EAEA,MAAcjB,gBAAgBkB,SAAS,GAAgC;AA7JzE;AA8JI,QAAI,KAAKvC,KAAKc,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKc,eAAepB,WAAS,UAAKoB,eAAepB,UAApB,mBAA2BgC,YAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKb,eAAepB;MAAOkC,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKxD,QAAQyD,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACAI,sCAAUJ,KAAKK,UAAQ,QAAA;;;;;;;;;AACvBD,sCAAUJ,KAAKM,WAAS,QAAA;;;;;;;;;AACxBF,sCAAUJ,KAAKP,QAAM,QAAA;;;;;;;;;AAErB,UAAIM,OAAOO,aAAaP,OAAOO,cAAcN,KAAKM,WAAW;AAC3DjD,uBAAIqB,KAAK,sBAAsB;UAAEqB;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMO,mBAAkBR,YAAOM,aAAPN,YAAmBS,qBAASC,KAAKT,KAAKP,SAASO,KAAKM,SAAS;AAErF,YAAMI,cAAcF,qBAASG,IAAIJ,iBAAiBP,KAAKK,QAAQ;AAC/D,YAAMO,eAAeJ,qBAASK,YAAYH,WAAAA,EAAaf,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWiB,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAKvE,QAAQyD,UAAUe,IAAIjB,OAAOI,IAAI;UAC5Dc,QAAQH,MAAMd,KAAKM;UACnBb,QAAQG,KAAKsB,IAAIlB,KAAKM,WAAWN,KAAKP,SAASqB,MAAMd,KAAKM,SAAS;QACrE,CAAA;AACAR,eAAOtB,KAAK;UACV2B,IAAIJ,OAAOI;UACXgB,aAAanB,KAAKP;UAClBa,WAAWN,KAAKM;UAChBc,aAAaN,MAAMd,KAAKM;UACxBe,SAASN;QACX,CAAA;AAEA,YAAIjB,OAAOL,UAAUD,QAAQ;AAC3B,iBAAOM;QACT;MACF;IACF;AAEA,WAAOA;EACT;AACF;;EA/EGwB;GAtGUjF,kBAAAA,WAAAA,QAAAA,IAAAA;;;AC3Bb,IAAAkF,gBAA0C;AAC1C,IAAAC,kBAA2C;AAC3C,IAAAC,oBAA0B;AAC1B,kBAA0B;AAC1B,IAAAC,cAAoB;AACpB,kBAAyB;AAEzB,IAAAC,eAAqC;;;;;;;;;;;;AAiBrC,IAAaC,WAAN,MAAAA,UAAA;EAULC,YAA6BC,SAAyB;mBAAzBA;SATZC,OAAO,IAAIC,wBAAAA;SACXC,QAAQ,IAAIC,mBAAAA;SAEZC,oBAAoB,IAAIC,wBAAwC,CAACC,QAChFC,sBAAUC,KAAKF,GAAAA,EAAKG,MAAK,CAAA;SAGVC,cAAc,oBAAIC,IAAAA;EAEoB;EAEvD,MAAMC,OAAO;EAAC;EAEd,MAAMC,QAAQ;AACZ,UAAM,KAAKb,KAAKc,QAAO;EACzB;;;;;;EAOA,MAAMC,SAASC,KAAcC,IAA+B;AAC1DC,yBAAI,YAAY;MAAED;IAAG,GAAA;;;;;;AACrB,UAAME,UAAU,MAAM,KAAKjB,MAAMkB,oBAAoB,YAAA;AACnD,YAAMC,kBAAkB,KAAKjB,kBAAkBkB,IAAIL,EAAAA;AAEnD,UAAII,iBAAiB;AACnBA,wBAAgBE;AAChB,eAAOF;MACT;AAEA,YAAMG,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUC,QAAQT,EAAAA;AAClD,YAAME,WAA2B;QAC/BQ,SAAS,IAAIC,sBAAAA;QACbL,SAAS;QACTM,MAAM;UACJZ;UACAa,WAAWN,6BAAMM;UACjBC,WAAUP,6BAAMO,aAAYC,WAAWxB,KAAKyB,sBAASC,OAAOV,KAAKO,QAAQ,CAAA;QAC3E;MACF;AAGA,WAAIP,6BAAMW,WAAUC,qBAASC,MAAMC,eAAe;AAChDnB,QAAAA,SAAQQ,QAAQY,KAAI;MACtB,OAAO;AACL,aAAKnC,kBAAkBoC,IAAIvB,IAAIE,QAAAA;AAC/B,aAAKsB,0BAAyB;MAChC;AAEA,aAAOtB;IACT,CAAA;AAEAH,+BAAK0B,UAAU,MACb,KAAKxC,MAAMkB,oBAAoB,YAAA;AAE7B,YAAMD,WAAU,KAAKf,kBAAkBkB,IAAIL,EAAAA;AAC3C,UAAI,CAACE,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQI,YAAY,GAAG;AAC3B,aAAKnB,kBAAkBuC,OAAO1B,EAAAA;MAChC;AACA,WAAKwB,0BAAyB;IAChC,CAAA;AAGF,WAAOzB,UAAM4B,mCAAkB5B,KAAKG,QAAQQ,QAAQkB,KAAI,CAAA,IAAM1B,QAAQQ,QAAQkB,KAAI;EACpF;EAEAC,kBAAkB;AAChB,UAAMC,YAAY,IAAIC,kBAAkB;MACtCvB,WAAW,KAAK1B,QAAQ0B;MACxBwB,QAAQ,YAAA;AACN/B,6BAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYwC,IAAIH,SAAAA;AACrBA,kBAAUI,eAAe,KAAKC,aAAY,CAAA;MAC5C;MACAC,SAAS,YAAA;AACPnC,6BAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYiC,OAAOI,SAAAA;MAC1B;MACAO,QAAQ,OAAOC,cAAAA;AAhHrB;AAiHQ,YAAI,CAAC,KAAKnD,kBAAkBoD,IAAID,UAAUtC,EAAE,GAAG;AAC7C;QACF;AACAC,6BAAI,YAAY;UAAEqC;QAAU,GAAA;;;;;;AAC5B,cAAM/B,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUgC,SAASF,SAAAA;AACnD,YAAI/B,KAAKW,UAAUC,qBAASC,MAAMC,eAAe;AAC/C,qBAAKlC,kBAAkBkB,IAAIiC,UAAUtC,EAAE,MAAvC,mBAA0CU,QAAQY;AAClD,eAAKnC,kBAAkBuC,OAAOY,UAAUtC,EAAE;QAC5C,OAAO;AACLyC,2CAAUlC,KAAKO,UAAQ,QAAA;;;;;;;;;AACvB,eAAK3B,kBAAkBkB,IAAIiC,UAAUtC,EAAE,EAAGY,KAAKE,WAAWE,sBAASC,OAAOV,KAAKO,QAAQ;QACzF;AAEA,aAAKU,0BAAyB;AAC9B,aAAKkB,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOZ;EACT;;;;EAKA,MAAMa,gBAAgBC,KAAiB;AACrC,SAAKF,kBAAiB;EACxB;EAEQP,eAAyB;AAC/B,WAAO;MACLU,OAAOC,MAAMvD,KAAK,KAAKJ,kBAAkB4D,OAAM,CAAA,EAAIC,IAAI,CAAC9C,YAAYA,QAAQU,IAAI;IAClF;EACF;EAEQ8B,oBAAoB;AAC1B,eAAWZ,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUmB,iBAAgB;IAC5B;EACF;EAEQzB,4BAA4B;AAClC,eAAWM,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUI,eAAe,KAAKC,aAAY,CAAA;IAC5C;EACF;AACF;AAjIavD,WAAAA,cAAAA;MADZsE,0BAAW,QAAQ,OAAA;GACPtE,QAAAA;;;ACxBb,uBAAiB;AAEjB,IAAAuE,gBAA6B;AAC7B,oBAA6B;AAC7B,IAAAC,oBAA0B;AAC1B,IAAAC,eAA0B;AAC1B,IAAAC,oBAAuB;AACvB,IAAAC,eAAyB;AAGzB,IAAAC,eAAwC;;;;;;;;;;;;AAOjC,IAAMC,qBAAqB;AAElC,IAAMC,gBAAgBC,yBAAOC,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;;EAEXC,YACmBC,YACjB;sBADiBA;EACf;EAEJ,MACMC,QAAQC,IAA+C;AAC3D,WAAO,KAAKC,SAASD,EAAAA;EACvB;;;;EAKA,MACME,IAAIF,IAAgBG,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKH,SAASD,EAAAA;AAErC,QAAI,CAACI,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAEC,SAAS,GAAGC,SAASH,SAASG,OAAM,IAAKJ;AAEjD,QAAIG,SAASC,SAASH,SAASG,QAAQ;AACrC,YAAM,IAAIF,MAAM,eAAA;IAClB;AAEA,QAAID,SAASI,UAAUC,sBAASC,MAAMC,eAAe;AACnD,YAAMC,QAAO,KAAKC,aAAab,EAAAA;AAC/B,aAAOY,MAAKE,KAAKR,QAAQC,MAAAA;IAC3B,WAAWJ,QAAQG,WAAWS,UAAaZ,QAAQI,WAAWQ,QAAW;AACvE,YAAM,IAAIV,MAAM,oBAAA;IAClB;AAEA,UAAMW,aAAaC,KAAKC,MAAMZ,SAASF,SAASe,SAAS;AACzD,UAAMC,WAAWH,KAAKI,MAAMf,SAASC,UAAUH,SAASe,SAAS;AAEjEG,qCAAUlB,SAASmB,UAAU,wBAAA;;;;;;;;;AAC7BD,qCAAUlB,SAASmB,SAAShB,SAAS,KAAKa,UAAU,2BAAA;;;;;;;;;AAEpD,UAAMI,UAAUC,sBAASC,MAAMtB,SAASmB,UAAUP,YAAYI,QAAAA,MAAcA,WAAWJ;AAEvF,QAAI,CAACQ,SAAS;AACZ,YAAM,IAAInB,MAAM,oBAAA;IAClB;AAEA,UAAMO,OAAO,KAAKC,aAAab,EAAAA;AAC/B,WAAOY,KAAKE,KAAKR,QAAQC,MAAAA;EAC3B;EAEA,MACMoB,OAA4B;AAQhC,UAAMC,QAAQ,IAAIC,KAAK,MAAM,KAAK/B,WAAW6B,KAAI,GAAIG,IAAI,CAACC,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWrB,QAAQgB,OAAO;AACxB,YAAM5B,KAAKkC,uBAAUC,KAAKvB,IAAAA,EAAMwB,aAAY;AAC5C,YAAMC,OAAO,MAAM,KAAKpC,SAASD,EAAAA;AACjC,UAAIqC,MAAM;AACRJ,YAAIK,KAAKD,IAAAA;MACX;IACF;AAEA,WAAOJ;EACT;EAEA,MACMM,IAAIC,MAAqC;AAC7C,UAAMxC,KAAK,IAAIyC,WAAW,MAAMC,2BAAaC,OAAO,WAAWH,IAAAA,CAAAA;AAC/D,UAAMjB,WAAWE,sBAASmB,KAAKJ,KAAKjC,SAASf,kBAAAA;AAE7C,UAAM6C,OAAiB;MACrBrC;MACAQ,OAAOC,sBAASC,MAAMC;MACtBJ,QAAQiC,KAAKjC;MACbY,WAAW3B;MACX+B;MACAsB,SAAS,IAAIC,KAAAA;MACbC,SAAS,IAAID,KAAAA;IACf;AAEA,UAAM,KAAKjC,aAAab,EAAAA,EAAIgD,MAAM,OAAGC,4BAAcT,IAAAA,CAAAA;AACnD,UAAM,KAAKU,WAAWlD,IAAIqC,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACMc,SAASC,OAAqC;AA1HtD;AA4HI,QAAIf,OAAO,MAAM,KAAKpC,SAASmD,MAAMpD,EAAE;AACvC,QAAI,CAACqC,MAAM;AACTf,uCAAU8B,MAAMC,aAAa,8BAAA;;;;;;;;;AAC7BhB,aAAO;QACLrC,IAAIoD,MAAMpD;QACVQ,OAAOC,sBAASC,MAAM4C;QACtB/C,QAAQ6C,MAAMC;QACdlC,YAAWiC,WAAMjC,cAANiC,YAAmB5D;QAC9BqD,SAAS,IAAIC,KAAAA;MACf;AACAT,WAAKd,WAAWE,sBAAS8B,MAAMlB,KAAK9B,SAAS8B,KAAKlB,SAAS;IAC7D;AAEA,QAAIiC,MAAMjC,aAAaiC,MAAMjC,cAAckB,KAAKlB,WAAW;AACzD,YAAM,IAAId,MAAM,oBAAA;IAClB;AAEAiB,qCAAUe,KAAKd,UAAU,wBAAA;;;;;;;;;AACzBD,qCAAU8B,MAAMI,gBAAgBzC,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAauC,MAAMpD,EAAE,EAAEgD,MAAMI,MAAMI,iBAAaP,4BAAcG,MAAMK,OAAO,CAAA;AAGtFhC,0BAASc,IAAIF,KAAKd,UAAUN,KAAKC,MAAMkC,MAAMI,cAAcnB,KAAKlB,SAAS,GAAG,IAAA;AAG5E,QAAIM,sBAASC,MAAMW,KAAKd,UAAU,GAAGc,KAAK9B,MAAM,IAAI8B,KAAKlB,aAAakB,KAAK9B,QAAQ;AACjF8B,WAAK7B,QAAQC,sBAASC,MAAMC;IAC9B;AACA0B,SAAKU,UAAU,IAAID,KAAAA;AAEnB,UAAM,KAAKI,WAAWE,MAAMpD,IAAIqC,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAca,WAAWlD,IAAgBqC,MAA+B;AACtE,UAAMqB,cAAUT,4BAAcxD,cAAckE,OAAOtB,IAAAA,CAAAA;AACnD,UAAMG,OAAOoB,OAAOC,MAAMH,QAAQnD,SAAS,CAAA;AAC3CiC,SAAKsB,cAAcJ,QAAQnD,QAAQ,CAAA;AACnCmD,YAAQK,KAAKvB,MAAM,CAAA;AAGnB,UAAM,KAAKwB,aAAahE,EAAAA,EAAIgD,MAAM,GAAGR,IAAAA;EACvC;EAEA,MAAcvC,SAASD,IAA+C;AACpE,UAAMY,OAAO,KAAKoD,aAAahE,EAAAA;AAC/B,UAAMiE,QAAQ,MAAMrD,KAAKsD,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAMzB,OAAO,MAAM5B,KAAKE,KAAK,GAAGmD,IAAAA;AAChC,UAAME,YAAY3B,KAAK4B,aAAa,CAAA;AACpC,WAAO3E,cAAc4E,OAAO7B,KAAK8B,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAahE,IAAgB;AACnC,WAAO,KAAKF,WAAWyE,gBAAgBC,iBAAAA,QAAKC,SAAKxB,4BAAcjD,EAAAA,EAAI0E,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQ7D,aAAab,IAAgB;AACnC,WAAO,KAAKF,WAAWyE,gBAAgBC,iBAAAA,QAAKC,SAAKxB,4BAAcjD,EAAAA,EAAI0E,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;AACF;;EA9JGC;GANU/E,UAAAA,WAAAA,WAAAA,IAAAA;;EAcV+E;GAdU/E,UAAAA,WAAAA,OAAAA,IAAAA;;EAmDV+E;GAnDU/E,UAAAA,WAAAA,QAAAA,IAAAA;;EA2EV+E;GA3EU/E,UAAAA,WAAAA,OAAAA,IAAAA;;EAgGV+E;GAhGU/E,UAAAA,WAAAA,YAAAA,IAAAA;",
|
|
6
|
+
"names": ["MIN_WANT_LIST_UPDATE_INTERVAL", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "RpcExtension", "constructor", "_params", "exposed", "BlobSyncService", "schema", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_ctx", "Context", "onError", "err", "log", "catch", "_lastWantListUpdate", "_localWantList", "blobs", "_updateWantList", "DeferredTask", "Date", "now", "sleep", "disposed", "list", "rpc", "want", "_currentUploads", "_upload", "blobChunks", "_pickBlobChunks", "blobChunk", "push", "RpcClosedError", "warn", "finally", "reconcileUploads", "remoteWantList", "onOpen", "context", "onClose", "dispose", "getHandlers", "wantList", "data", "onPush", "updateWantList", "schedule", "amount", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "invariant", "bitfield", "chunkSize", "requestBitfield", "BitField", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "synchronized", "import_async", "import_context", "import_invariant", "import_log", "import_util", "BlobSync", "constructor", "_params", "_ctx", "Context", "_lock", "Lock", "_downloadRequests", "ComplexMap", "key", "PublicKey", "from", "toHex", "_extensions", "Set", "open", "close", "dispose", "download", "ctx", "id", "log", "request", "executeSynchronized", "existingRequest", "get", "counter", "meta", "blobStore", "getMeta", "trigger", "Trigger", "want", "chunkSize", "bitfield", "Uint8Array", "BitField", "invert", "state", "BlobMeta", "State", "FULLY_PRESENT", "wake", "set", "_updateExtensionsWantList", "onDispose", "delete", "cancelWithContext", "wait", "createExtension", "extension", "BlobSyncExtension", "onOpen", "add", "updateWantList", "_getWantList", "onClose", "onPush", "blobChunk", "has", "setChunk", "invariant", "_reconcileUploads", "notifyBlobAdded", "_id", "blobs", "Array", "values", "map", "reconcileUploads", "trackLeaks", "import_async", "import_invariant", "import_keys", "import_protocols", "import_blob", "import_util", "DEFAULT_CHUNK_SIZE", "BlobMetaCodec", "schema", "getCodecForType", "BlobStore", "constructor", "_directory", "getMeta", "id", "_getMeta", "get", "options", "metadata", "Error", "offset", "length", "state", "BlobMeta", "State", "FULLY_PRESENT", "file", "_getDataFile", "read", "undefined", "beginChunk", "Math", "floor", "chunkSize", "endChunk", "ceil", "invariant", "bitfield", "present", "BitField", "count", "list", "files", "Set", "map", "f", "split", "res", "PublicKey", "from", "asUint8Array", "meta", "push", "set", "data", "Uint8Array", "subtleCrypto", "digest", "ones", "created", "Date", "updated", "write", "arrayToBuffer", "_writeMeta", "setChunk", "chunk", "totalLength", "PARTIALLY_PRESENT", "zeros", "chunkOffset", "payload", "encoded", "encode", "Buffer", "alloc", "writeUInt32LE", "copy", "_getMetaFile", "size", "stat", "protoSize", "readUInt32LE", "decode", "subarray", "getOrCreateFile", "path", "join", "toString", "synchronized"]
|
|
7
7
|
}
|
package/dist/lib/node/meta.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":
|
|
1
|
+
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":24170,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}]},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytes":17801,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"}]},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytes":22348,"imports":[{"path":"node:path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}]},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytes":703,"imports":[{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts","kind":"import-statement","original":"./blob-sync"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts","kind":"import-statement","original":"./blob-store"}]}},"outputs":{"packages/core/mesh/teleport-extension-object-sync/dist/lib/node/index.cjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":29937},"packages/core/mesh/teleport-extension-object-sync/dist/lib/node/index.cjs":{"imports":[{"path":"@dxos/async","kind":"require-call","external":true},{"path":"@dxos/context","kind":"require-call","external":true},{"path":"@dxos/invariant","kind":"require-call","external":true},{"path":"@dxos/log","kind":"require-call","external":true},{"path":"@dxos/protocols","kind":"require-call","external":true},{"path":"@dxos/teleport","kind":"require-call","external":true},{"path":"@dxos/util","kind":"require-call","external":true},{"path":"@dxos/async","kind":"require-call","external":true},{"path":"@dxos/context","kind":"require-call","external":true},{"path":"@dxos/invariant","kind":"require-call","external":true},{"path":"@dxos/keys","kind":"require-call","external":true},{"path":"@dxos/log","kind":"require-call","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"require-call","external":true},{"path":"@dxos/util","kind":"require-call","external":true},{"path":"node:path","kind":"require-call","external":true},{"path":"@dxos/async","kind":"require-call","external":true},{"path":"@dxos/crypto","kind":"require-call","external":true},{"path":"@dxos/invariant","kind":"require-call","external":true},{"path":"@dxos/keys","kind":"require-call","external":true},{"path":"@dxos/protocols","kind":"require-call","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"require-call","external":true},{"path":"@dxos/util","kind":"require-call","external":true}],"exports":[],"entryPoint":"packages/core/mesh/teleport-extension-object-sync/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytesInOutput":245},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytesInOutput":7537},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytesInOutput":5493},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytesInOutput":6933}},"bytes":22088}}}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blob-sync-extension.d.ts","sourceRoot":"","sources":["../../../src/blob-sync-extension.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"blob-sync-extension.d.ts","sourceRoot":"","sources":["../../../src/blob-sync-extension.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,mDAAmD,CAAC;AACzG,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,EAAE,SAAS,CAAC;IACrB,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5C,CAAC;AAMF;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,YAAY,CAAC,aAAa,EAAE,aAAa,CAAC;IAyD7E,OAAO,CAAC,QAAQ,CAAC,OAAO;IAxD1B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqD;IAE1E,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,cAAc,CAA2B;IAEjD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAY7B;IAEH,OAAO,CAAC,eAAe,CAAK;IAE5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CA2BrB;IAEH;;OAEG;IACI,cAAc,EAAE,QAAQ,CAAiB;gBAG7B,OAAO,EAAE,uBAAuB;IAgBpC,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhD,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;cAO9C,WAAW,IAAI,OAAO,CAAC,aAAa,CAAC;IAiB/C,IAAI,CAAC,IAAI,EAAE,SAAS;IAQ1B,cAAc,CAAC,QAAQ,EAAE,QAAQ;IAQjC,gBAAgB;YAOF,eAAe;CAuD9B;AAED,KAAK,aAAa,GAAG;IACnB,eAAe,EAAE,eAAe,CAAC;CAClC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blob-sync.d.ts","sourceRoot":"","sources":["../../../src/blob-sync.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"blob-sync.d.ts","sourceRoot":"","sources":["../../../src/blob-sync.ts"],"names":[],"mappings":"AAKA,OAAO,EAAqB,OAAO,EAAE,MAAM,eAAe,CAAC;AAQ3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE1D,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AASF,qBACa,QAAQ;IAUP,OAAO,CAAC,QAAQ,CAAC,OAAO;IATpC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IAEpC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAEhC;IAEF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgC;gBAE/B,OAAO,EAAE,cAAc;IAE9C,IAAI;IAEJ,KAAK;IAIX;;;;OAIG;IACG,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiD3D,eAAe;IAiCf;;OAEG;IACG,eAAe,CAAC,GAAG,EAAE,UAAU;IAIrC,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,yBAAyB;CAKlC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/teleport-extension-object-sync",
|
|
3
|
-
"version": "0.1.56-main.
|
|
3
|
+
"version": "0.1.56-main.a25bb93",
|
|
4
4
|
"description": "Teleport extension to synchronize opaque data objects.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -16,19 +16,19 @@
|
|
|
16
16
|
"src"
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"
|
|
20
|
-
"@dxos/
|
|
21
|
-
"@dxos/
|
|
22
|
-
"@dxos/
|
|
23
|
-
"@dxos/
|
|
24
|
-
"@dxos/keys": "0.1.56-main.
|
|
25
|
-
"@dxos/log": "0.1.56-main.
|
|
26
|
-
"@dxos/node-std": "0.1.56-main.
|
|
27
|
-
"@dxos/protocols": "0.1.56-main.
|
|
28
|
-
"@dxos/random-access-storage": "0.1.56-main.
|
|
29
|
-
"@dxos/rpc": "0.1.56-main.
|
|
30
|
-
"@dxos/teleport": "0.1.56-main.
|
|
31
|
-
"@dxos/util": "0.1.56-main.
|
|
19
|
+
"@dxos/async": "0.1.56-main.a25bb93",
|
|
20
|
+
"@dxos/context": "0.1.56-main.a25bb93",
|
|
21
|
+
"@dxos/crypto": "0.1.56-main.a25bb93",
|
|
22
|
+
"@dxos/debug": "0.1.56-main.a25bb93",
|
|
23
|
+
"@dxos/invariant": "0.1.56-main.a25bb93",
|
|
24
|
+
"@dxos/keys": "0.1.56-main.a25bb93",
|
|
25
|
+
"@dxos/log": "0.1.56-main.a25bb93",
|
|
26
|
+
"@dxos/node-std": "0.1.56-main.a25bb93",
|
|
27
|
+
"@dxos/protocols": "0.1.56-main.a25bb93",
|
|
28
|
+
"@dxos/random-access-storage": "0.1.56-main.a25bb93",
|
|
29
|
+
"@dxos/rpc": "0.1.56-main.a25bb93",
|
|
30
|
+
"@dxos/teleport": "0.1.56-main.a25bb93",
|
|
31
|
+
"@dxos/util": "0.1.56-main.a25bb93"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"typescript": "^5.0.4"
|
package/src/blob-store.ts
CHANGED
|
@@ -6,8 +6,8 @@ import path from 'node:path';
|
|
|
6
6
|
|
|
7
7
|
import { synchronized } from '@dxos/async';
|
|
8
8
|
import { subtleCrypto } from '@dxos/crypto';
|
|
9
|
+
import { invariant } from '@dxos/invariant';
|
|
9
10
|
import { PublicKey } from '@dxos/keys';
|
|
10
|
-
import { invariant } from '@dxos/log';
|
|
11
11
|
import { schema } from '@dxos/protocols';
|
|
12
12
|
import { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';
|
|
13
13
|
import { BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';
|
|
@@ -4,10 +4,9 @@
|
|
|
4
4
|
// Copyright 2023 DXOS.org
|
|
5
5
|
//
|
|
6
6
|
|
|
7
|
-
import invariant from 'tiny-invariant';
|
|
8
|
-
|
|
9
7
|
import { DeferredTask, sleep, synchronized } from '@dxos/async';
|
|
10
8
|
import { Context } from '@dxos/context';
|
|
9
|
+
import { invariant } from '@dxos/invariant';
|
|
11
10
|
import { log } from '@dxos/log';
|
|
12
11
|
import { RpcClosedError, schema } from '@dxos/protocols';
|
|
13
12
|
import { BlobChunk, BlobSyncService, WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';
|
package/src/blob-sync.ts
CHANGED
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
// Copyright 2023 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import invariant from 'tiny-invariant';
|
|
6
|
-
|
|
7
5
|
import { trackLeaks, Trigger, Lock } from '@dxos/async';
|
|
8
6
|
import { cancelWithContext, Context } from '@dxos/context';
|
|
7
|
+
import { invariant } from '@dxos/invariant';
|
|
9
8
|
import { PublicKey } from '@dxos/keys';
|
|
10
9
|
import { log } from '@dxos/log';
|
|
11
10
|
import { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';
|