@dxos/teleport-extension-object-sync 0.3.7 → 0.3.8-main.152ec6f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -18,7 +18,7 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
18
18
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
19
19
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
20
20
|
}
|
|
21
|
-
var __dxlog_file = "/home/
|
|
21
|
+
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
|
|
22
22
|
var MIN_WANT_LIST_UPDATE_INTERVAL = false ? 5 : 500;
|
|
23
23
|
var MAX_CONCURRENT_UPLOADS = 20;
|
|
24
24
|
var BlobSyncExtension = class extends RpcExtension {
|
|
@@ -279,7 +279,7 @@ function _ts_decorate2(decorators, target, key, desc) {
|
|
|
279
279
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
280
280
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
281
281
|
}
|
|
282
|
-
var __dxlog_file2 = "/home/
|
|
282
|
+
var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
|
|
283
283
|
var BlobSync = class BlobSync2 {
|
|
284
284
|
constructor(_params) {
|
|
285
285
|
this._params = _params;
|
|
@@ -444,7 +444,7 @@ function _ts_decorate3(decorators, target, key, desc) {
|
|
|
444
444
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
445
445
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
446
446
|
}
|
|
447
|
-
var __dxlog_file3 = "/home/
|
|
447
|
+
var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
|
|
448
448
|
var DEFAULT_CHUNK_SIZE = 4096;
|
|
449
449
|
var BlobMetaCodec = schema2.getCodecForType("dxos.echo.blob.BlobMeta");
|
|
450
450
|
var BlobStore = class {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":
|
|
1
|
+
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":24384,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytes":17817,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytes":22281,"imports":[{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytes":703,"imports":[{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts","kind":"import-statement","original":"./blob-sync"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts","kind":"import-statement","original":"./blob-store"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-object-sync/dist/lib/browser/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":29905},"packages/core/mesh/teleport-extension-object-sync/dist/lib/browser/index.mjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/node-std/path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["BlobStore","BlobSync","BlobSyncExtension","DEFAULT_CHUNK_SIZE"],"entryPoint":"packages/core/mesh/teleport-extension-object-sync/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytesInOutput":7121},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytesInOutput":0},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytesInOutput":5176},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytesInOutput":6547}},"bytes":19215}}}
|
package/dist/lib/node/index.cjs
CHANGED
|
@@ -26,18 +26,14 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
26
26
|
mod
|
|
27
27
|
));
|
|
28
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
var src_exports = {};
|
|
32
|
-
__export(src_exports, {
|
|
29
|
+
var node_exports = {};
|
|
30
|
+
__export(node_exports, {
|
|
33
31
|
BlobStore: () => BlobStore,
|
|
34
32
|
BlobSync: () => BlobSync,
|
|
35
33
|
BlobSyncExtension: () => BlobSyncExtension,
|
|
36
34
|
DEFAULT_CHUNK_SIZE: () => DEFAULT_CHUNK_SIZE
|
|
37
35
|
});
|
|
38
|
-
module.exports = __toCommonJS(
|
|
39
|
-
|
|
40
|
-
// packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts
|
|
36
|
+
module.exports = __toCommonJS(node_exports);
|
|
41
37
|
var import_async = require("@dxos/async");
|
|
42
38
|
var import_context = require("@dxos/context");
|
|
43
39
|
var import_invariant = require("@dxos/invariant");
|
|
@@ -45,6 +41,21 @@ var import_log = require("@dxos/log");
|
|
|
45
41
|
var import_protocols = require("@dxos/protocols");
|
|
46
42
|
var import_teleport = require("@dxos/teleport");
|
|
47
43
|
var import_util = require("@dxos/util");
|
|
44
|
+
var import_async2 = require("@dxos/async");
|
|
45
|
+
var import_context2 = require("@dxos/context");
|
|
46
|
+
var import_invariant2 = require("@dxos/invariant");
|
|
47
|
+
var import_keys = require("@dxos/keys");
|
|
48
|
+
var import_log2 = require("@dxos/log");
|
|
49
|
+
var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
50
|
+
var import_util2 = require("@dxos/util");
|
|
51
|
+
var import_node_path = __toESM(require("node:path"));
|
|
52
|
+
var import_async3 = require("@dxos/async");
|
|
53
|
+
var import_crypto = require("@dxos/crypto");
|
|
54
|
+
var import_invariant3 = require("@dxos/invariant");
|
|
55
|
+
var import_keys2 = require("@dxos/keys");
|
|
56
|
+
var import_protocols2 = require("@dxos/protocols");
|
|
57
|
+
var import_blob2 = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
58
|
+
var import_util3 = require("@dxos/util");
|
|
48
59
|
function _ts_decorate(decorators, target, key, desc) {
|
|
49
60
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
50
61
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -55,7 +66,7 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
55
66
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
56
67
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
57
68
|
}
|
|
58
|
-
var __dxlog_file = "/home/
|
|
69
|
+
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
|
|
59
70
|
var MIN_WANT_LIST_UPDATE_INTERVAL = process.env.NODE_ENV === "test" ? 5 : 500;
|
|
60
71
|
var MAX_CONCURRENT_UPLOADS = 20;
|
|
61
72
|
var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
@@ -297,15 +308,6 @@ var BlobSyncExtension = class extends import_teleport.RpcExtension {
|
|
|
297
308
|
_ts_decorate([
|
|
298
309
|
import_async.synchronized
|
|
299
310
|
], BlobSyncExtension.prototype, "push", null);
|
|
300
|
-
|
|
301
|
-
// packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts
|
|
302
|
-
var import_async2 = require("@dxos/async");
|
|
303
|
-
var import_context2 = require("@dxos/context");
|
|
304
|
-
var import_invariant2 = require("@dxos/invariant");
|
|
305
|
-
var import_keys = require("@dxos/keys");
|
|
306
|
-
var import_log2 = require("@dxos/log");
|
|
307
|
-
var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
308
|
-
var import_util2 = require("@dxos/util");
|
|
309
311
|
function _ts_decorate2(decorators, target, key, desc) {
|
|
310
312
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
311
313
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -316,7 +318,7 @@ function _ts_decorate2(decorators, target, key, desc) {
|
|
|
316
318
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
317
319
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
318
320
|
}
|
|
319
|
-
var __dxlog_file2 = "/home/
|
|
321
|
+
var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
|
|
320
322
|
var BlobSync = class BlobSync2 {
|
|
321
323
|
constructor(_params) {
|
|
322
324
|
this._params = _params;
|
|
@@ -461,16 +463,6 @@ var BlobSync = class BlobSync2 {
|
|
|
461
463
|
BlobSync = _ts_decorate2([
|
|
462
464
|
(0, import_async2.trackLeaks)("open", "close")
|
|
463
465
|
], BlobSync);
|
|
464
|
-
|
|
465
|
-
// packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts
|
|
466
|
-
var import_node_path = __toESM(require("node:path"));
|
|
467
|
-
var import_async3 = require("@dxos/async");
|
|
468
|
-
var import_crypto = require("@dxos/crypto");
|
|
469
|
-
var import_invariant3 = require("@dxos/invariant");
|
|
470
|
-
var import_keys2 = require("@dxos/keys");
|
|
471
|
-
var import_protocols2 = require("@dxos/protocols");
|
|
472
|
-
var import_blob2 = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
473
|
-
var import_util3 = require("@dxos/util");
|
|
474
466
|
function _ts_decorate3(decorators, target, key, desc) {
|
|
475
467
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
476
468
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -481,7 +473,7 @@ function _ts_decorate3(decorators, target, key, desc) {
|
|
|
481
473
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
482
474
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
483
475
|
}
|
|
484
|
-
var __dxlog_file3 = "/home/
|
|
476
|
+
var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
|
|
485
477
|
var DEFAULT_CHUNK_SIZE = 4096;
|
|
486
478
|
var BlobMetaCodec = import_protocols2.schema.getCodecForType("dxos.echo.blob.BlobMeta");
|
|
487
479
|
var BlobStore = class {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
|
-
"sources": ["../../../src/
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2022 DXOS.org\n//\n\nexport * from './blob-sync-extension';\nexport * from './blob-sync';\nexport * from './blob-store';\n", "//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { RpcClosedError, schema } from '@dxos/protocols';\nimport { type BlobChunk, type BlobSyncService, type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = process.env.NODE_ENV === 'test' ? 5 : 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk) {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList) {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads() {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { trackLeaks, Trigger, Lock } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _lock = new Lock();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open() {}\n\n async close() {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._lock.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._lock.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension() {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array) {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads() {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList() {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { schema } from '@dxos/protocols';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n constructor(private readonly _directory: Directory) {}\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;;;ACMA,mBAAkD;AAClD,qBAAwB;AACxB,uBAA0B;AAC1B,iBAAoB;AACpB,uBAAuC;AAEvC,sBAAoD;AACpD,kBAAyB;;;;;;;;;;;;AAWzB,IAAMA,gCAAgCC,QAAQC,IAAIC,aAAa,SAAS,IAAI;AAE5E,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCC,6BAAAA;EAwDrCC,YACmBC,SACjB;AACA,UAAM;MACJC,SAAS;QACPC,iBAAiBC,wBAAOC,WAAW,6CAAA;MACrC;MACAC,WAAW;QACTH,iBAAiBC,wBAAOC,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA;mBAbiBR;SAxDFS,OAAO,IAAIC,uBAAQ;MAAEC,SAAS,CAACC,QAAQC,eAAIC,MAAMF,KAAAA,QAAAA;;;;;;IAAK,CAAA;SAE/DG,sBAAsB;SACtBC,iBAA2B;MAAEC,OAAO,CAAA;IAAG;SAE9BC,kBAAkB,IAAIC,0BAAa,KAAKV,MAAM,YAAA;AAE7D,UAAI,KAAKM,sBAAsBvB,gCAAgC4B,KAAKC,IAAG,GAAI;AACzE,kBAAMC,oBAAM,KAAKP,sBAAsBvB,gCAAgC4B,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAKZ,KAAKc,UAAU;AACtB;QACF;MACF;AAEAV,0BAAI,QAAQ;QAAEW,MAAM,KAAKR;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKS,IAAIvB,gBAAgBwB,KAAK,KAAKV,cAAc;AACvD,WAAKD,sBAAsBK,KAAKC,IAAG;IACrC,CAAA;SAEQM,kBAAkB;SAETC,UAAU,IAAIT,0BAAa,KAAKV,MAAM,YAAA;AACrD,UAAI,KAAKkB,mBAAmB/B,wBAAwB;AAClD;MACF;AACA,YAAMiC,aAAa,MAAM,KAAKC,gBAAgBlC,yBAAyB,KAAK+B,eAAe;AAC3F,UAAI,CAACE,YAAY;AACf;MACF;AACA,iBAAWE,aAAaF,YAAY;AAClC,YAAI,KAAKpB,KAAKc,UAAU;AACtB;QACF;AAEA,aAAKI;AAEL,aAAKK,KAAKD,SAAAA,EACPjB,MAAM,CAACF,QAAAA;AACN,cAAIA,eAAeqB,iCAAgB;AACjC;UACF;AACApB,yBAAIqB,KAAK,eAAe;YAAEtB;UAAI,GAAA;;;;;;QAChC,CAAA,EACCuB,QAAQ,MAAA;AACP,eAAKR;AACL,eAAKS,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA;SAKOC,iBAA2B;MAAEpB,OAAO,CAAA;IAAG;EAiB9C;EAEA,MAAeqB,OAAOC,SAA0C;AAC9D1B,wBAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMyB,OAAOC,OAAAA;AACnB,UAAM,KAAKvC,QAAQsC,OAAM;EAC3B;EAEA,MAAeE,QAAQ5B,KAAwC;AAC7DC,wBAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKJ,KAAKgC,QAAO;AACvB,UAAM,KAAKzC,QAAQwC,QAAO;AAC1B,UAAM,MAAMA,QAAQ5B,GAAAA;EACtB;EAEA,MAAgB8B,cAAsC;AACpD,WAAO;MACLxC,iBAAiB;QACfwB,MAAM,OAAOiB,aAAAA;AACX9B,8BAAI,eAAe;YAAEwB,gBAAgBM;UAAS,GAAA;;;;;;AAC9C,eAAKN,iBAAiBM;AACtB,eAAKP,iBAAgB;QACvB;QACAJ,MAAM,OAAOY,SAAAA;AACX/B,8BAAI,YAAY;YAAE+B;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAK5C,QAAQ6C,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMZ,KAAKY,MAAiB;AAC1B,QAAI,KAAKnC,KAAKc,UAAU;AACtB;IACF;AACAV,wBAAI,QAAQ;MAAE+B;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKnB,IAAIvB,gBAAgB8B,KAAKY,IAAAA;EACtC;EAEAE,eAAeH,UAAoB;AACjC,QAAI,KAAKlC,KAAKc,UAAU;AACtB;IACF;AACA,SAAKP,iBAAiB2B;AACtB,SAAKzB,gBAAgB6B,SAAQ;EAC/B;EAEAX,mBAAmB;AACjB,QAAI,KAAK3B,KAAKc,UAAU;AACtB;IACF;AACA,SAAKK,QAAQmB,SAAQ;EACvB;EAEA,MAAcjB,gBAAgBkB,SAAS,GAAgC;AACrE,QAAI,KAAKvC,KAAKc,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKc,eAAepB,SAAS,KAAKoB,eAAepB,OAAOgC,WAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKb,eAAepB;MAAOkC,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKxD,QAAQyD,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACAI,sCAAUJ,KAAKK,UAAQ,QAAA;;;;;;;;;AACvBD,sCAAUJ,KAAKM,WAAS,QAAA;;;;;;;;;AACxBF,sCAAUJ,KAAKP,QAAM,QAAA;;;;;;;;;AAErB,UAAIM,OAAOO,aAAaP,OAAOO,cAAcN,KAAKM,WAAW;AAC3DjD,uBAAIqB,KAAK,sBAAsB;UAAEqB;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMO,kBAAkBR,OAAOM,YAAYG,qBAASC,KAAKT,KAAKP,SAASO,KAAKM,SAAS;AAErF,YAAMI,cAAcF,qBAASG,IAAIJ,iBAAiBP,KAAKK,QAAQ;AAC/D,YAAMO,eAAeJ,qBAASK,YAAYH,WAAAA,EAAaf,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWiB,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAKvE,QAAQyD,UAAUe,IAAIjB,OAAOI,IAAI;UAC5Dc,QAAQH,MAAMd,KAAKM;UACnBb,QAAQG,KAAKsB,IAAIlB,KAAKM,WAAWN,KAAKP,SAASqB,MAAMd,KAAKM,SAAS;QACrE,CAAA;AACAR,eAAOtB,KAAK;UACV2B,IAAIJ,OAAOI;UACXgB,aAAanB,KAAKP;UAClBa,WAAWN,KAAKM;UAChBc,aAAaN,MAAMd,KAAKM;UACxBe,SAASN;QACX,CAAA;AAEA,YAAIjB,OAAOL,UAAUD,QAAQ;AAC3B,iBAAOM;QACT;MACF;IACF;AAEA,WAAOA;EACT;AACF;;EA/EGwB;GAtGUjF,kBAAAA,WAAAA,QAAAA,IAAAA;;;AC3Bb,IAAAkF,gBAA0C;AAC1C,IAAAC,kBAA2C;AAC3C,IAAAC,oBAA0B;AAC1B,kBAA0B;AAC1B,IAAAC,cAAoB;AACpB,kBAAyB;AAEzB,IAAAC,eAAqC;;;;;;;;;;;;AAiBrC,IAAaC,WAAN,MAAAA,UAAA;EAULC,YAA6BC,SAAyB;mBAAzBA;SATZC,OAAO,IAAIC,wBAAAA;SACXC,QAAQ,IAAIC,mBAAAA;SAEZC,oBAAoB,IAAIC,wBAAwC,CAACC,QAChFC,sBAAUC,KAAKF,GAAAA,EAAKG,MAAK,CAAA;SAGVC,cAAc,oBAAIC,IAAAA;EAEoB;EAEvD,MAAMC,OAAO;EAAC;EAEd,MAAMC,QAAQ;AACZ,UAAM,KAAKb,KAAKc,QAAO;EACzB;;;;;;EAOA,MAAMC,SAASC,KAAcC,IAA+B;AAC1DC,yBAAI,YAAY;MAAED;IAAG,GAAA;;;;;;AACrB,UAAME,UAAU,MAAM,KAAKjB,MAAMkB,oBAAoB,YAAA;AACnD,YAAMC,kBAAkB,KAAKjB,kBAAkBkB,IAAIL,EAAAA;AAEnD,UAAII,iBAAiB;AACnBA,wBAAgBE;AAChB,eAAOF;MACT;AAEA,YAAMG,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUC,QAAQT,EAAAA;AAClD,YAAME,WAA2B;QAC/BQ,SAAS,IAAIC,sBAAAA;QACbL,SAAS;QACTM,MAAM;UACJZ;UACAa,WAAWN,MAAMM;UACjBC,UAAUP,MAAMO,YAAYC,WAAWxB,KAAKyB,sBAASC,OAAOV,KAAKO,QAAQ,CAAA;QAC3E;MACF;AAGA,UAAIP,MAAMW,UAAUC,qBAASC,MAAMC,eAAe;AAChDnB,QAAAA,SAAQQ,QAAQY,KAAI;MACtB,OAAO;AACL,aAAKnC,kBAAkBoC,IAAIvB,IAAIE,QAAAA;AAC/B,aAAKsB,0BAAyB;MAChC;AAEA,aAAOtB;IACT,CAAA;AAEAH,SAAK0B,UAAU,MACb,KAAKxC,MAAMkB,oBAAoB,YAAA;AAE7B,YAAMD,WAAU,KAAKf,kBAAkBkB,IAAIL,EAAAA;AAC3C,UAAI,CAACE,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQI,YAAY,GAAG;AAC3B,aAAKnB,kBAAkBuC,OAAO1B,EAAAA;MAChC;AACA,WAAKwB,0BAAyB;IAChC,CAAA,CAAA;AAGF,WAAOzB,UAAM4B,mCAAkB5B,KAAKG,QAAQQ,QAAQkB,KAAI,CAAA,IAAM1B,QAAQQ,QAAQkB,KAAI;EACpF;EAEAC,kBAAkB;AAChB,UAAMC,YAAY,IAAIC,kBAAkB;MACtCvB,WAAW,KAAK1B,QAAQ0B;MACxBwB,QAAQ,YAAA;AACN/B,6BAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYwC,IAAIH,SAAAA;AACrBA,kBAAUI,eAAe,KAAKC,aAAY,CAAA;MAC5C;MACAC,SAAS,YAAA;AACPnC,6BAAI,oBAAA,QAAA;;;;;;AACJ,aAAKR,YAAYiC,OAAOI,SAAAA;MAC1B;MACAO,QAAQ,OAAOC,cAAAA;AACb,YAAI,CAAC,KAAKnD,kBAAkBoD,IAAID,UAAUtC,EAAE,GAAG;AAC7C;QACF;AACAC,6BAAI,YAAY;UAAEqC;QAAU,GAAA;;;;;;AAC5B,cAAM/B,OAAO,MAAM,KAAKzB,QAAQ0B,UAAUgC,SAASF,SAAAA;AACnD,YAAI/B,KAAKW,UAAUC,qBAASC,MAAMC,eAAe;AAC/C,eAAKlC,kBAAkBkB,IAAIiC,UAAUtC,EAAE,GAAGU,QAAQY,KAAAA;AAClD,eAAKnC,kBAAkBuC,OAAOY,UAAUtC,EAAE;QAC5C,OAAO;AACLyC,2CAAUlC,KAAKO,UAAQ,QAAA;;;;;;;;;AACvB,eAAK3B,kBAAkBkB,IAAIiC,UAAUtC,EAAE,EAAGY,KAAKE,WAAWE,sBAASC,OAAOV,KAAKO,QAAQ;QACzF;AAEA,aAAKU,0BAAyB;AAC9B,aAAKkB,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOZ;EACT;;;;EAKA,MAAMa,gBAAgBC,KAAiB;AACrC,SAAKF,kBAAiB;EACxB;EAEQP,eAAyB;AAC/B,WAAO;MACLU,OAAOC,MAAMvD,KAAK,KAAKJ,kBAAkB4D,OAAM,CAAA,EAAIC,IAAI,CAAC9C,YAAYA,QAAQU,IAAI;IAClF;EACF;EAEQ8B,oBAAoB;AAC1B,eAAWZ,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUmB,iBAAgB;IAC5B;EACF;EAEQzB,4BAA4B;AAClC,eAAWM,aAAa,KAAKrC,aAAa;AACxCqC,gBAAUI,eAAe,KAAKC,aAAY,CAAA;IAC5C;EACF;AACF;AAjIavD,WAAAA,cAAAA;MADZsE,0BAAW,QAAQ,OAAA;GACPtE,QAAAA;;;ACxBb,uBAAiB;AAEjB,IAAAuE,gBAA6B;AAC7B,oBAA6B;AAC7B,IAAAC,oBAA0B;AAC1B,IAAAC,eAA0B;AAC1B,IAAAC,oBAAuB;AACvB,IAAAC,eAAyB;AAGzB,IAAAC,eAAwC;;;;;;;;;;;;AAOjC,IAAMC,qBAAqB;AAElC,IAAMC,gBAAgBC,yBAAOC,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;EACXC,YAA6BC,YAAuB;sBAAvBA;EAAwB;EAErD,MACMC,QAAQC,IAA+C;AAC3D,WAAO,KAAKC,SAASD,EAAAA;EACvB;;;;EAKA,MACME,IAAIF,IAAgBG,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKH,SAASD,EAAAA;AAErC,QAAI,CAACI,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAEC,SAAS,GAAGC,SAASH,SAASG,OAAM,IAAKJ;AAEjD,QAAIG,SAASC,SAASH,SAASG,QAAQ;AACrC,YAAM,IAAIF,MAAM,eAAA;IAClB;AAEA,QAAID,SAASI,UAAUC,sBAASC,MAAMC,eAAe;AACnD,YAAMC,QAAO,KAAKC,aAAab,EAAAA;AAC/B,aAAOY,MAAKE,KAAKR,QAAQC,MAAAA;IAC3B,WAAWJ,QAAQG,WAAWS,UAAaZ,QAAQI,WAAWQ,QAAW;AACvE,YAAM,IAAIV,MAAM,oBAAA;IAClB;AAEA,UAAMW,aAAaC,KAAKC,MAAMZ,SAASF,SAASe,SAAS;AACzD,UAAMC,WAAWH,KAAKI,MAAMf,SAASC,UAAUH,SAASe,SAAS;AAEjEG,qCAAUlB,SAASmB,UAAU,wBAAA;;;;;;;;;AAC7BD,qCAAUlB,SAASmB,SAAShB,SAAS,KAAKa,UAAU,2BAAA;;;;;;;;;AAEpD,UAAMI,UAAUC,sBAASC,MAAMtB,SAASmB,UAAUP,YAAYI,QAAAA,MAAcA,WAAWJ;AAEvF,QAAI,CAACQ,SAAS;AACZ,YAAM,IAAInB,MAAM,oBAAA;IAClB;AAEA,UAAMO,OAAO,KAAKC,aAAab,EAAAA;AAC/B,WAAOY,KAAKE,KAAKR,QAAQC,MAAAA;EAC3B;EAEA,MACMoB,OAA4B;AAQhC,UAAMC,QAAQ,IAAIC,KAAK,MAAM,KAAK/B,WAAW6B,KAAI,GAAIG,IAAI,CAACC,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWrB,QAAQgB,OAAO;AACxB,YAAM5B,KAAKkC,uBAAUC,KAAKvB,IAAAA,EAAMwB,aAAY;AAC5C,YAAMC,OAAO,MAAM,KAAKpC,SAASD,EAAAA;AACjC,UAAIqC,MAAM;AACRJ,YAAIK,KAAKD,IAAAA;MACX;IACF;AAEA,WAAOJ;EACT;EAEA,MACMM,IAAIC,MAAqC;AAC7C,UAAMxC,KAAK,IAAIyC,WAAW,MAAMC,2BAAaC,OAAO,WAAWH,IAAAA,CAAAA;AAC/D,UAAMjB,WAAWE,sBAASmB,KAAKJ,KAAKjC,SAASf,kBAAAA;AAE7C,UAAM6C,OAAiB;MACrBrC;MACAQ,OAAOC,sBAASC,MAAMC;MACtBJ,QAAQiC,KAAKjC;MACbY,WAAW3B;MACX+B;MACAsB,SAAS,oBAAIC,KAAAA;MACbC,SAAS,oBAAID,KAAAA;IACf;AAEA,UAAM,KAAKjC,aAAab,EAAAA,EAAIgD,MAAM,OAAGC,4BAAcT,IAAAA,CAAAA;AACnD,UAAM,KAAKU,WAAWlD,IAAIqC,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACMc,SAASC,OAAqC;AAElD,QAAIf,OAAO,MAAM,KAAKpC,SAASmD,MAAMpD,EAAE;AACvC,QAAI,CAACqC,MAAM;AACTf,uCAAU8B,MAAMC,aAAa,8BAAA;;;;;;;;;AAC7BhB,aAAO;QACLrC,IAAIoD,MAAMpD;QACVQ,OAAOC,sBAASC,MAAM4C;QACtB/C,QAAQ6C,MAAMC;QACdlC,WAAWiC,MAAMjC,aAAa3B;QAC9BqD,SAAS,oBAAIC,KAAAA;MACf;AACAT,WAAKd,WAAWE,sBAAS8B,MAAMlB,KAAK9B,SAAS8B,KAAKlB,SAAS;IAC7D;AAEA,QAAIiC,MAAMjC,aAAaiC,MAAMjC,cAAckB,KAAKlB,WAAW;AACzD,YAAM,IAAId,MAAM,oBAAA;IAClB;AAEAiB,qCAAUe,KAAKd,UAAU,wBAAA;;;;;;;;;AACzBD,qCAAU8B,MAAMI,gBAAgBzC,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAauC,MAAMpD,EAAE,EAAEgD,MAAMI,MAAMI,iBAAaP,4BAAcG,MAAMK,OAAO,CAAA;AAGtFhC,0BAASc,IAAIF,KAAKd,UAAUN,KAAKC,MAAMkC,MAAMI,cAAcnB,KAAKlB,SAAS,GAAG,IAAA;AAG5E,QAAIM,sBAASC,MAAMW,KAAKd,UAAU,GAAGc,KAAK9B,MAAM,IAAI8B,KAAKlB,aAAakB,KAAK9B,QAAQ;AACjF8B,WAAK7B,QAAQC,sBAASC,MAAMC;IAC9B;AACA0B,SAAKU,UAAU,oBAAID,KAAAA;AAEnB,UAAM,KAAKI,WAAWE,MAAMpD,IAAIqC,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAca,WAAWlD,IAAgBqC,MAA+B;AACtE,UAAMqB,cAAUT,4BAAcxD,cAAckE,OAAOtB,IAAAA,CAAAA;AACnD,UAAMG,OAAOoB,OAAOC,MAAMH,QAAQnD,SAAS,CAAA;AAC3CiC,SAAKsB,cAAcJ,QAAQnD,QAAQ,CAAA;AACnCmD,YAAQK,KAAKvB,MAAM,CAAA;AAGnB,UAAM,KAAKwB,aAAahE,EAAAA,EAAIgD,MAAM,GAAGR,IAAAA;EACvC;EAEA,MAAcvC,SAASD,IAA+C;AACpE,UAAMY,OAAO,KAAKoD,aAAahE,EAAAA;AAC/B,UAAMiE,QAAQ,MAAMrD,KAAKsD,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAMzB,OAAO,MAAM5B,KAAKE,KAAK,GAAGmD,IAAAA;AAChC,UAAME,YAAY3B,KAAK4B,aAAa,CAAA;AACpC,WAAO3E,cAAc4E,OAAO7B,KAAK8B,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAahE,IAAgB;AACnC,WAAO,KAAKF,WAAWyE,gBAAgBC,iBAAAA,QAAKC,SAAKxB,4BAAcjD,EAAAA,EAAI0E,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQ7D,aAAab,IAAgB;AACnC,WAAO,KAAKF,WAAWyE,gBAAgBC,iBAAAA,QAAKC,SAAKxB,4BAAcjD,EAAAA,EAAI0E,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;AACF;;EA9JGC;GAHU/E,UAAAA,WAAAA,WAAAA,IAAAA;;EAWV+E;GAXU/E,UAAAA,WAAAA,OAAAA,IAAAA;;EAgDV+E;GAhDU/E,UAAAA,WAAAA,QAAAA,IAAAA;;EAwEV+E;GAxEU/E,UAAAA,WAAAA,OAAAA,IAAAA;;EA6FV+E;GA7FU/E,UAAAA,WAAAA,YAAAA,IAAAA;",
|
|
6
|
-
"names": ["MIN_WANT_LIST_UPDATE_INTERVAL", "process", "env", "NODE_ENV", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "RpcExtension", "constructor", "_params", "exposed", "BlobSyncService", "schema", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_ctx", "Context", "onError", "err", "log", "catch", "_lastWantListUpdate", "_localWantList", "blobs", "_updateWantList", "DeferredTask", "Date", "now", "sleep", "disposed", "list", "rpc", "want", "_currentUploads", "_upload", "blobChunks", "_pickBlobChunks", "blobChunk", "push", "RpcClosedError", "warn", "finally", "reconcileUploads", "remoteWantList", "onOpen", "context", "onClose", "dispose", "getHandlers", "wantList", "data", "onPush", "updateWantList", "schedule", "amount", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "invariant", "bitfield", "chunkSize", "requestBitfield", "BitField", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "synchronized", "
|
|
3
|
+
"sources": ["../../../src/blob-sync-extension.ts", "../../../src/blob-sync.ts", "../../../src/blob-store.ts"],
|
|
4
|
+
"sourcesContent": ["//\n\n//\n// Copyright 2023 DXOS.org\n//\n\nimport { DeferredTask, sleep, synchronized } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { RpcClosedError, schema } from '@dxos/protocols';\nimport { type BlobChunk, type BlobSyncService, type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type ExtensionContext, RpcExtension } from '@dxos/teleport';\nimport { BitField } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\n\nexport type BlobSyncExtensionParams = {\n blobStore: BlobStore;\n onOpen: () => Promise<void>;\n onClose: () => Promise<void>;\n onPush: (data: BlobChunk) => Promise<void>;\n};\n\nconst MIN_WANT_LIST_UPDATE_INTERVAL = process.env.NODE_ENV === 'test' ? 5 : 500;\n\nconst MAX_CONCURRENT_UPLOADS = 20;\n\n/**\n * Manages replication between a set of feeds for a single teleport session.\n */\nexport class BlobSyncExtension extends RpcExtension<ServiceBundle, ServiceBundle> {\n private readonly _ctx = new Context({ onError: (err) => log.catch(err) });\n\n private _lastWantListUpdate = 0;\n private _localWantList: WantList = { blobs: [] };\n\n private readonly _updateWantList = new DeferredTask(this._ctx, async () => {\n // Throttle want list updates.\n if (this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL > Date.now()) {\n await sleep(this._lastWantListUpdate + MIN_WANT_LIST_UPDATE_INTERVAL - Date.now());\n if (this._ctx.disposed) {\n return;\n }\n }\n\n log('want', { list: this._localWantList });\n await this.rpc.BlobSyncService.want(this._localWantList);\n this._lastWantListUpdate = Date.now();\n });\n\n private _currentUploads = 0;\n\n private readonly _upload = new DeferredTask(this._ctx, async () => {\n if (this._currentUploads >= MAX_CONCURRENT_UPLOADS) {\n return;\n }\n const blobChunks = await this._pickBlobChunks(MAX_CONCURRENT_UPLOADS - this._currentUploads);\n if (!blobChunks) {\n return;\n }\n for (const blobChunk of blobChunks) {\n if (this._ctx.disposed) {\n break;\n }\n\n this._currentUploads++;\n\n this.push(blobChunk)\n .catch((err) => {\n if (err instanceof RpcClosedError) {\n return;\n }\n log.warn('push failed', { err });\n })\n .finally(() => {\n this._currentUploads--;\n this.reconcileUploads();\n });\n }\n });\n\n /**\n * Set of id's remote peer wants.\n */\n public remoteWantList: WantList = { blobs: [] };\n\n constructor(\n private readonly _params: BlobSyncExtensionParams, // to not conflict with the base class\n ) {\n super({\n exposed: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n requested: {\n BlobSyncService: schema.getService('dxos.mesh.teleport.blobsync.BlobSyncService'),\n },\n timeout: 20_000,\n encodingOptions: {\n preserveAny: true,\n },\n });\n }\n\n override async onOpen(context: ExtensionContext): Promise<void> {\n log('open');\n await super.onOpen(context);\n await this._params.onOpen();\n }\n\n override async onClose(err?: Error | undefined): Promise<void> {\n log('close');\n await this._ctx.dispose();\n await this._params.onClose();\n await super.onClose(err);\n }\n\n protected async getHandlers(): Promise<ServiceBundle> {\n return {\n BlobSyncService: {\n want: async (wantList) => {\n log('remote want', { remoteWantList: wantList });\n this.remoteWantList = wantList;\n this.reconcileUploads();\n },\n push: async (data) => {\n log('received', { data });\n await this._params.onPush(data);\n },\n },\n };\n }\n\n @synchronized\n async push(data: BlobChunk) {\n if (this._ctx.disposed) {\n return;\n }\n log('push', { data });\n await this.rpc.BlobSyncService.push(data);\n }\n\n updateWantList(wantList: WantList) {\n if (this._ctx.disposed) {\n return;\n }\n this._localWantList = wantList;\n this._updateWantList.schedule();\n }\n\n reconcileUploads() {\n if (this._ctx.disposed) {\n return;\n }\n this._upload.schedule();\n }\n\n private async _pickBlobChunks(amount = 1): Promise<BlobChunk[] | void> {\n if (this._ctx.disposed) {\n return;\n }\n\n if (!this.remoteWantList.blobs || this.remoteWantList.blobs?.length === 0) {\n return;\n }\n\n const shuffled = [...this.remoteWantList.blobs].sort(() => Math.random() - 0.5);\n\n const chunks: BlobChunk[] = [];\n\n for (const header of shuffled) {\n const meta = await this._params.blobStore.getMeta(header.id);\n\n if (!meta) {\n // Skip this header\n continue;\n }\n invariant(meta.bitfield);\n invariant(meta.chunkSize);\n invariant(meta.length);\n\n if (header.chunkSize && header.chunkSize !== meta.chunkSize) {\n log.warn('Invalid chunk size', { header, meta });\n continue;\n }\n\n const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);\n\n const presentData = BitField.and(requestBitfield, meta.bitfield);\n const chunkIndices = BitField.findIndexes(presentData).sort(() => Math.random() - 0.5);\n\n for (const idx of chunkIndices) {\n const chunkData = await this._params.blobStore.get(header.id, {\n offset: idx * meta.chunkSize,\n length: Math.min(meta.chunkSize, meta.length - idx * meta.chunkSize),\n });\n chunks.push({\n id: header.id,\n totalLength: meta.length,\n chunkSize: meta.chunkSize,\n chunkOffset: idx * meta.chunkSize,\n payload: chunkData,\n });\n\n if (chunks.length >= amount) {\n return chunks;\n }\n }\n }\n\n return chunks;\n }\n}\n\ntype ServiceBundle = {\n BlobSyncService: BlobSyncService;\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { trackLeaks, Trigger, Lock } from '@dxos/async';\nimport { cancelWithContext, Context } from '@dxos/context';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type WantList } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { BitField, ComplexMap } from '@dxos/util';\n\nimport { type BlobStore } from './blob-store';\nimport { BlobSyncExtension } from './blob-sync-extension';\n\nexport type BlobSyncParams = {\n blobStore: BlobStore;\n};\n\ntype DownloadRequest = {\n trigger: Trigger<void>;\n counter: number;\n want: WantList.Entry;\n};\n\n// TODO(dmaretskyi): Rename to blob-sync.\n@trackLeaks('open', 'close')\nexport class BlobSync {\n private readonly _ctx = new Context();\n private readonly _lock = new Lock();\n\n private readonly _downloadRequests = new ComplexMap<Uint8Array, DownloadRequest>((key) =>\n PublicKey.from(key).toHex(),\n );\n\n private readonly _extensions = new Set<BlobSyncExtension>();\n\n constructor(private readonly _params: BlobSyncParams) {}\n\n async open() {}\n\n async close() {\n await this._ctx.dispose();\n }\n\n /**\n * Resolves when the object with the given id is fully downloaded in the blob store.\n *\n * @param id hex-encoded id of the object to download.\n */\n async download(ctx: Context, id: Uint8Array): Promise<void> {\n log('download', { id });\n const request = await this._lock.executeSynchronized(async () => {\n const existingRequest = this._downloadRequests.get(id);\n\n if (existingRequest) {\n existingRequest.counter++;\n return existingRequest;\n }\n\n const meta = await this._params.blobStore.getMeta(id);\n const request: DownloadRequest = {\n trigger: new Trigger(),\n counter: 1,\n want: {\n id,\n chunkSize: meta?.chunkSize,\n bitfield: meta?.bitfield && Uint8Array.from(BitField.invert(meta.bitfield)),\n },\n };\n\n // Check if the object is already fully downloaded.\n if (meta?.state === BlobMeta.State.FULLY_PRESENT) {\n request.trigger.wake();\n } else {\n this._downloadRequests.set(id, request);\n this._updateExtensionsWantList();\n }\n\n return request;\n });\n\n ctx?.onDispose(() =>\n this._lock.executeSynchronized(async () => {\n // Remove request if context is disposed and nobody else requests it.\n const request = this._downloadRequests.get(id);\n if (!request) {\n return;\n }\n if (--request.counter === 0) {\n this._downloadRequests.delete(id);\n }\n this._updateExtensionsWantList();\n }),\n );\n\n return ctx ? cancelWithContext(ctx, request.trigger.wait()) : request.trigger.wait();\n }\n\n createExtension() {\n const extension = new BlobSyncExtension({\n blobStore: this._params.blobStore,\n onOpen: async () => {\n log('extension opened');\n this._extensions.add(extension);\n extension.updateWantList(this._getWantList());\n },\n onClose: async () => {\n log('extension closed');\n this._extensions.delete(extension);\n },\n onPush: async (blobChunk) => {\n if (!this._downloadRequests.has(blobChunk.id)) {\n return;\n }\n log('received', { blobChunk });\n const meta = await this._params.blobStore.setChunk(blobChunk);\n if (meta.state === BlobMeta.State.FULLY_PRESENT) {\n this._downloadRequests.get(blobChunk.id)?.trigger.wake();\n this._downloadRequests.delete(blobChunk.id);\n } else {\n invariant(meta.bitfield);\n this._downloadRequests.get(blobChunk.id)!.want.bitfield = BitField.invert(meta.bitfield);\n }\n\n this._updateExtensionsWantList();\n this._reconcileUploads();\n },\n });\n return extension;\n }\n\n /**\n * Notify extensions that a blob with the given id was added to the blob store.\n */\n async notifyBlobAdded(_id: Uint8Array) {\n this._reconcileUploads();\n }\n\n private _getWantList(): WantList {\n return {\n blobs: Array.from(this._downloadRequests.values()).map((request) => request.want),\n };\n }\n\n private _reconcileUploads() {\n for (const extension of this._extensions) {\n extension.reconcileUploads();\n }\n }\n\n private _updateExtensionsWantList() {\n for (const extension of this._extensions) {\n extension.updateWantList(this._getWantList());\n }\n }\n}\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport path from 'node:path';\n\nimport { synchronized } from '@dxos/async';\nimport { subtleCrypto } from '@dxos/crypto';\nimport { invariant } from '@dxos/invariant';\nimport { PublicKey } from '@dxos/keys';\nimport { schema } from '@dxos/protocols';\nimport { BlobMeta } from '@dxos/protocols/proto/dxos/echo/blob';\nimport { type BlobChunk } from '@dxos/protocols/proto/dxos/mesh/teleport/blobsync';\nimport { type Directory } from '@dxos/random-access-storage';\nimport { BitField, arrayToBuffer } from '@dxos/util';\n\nexport type GetOptions = {\n offset?: number;\n length?: number;\n};\n\nexport const DEFAULT_CHUNK_SIZE = 4096;\n\nconst BlobMetaCodec = schema.getCodecForType('dxos.echo.blob.BlobMeta');\n\nexport class BlobStore {\n constructor(private readonly _directory: Directory) {}\n\n @synchronized\n async getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n return this._getMeta(id);\n }\n\n /**\n * @throws If range is not available.\n */\n @synchronized\n async get(id: Uint8Array, options: GetOptions = {}): Promise<Uint8Array> {\n const metadata = await this._getMeta(id);\n\n if (!metadata) {\n throw new Error('Blob not available');\n }\n\n const { offset = 0, length = metadata.length } = options;\n\n if (offset + length > metadata.length) {\n throw new Error('Invalid range');\n }\n\n if (metadata.state === BlobMeta.State.FULLY_PRESENT) {\n const file = this._getDataFile(id);\n return file.read(offset, length);\n } else if (options.offset === undefined && options.length === undefined) {\n throw new Error('Blob not available');\n }\n\n const beginChunk = Math.floor(offset / metadata.chunkSize);\n const endChunk = Math.ceil((offset + length) / metadata.chunkSize);\n\n invariant(metadata.bitfield, 'Bitfield not present');\n invariant(metadata.bitfield.length * 8 >= endChunk, 'Invalid bitfield length');\n\n const present = BitField.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;\n\n if (!present) {\n throw new Error('Blob not available');\n }\n\n const file = this._getDataFile(id);\n return file.read(offset, length);\n }\n\n @synchronized\n async list(): Promise<BlobMeta[]> {\n /*\n Weird path formatting:\n\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_meta\"\n \"e9b9aa7a21c2c55a9eca333cd59975633157562ca0a0f4f243d4778f192c291e_data\"\n \"5001de5a47191357c075aeee6451c4cc323f3a8ada24dd1191e83403608a38d5_meta\n */\n const files = new Set((await this._directory.list()).map((f) => f.split('_')[0]));\n\n const res: BlobMeta[] = [];\n\n for (const file of files) {\n const id = PublicKey.from(file).asUint8Array();\n const meta = await this._getMeta(id);\n if (meta) {\n res.push(meta);\n }\n }\n\n return res;\n }\n\n @synchronized\n async set(data: Uint8Array): Promise<BlobMeta> {\n const id = new Uint8Array(await subtleCrypto.digest('SHA-256', data));\n const bitfield = BitField.ones(data.length / DEFAULT_CHUNK_SIZE);\n\n const meta: BlobMeta = {\n id,\n state: BlobMeta.State.FULLY_PRESENT,\n length: data.length,\n chunkSize: DEFAULT_CHUNK_SIZE,\n bitfield,\n created: new Date(),\n updated: new Date(),\n };\n\n await this._getDataFile(id).write(0, arrayToBuffer(data));\n await this._writeMeta(id, meta);\n return meta;\n }\n\n // TODO(dmaretskyi): Optimize locking.\n @synchronized\n async setChunk(chunk: BlobChunk): Promise<BlobMeta> {\n // Init metadata.\n let meta = await this._getMeta(chunk.id);\n if (!meta) {\n invariant(chunk.totalLength, 'totalLength is not present');\n meta = {\n id: chunk.id,\n state: BlobMeta.State.PARTIALLY_PRESENT,\n length: chunk.totalLength,\n chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,\n created: new Date(),\n };\n meta.bitfield = BitField.zeros(meta.length / meta.chunkSize);\n }\n\n if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {\n throw new Error('Invalid chunk size');\n }\n\n invariant(meta.bitfield, 'Bitfield not present');\n invariant(chunk.chunkOffset !== undefined, 'chunkOffset is not present');\n\n // Write chunk.\n await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));\n\n // Update bitfield.\n BitField.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);\n\n // Update metadata.\n if (BitField.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {\n meta.state = BlobMeta.State.FULLY_PRESENT;\n }\n meta.updated = new Date();\n\n await this._writeMeta(chunk.id, meta);\n\n return meta;\n }\n\n private async _writeMeta(id: Uint8Array, meta: BlobMeta): Promise<void> {\n const encoded = arrayToBuffer(BlobMetaCodec.encode(meta));\n const data = Buffer.alloc(encoded.length + 4);\n data.writeUInt32LE(encoded.length, 0);\n encoded.copy(data, 4);\n\n // Write metadata.\n await this._getMetaFile(id).write(0, data);\n }\n\n private async _getMeta(id: Uint8Array): Promise<BlobMeta | undefined> {\n const file = this._getMetaFile(id);\n const size = (await file.stat()).size;\n if (size === 0) {\n return;\n }\n const data = await file.read(0, size);\n const protoSize = data.readUInt32LE(0);\n return BlobMetaCodec.decode(data.subarray(4, protoSize + 4));\n }\n\n private _getMetaFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'meta'));\n }\n\n private _getDataFile(id: Uint8Array) {\n return this._directory.getOrCreateFile(path.join(arrayToBuffer(id).toString('hex'), 'data'));\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAMA,mBAAkD;AAClD,qBAAwB;AACxB,uBAA0B;AAC1B,iBAAoB;AACpB,uBAAuC;AAEvC,sBAAoD;AACpD,kBAAyB;ACTzB,IAAAA,gBAA0C;AAC1C,IAAAC,kBAA2C;AAC3C,IAAAC,oBAA0B;AAC1B,kBAA0B;AAC1B,IAAAC,cAAoB;AACpB,kBAAyB;AAEzB,IAAAC,eAAqC;ACPrC,uBAAiB;AAEjB,IAAAJ,gBAA6B;AAC7B,oBAA6B;AAC7B,IAAAE,oBAA0B;AAC1B,IAAAG,eAA0B;AAC1B,IAAAC,oBAAuB;AACvB,IAAAC,eAAyB;AAGzB,IAAAH,eAAwC;;;;;;;;;;;;AFUxC,IAAMI,gCAAgCC,QAAQC,IAAIC,aAAa,SAAS,IAAI;AAE5E,IAAMC,yBAAyB;AAKxB,IAAMC,oBAAN,cAAgCC,6BAAAA;EAwDrCC,YACmBC,SACjB;AACA,UAAM;MACJC,SAAS;QACPC,iBAAiBC,wBAAOC,WAAW,6CAAA;MACrC;MACAC,WAAW;QACTH,iBAAiBC,wBAAOC,WAAW,6CAAA;MACrC;MACAE,SAAS;MACTC,iBAAiB;QACfC,aAAa;MACf;IACF,CAAA;mBAbiBR;SAxDFS,OAAO,IAAIC,uBAAQ;MAAEC,SAAS,CAACC,QAAQC,eAAIC,MAAMF,KAAAA,QAAAA;;;;;;IAAK,CAAA;SAE/DG,sBAAsB;SACtBC,iBAA2B;MAAEC,OAAO,CAAA;IAAG;SAE9BC,kBAAkB,IAAIC,0BAAa,KAAKV,MAAM,YAAA;AAE7D,UAAI,KAAKM,sBAAsBvB,gCAAgC4B,KAAKC,IAAG,GAAI;AACzE,kBAAMC,oBAAM,KAAKP,sBAAsBvB,gCAAgC4B,KAAKC,IAAG,CAAA;AAC/E,YAAI,KAAKZ,KAAKc,UAAU;AACtB;QACF;MACF;AAEAV,0BAAI,QAAQ;QAAEW,MAAM,KAAKR;MAAe,GAAA;;;;;;AACxC,YAAM,KAAKS,IAAIvB,gBAAgBwB,KAAK,KAAKV,cAAc;AACvD,WAAKD,sBAAsBK,KAAKC,IAAG;IACrC,CAAA;SAEQM,kBAAkB;SAETC,UAAU,IAAIT,0BAAa,KAAKV,MAAM,YAAA;AACrD,UAAI,KAAKkB,mBAAmB/B,wBAAwB;AAClD;MACF;AACA,YAAMiC,aAAa,MAAM,KAAKC,gBAAgBlC,yBAAyB,KAAK+B,eAAe;AAC3F,UAAI,CAACE,YAAY;AACf;MACF;AACA,iBAAWE,aAAaF,YAAY;AAClC,YAAI,KAAKpB,KAAKc,UAAU;AACtB;QACF;AAEA,aAAKI;AAEL,aAAKK,KAAKD,SAAAA,EACPjB,MAAM,CAACF,QAAAA;AACN,cAAIA,eAAeqB,iCAAgB;AACjC;UACF;AACApB,yBAAIqB,KAAK,eAAe;YAAEtB;UAAI,GAAA;;;;;;QAChC,CAAA,EACCuB,QAAQ,MAAA;AACP,eAAKR;AACL,eAAKS,iBAAgB;QACvB,CAAA;MACJ;IACF,CAAA;SAKOC,iBAA2B;MAAEpB,OAAO,CAAA;IAAG;EAiB9C;EAEA,MAAeqB,OAAOC,SAA0C;AAC9D1B,wBAAI,QAAA,QAAA;;;;;;AACJ,UAAM,MAAMyB,OAAOC,OAAAA;AACnB,UAAM,KAAKvC,QAAQsC,OAAM;EAC3B;EAEA,MAAeE,QAAQ5B,KAAwC;AAC7DC,wBAAI,SAAA,QAAA;;;;;;AACJ,UAAM,KAAKJ,KAAKgC,QAAO;AACvB,UAAM,KAAKzC,QAAQwC,QAAO;AAC1B,UAAM,MAAMA,QAAQ5B,GAAAA;EACtB;EAEA,MAAgB8B,cAAsC;AACpD,WAAO;MACLxC,iBAAiB;QACfwB,MAAM,OAAOiB,aAAAA;AACX9B,8BAAI,eAAe;YAAEwB,gBAAgBM;UAAS,GAAA;;;;;;AAC9C,eAAKN,iBAAiBM;AACtB,eAAKP,iBAAgB;QACvB;QACAJ,MAAM,OAAOY,SAAAA;AACX/B,8BAAI,YAAY;YAAE+B;UAAK,GAAA;;;;;;AACvB,gBAAM,KAAK5C,QAAQ6C,OAAOD,IAAAA;QAC5B;MACF;IACF;EACF;EAEA,MACMZ,KAAKY,MAAiB;AAC1B,QAAI,KAAKnC,KAAKc,UAAU;AACtB;IACF;AACAV,wBAAI,QAAQ;MAAE+B;IAAK,GAAA;;;;;;AACnB,UAAM,KAAKnB,IAAIvB,gBAAgB8B,KAAKY,IAAAA;EACtC;EAEAE,eAAeH,UAAoB;AACjC,QAAI,KAAKlC,KAAKc,UAAU;AACtB;IACF;AACA,SAAKP,iBAAiB2B;AACtB,SAAKzB,gBAAgB6B,SAAQ;EAC/B;EAEAX,mBAAmB;AACjB,QAAI,KAAK3B,KAAKc,UAAU;AACtB;IACF;AACA,SAAKK,QAAQmB,SAAQ;EACvB;EAEA,MAAcjB,gBAAgBkB,SAAS,GAAgC;AACrE,QAAI,KAAKvC,KAAKc,UAAU;AACtB;IACF;AAEA,QAAI,CAAC,KAAKc,eAAepB,SAAS,KAAKoB,eAAepB,OAAOgC,WAAW,GAAG;AACzE;IACF;AAEA,UAAMC,WAAW;SAAI,KAAKb,eAAepB;MAAOkC,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAE3E,UAAMC,SAAsB,CAAA;AAE5B,eAAWC,UAAUL,UAAU;AAC7B,YAAMM,OAAO,MAAM,KAAKxD,QAAQyD,UAAUC,QAAQH,OAAOI,EAAE;AAE3D,UAAI,CAACH,MAAM;AAET;MACF;AACAI,sCAAUJ,KAAKK,UAAQ,QAAA;;;;;;;;;AACvBD,sCAAUJ,KAAKM,WAAS,QAAA;;;;;;;;;AACxBF,sCAAUJ,KAAKP,QAAM,QAAA;;;;;;;;;AAErB,UAAIM,OAAOO,aAAaP,OAAOO,cAAcN,KAAKM,WAAW;AAC3DjD,uBAAIqB,KAAK,sBAAsB;UAAEqB;UAAQC;QAAK,GAAA;;;;;;AAC9C;MACF;AAEA,YAAMO,kBAAkBR,OAAOM,YAAYG,qBAASC,KAAKT,KAAKP,SAASO,KAAKM,SAAS;AAErF,YAAMI,cAAcF,qBAASG,IAAIJ,iBAAiBP,KAAKK,QAAQ;AAC/D,YAAMO,eAAeJ,qBAASK,YAAYH,WAAAA,EAAaf,KAAK,MAAMC,KAAKC,OAAM,IAAK,GAAA;AAElF,iBAAWiB,OAAOF,cAAc;AAC9B,cAAMG,YAAY,MAAM,KAAKvE,QAAQyD,UAAUe,IAAIjB,OAAOI,IAAI;UAC5Dc,QAAQH,MAAMd,KAAKM;UACnBb,QAAQG,KAAKsB,IAAIlB,KAAKM,WAAWN,KAAKP,SAASqB,MAAMd,KAAKM,SAAS;QACrE,CAAA;AACAR,eAAOtB,KAAK;UACV2B,IAAIJ,OAAOI;UACXgB,aAAanB,KAAKP;UAClBa,WAAWN,KAAKM;UAChBc,aAAaN,MAAMd,KAAKM;UACxBe,SAASN;QACX,CAAA;AAEA,YAAIjB,OAAOL,UAAUD,QAAQ;AAC3B,iBAAOM;QACT;MACF;IACF;AAEA,WAAOA;EACT;AACF;;EA/EGwB;GAtGUjF,kBAAAA,WAAAA,QAAAA,IAAAA;;;;;;;;;;;;ACHb,IAAakF,WAAN,MAAAA,UAAA;EAULhF,YAA6BC,SAAyB;mBAAzBA;SATZS,OAAO,IAAIC,gBAAAA,QAAAA;SACXsE,QAAQ,IAAIC,mBAAAA;SAEZC,oBAAoB,IAAIC,wBAAwC,CAACC,QAChFC,sBAAUC,KAAKF,GAAAA,EAAKG,MAAK,CAAA;SAGVC,cAAc,oBAAIC,IAAAA;EAEoB;EAEvD,MAAMC,OAAO;EAAC;EAEd,MAAMC,QAAQ;AACZ,UAAM,KAAKlF,KAAKgC,QAAO;EACzB;;;;;;EAOA,MAAMmD,SAASC,KAAclC,IAA+B;AAC1D9C,oBAAAA,KAAI,YAAY;MAAE8C;IAAG,GAAA;;;;;;AACrB,UAAMmC,UAAU,MAAM,KAAKd,MAAMe,oBAAoB,YAAA;AACnD,YAAMC,kBAAkB,KAAKd,kBAAkBV,IAAIb,EAAAA;AAEnD,UAAIqC,iBAAiB;AACnBA,wBAAgBC;AAChB,eAAOD;MACT;AAEA,YAAMxC,OAAO,MAAM,KAAKxD,QAAQyD,UAAUC,QAAQC,EAAAA;AAClD,YAAMmC,WAA2B;QAC/BI,SAAS,IAAIC,sBAAAA;QACbF,SAAS;QACTvE,MAAM;UACJiC;UACAG,WAAWN,MAAMM;UACjBD,UAAUL,MAAMK,YAAYuC,WAAWd,KAAKtB,aAAAA,SAASqC,OAAO7C,KAAKK,QAAQ,CAAA;QAC3E;MACF;AAGA,UAAIL,MAAM8C,UAAUC,qBAASC,MAAMC,eAAe;AAChDX,iBAAQI,QAAQQ,KAAI;MACtB,OAAO;AACL,aAAKxB,kBAAkByB,IAAIhD,IAAImC,QAAAA;AAC/B,aAAKc,0BAAyB;MAChC;AAEA,aAAOd;IACT,CAAA;AAEAD,SAAKgB,UAAU,MACb,KAAK7B,MAAMe,oBAAoB,YAAA;AAE7B,YAAMD,WAAU,KAAKZ,kBAAkBV,IAAIb,EAAAA;AAC3C,UAAI,CAACmC,UAAS;AACZ;MACF;AACA,UAAI,EAAEA,SAAQG,YAAY,GAAG;AAC3B,aAAKf,kBAAkB4B,OAAOnD,EAAAA;MAChC;AACA,WAAKiD,0BAAyB;IAChC,CAAA,CAAA;AAGF,WAAOf,UAAMkB,mCAAkBlB,KAAKC,QAAQI,QAAQc,KAAI,CAAA,IAAMlB,QAAQI,QAAQc,KAAI;EACpF;EAEAC,kBAAkB;AAChB,UAAMC,YAAY,IAAIrH,kBAAkB;MACtC4D,WAAW,KAAKzD,QAAQyD;MACxBnB,QAAQ,YAAA;AACNzB,wBAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAK2E,YAAY2B,IAAID,SAAAA;AACrBA,kBAAUpE,eAAe,KAAKsE,aAAY,CAAA;MAC5C;MACA5E,SAAS,YAAA;AACP3B,wBAAAA,KAAI,oBAAA,QAAA;;;;;;AACJ,aAAK2E,YAAYsB,OAAOI,SAAAA;MAC1B;MACArE,QAAQ,OAAOd,cAAAA;AACb,YAAI,CAAC,KAAKmD,kBAAkBmC,IAAItF,UAAU4B,EAAE,GAAG;AAC7C;QACF;AACA9C,wBAAAA,KAAI,YAAY;UAAEkB;QAAU,GAAA;;;;;;AAC5B,cAAMyB,OAAO,MAAM,KAAKxD,QAAQyD,UAAU6D,SAASvF,SAAAA;AACnD,YAAIyB,KAAK8C,UAAUC,qBAASC,MAAMC,eAAe;AAC/C,eAAKvB,kBAAkBV,IAAIzC,UAAU4B,EAAE,GAAGuC,QAAQQ,KAAAA;AAClD,eAAKxB,kBAAkB4B,OAAO/E,UAAU4B,EAAE;QAC5C,OAAO;AACLC,gCAAAA,WAAUJ,KAAKK,UAAQ,QAAA;;;;;;;;;AACvB,eAAKqB,kBAAkBV,IAAIzC,UAAU4B,EAAE,EAAGjC,KAAKmC,WAAWG,aAAAA,SAASqC,OAAO7C,KAAKK,QAAQ;QACzF;AAEA,aAAK+C,0BAAyB;AAC9B,aAAKW,kBAAiB;MACxB;IACF,CAAA;AACA,WAAOL;EACT;;;;EAKA,MAAMM,gBAAgBC,KAAiB;AACrC,SAAKF,kBAAiB;EACxB;EAEQH,eAAyB;AAC/B,WAAO;MACLnG,OAAOyG,MAAMpC,KAAK,KAAKJ,kBAAkByC,OAAM,CAAA,EAAIC,IAAI,CAAC9B,YAAYA,QAAQpE,IAAI;IAClF;EACF;EAEQ6F,oBAAoB;AAC1B,eAAWL,aAAa,KAAK1B,aAAa;AACxC0B,gBAAU9E,iBAAgB;IAC5B;EACF;EAEQwE,4BAA4B;AAClC,eAAWM,aAAa,KAAK1B,aAAa;AACxC0B,gBAAUpE,eAAe,KAAKsE,aAAY,CAAA;IAC5C;EACF;AACF;AAjIarC,WAAAA,cAAAA;MADZ8C,0BAAW,QAAQ,OAAA;GACP9C,QAAAA;;;;;;;;;;;;ACPN,IAAM+C,qBAAqB;AAElC,IAAMC,gBAAgB5H,kBAAAA,OAAO6H,gBAAgB,yBAAA;AAEtC,IAAMC,YAAN,MAAMA;EACXlI,YAA6BmI,YAAuB;sBAAvBA;EAAwB;EAErD,MACMxE,QAAQC,IAA+C;AAC3D,WAAO,KAAKwE,SAASxE,EAAAA;EACvB;;;;EAKA,MACMa,IAAIb,IAAgByE,UAAsB,CAAC,GAAwB;AACvE,UAAMC,WAAW,MAAM,KAAKF,SAASxE,EAAAA;AAErC,QAAI,CAAC0E,UAAU;AACb,YAAM,IAAIC,MAAM,oBAAA;IAClB;AAEA,UAAM,EAAE7D,SAAS,GAAGxB,SAASoF,SAASpF,OAAM,IAAKmF;AAEjD,QAAI3D,SAASxB,SAASoF,SAASpF,QAAQ;AACrC,YAAM,IAAIqF,MAAM,eAAA;IAClB;AAEA,QAAID,SAAS/B,UAAUC,aAAAA,SAASC,MAAMC,eAAe;AACnD,YAAM8B,QAAO,KAAKC,aAAa7E,EAAAA;AAC/B,aAAO4E,MAAKE,KAAKhE,QAAQxB,MAAAA;IAC3B,WAAWmF,QAAQ3D,WAAWiE,UAAaN,QAAQnF,WAAWyF,QAAW;AACvE,YAAM,IAAIJ,MAAM,oBAAA;IAClB;AAEA,UAAMK,aAAavF,KAAKwF,MAAMnE,SAAS4D,SAASvE,SAAS;AACzD,UAAM+E,WAAWzF,KAAK0F,MAAMrE,SAASxB,UAAUoF,SAASvE,SAAS;AAEjEF,0BAAAA,WAAUyE,SAASxE,UAAU,wBAAA;;;;;;;;;AAC7BD,0BAAAA,WAAUyE,SAASxE,SAASZ,SAAS,KAAK4F,UAAU,2BAAA;;;;;;;;;AAEpD,UAAME,UAAU/E,aAAAA,SAASgF,MAAMX,SAASxE,UAAU8E,YAAYE,QAAAA,MAAcA,WAAWF;AAEvF,QAAI,CAACI,SAAS;AACZ,YAAM,IAAIT,MAAM,oBAAA;IAClB;AAEA,UAAMC,OAAO,KAAKC,aAAa7E,EAAAA;AAC/B,WAAO4E,KAAKE,KAAKhE,QAAQxB,MAAAA;EAC3B;EAEA,MACMzB,OAA4B;AAQhC,UAAMyH,QAAQ,IAAIxD,KAAK,MAAM,KAAKyC,WAAW1G,KAAI,GAAIoG,IAAI,CAACsB,MAAMA,EAAEC,MAAM,GAAA,EAAK,CAAA,CAAE,CAAA;AAE/E,UAAMC,MAAkB,CAAA;AAExB,eAAWb,QAAQU,OAAO;AACxB,YAAMtF,KAAK0B,aAAAA,UAAUC,KAAKiD,IAAAA,EAAMc,aAAY;AAC5C,YAAM7F,OAAO,MAAM,KAAK2E,SAASxE,EAAAA;AACjC,UAAIH,MAAM;AACR4F,YAAIpH,KAAKwB,IAAAA;MACX;IACF;AAEA,WAAO4F;EACT;EAEA,MACMzC,IAAI/D,MAAqC;AAC7C,UAAMe,KAAK,IAAIyC,WAAW,MAAMkD,2BAAaC,OAAO,WAAW3G,IAAAA,CAAAA;AAC/D,UAAMiB,WAAWG,aAAAA,SAASC,KAAKrB,KAAKK,SAAS6E,kBAAAA;AAE7C,UAAMtE,OAAiB;MACrBG;MACA2C,OAAOC,aAAAA,SAASC,MAAMC;MACtBxD,QAAQL,KAAKK;MACba,WAAWgE;MACXjE;MACA2F,SAAS,oBAAIpI,KAAAA;MACbqI,SAAS,oBAAIrI,KAAAA;IACf;AAEA,UAAM,KAAKoH,aAAa7E,EAAAA,EAAI+F,MAAM,OAAGC,4BAAc/G,IAAAA,CAAAA;AACnD,UAAM,KAAKgH,WAAWjG,IAAIH,IAAAA;AAC1B,WAAOA;EACT;;EAGA,MACM8D,SAASuC,OAAqC;AAElD,QAAIrG,OAAO,MAAM,KAAK2E,SAAS0B,MAAMlG,EAAE;AACvC,QAAI,CAACH,MAAM;AACTI,4BAAAA,WAAUiG,MAAMlF,aAAa,8BAAA;;;;;;;;;AAC7BnB,aAAO;QACLG,IAAIkG,MAAMlG;QACV2C,OAAOC,aAAAA,SAASC,MAAMsD;QACtB7G,QAAQ4G,MAAMlF;QACdb,WAAW+F,MAAM/F,aAAagE;QAC9B0B,SAAS,oBAAIpI,KAAAA;MACf;AACAoC,WAAKK,WAAWG,aAAAA,SAAS+F,MAAMvG,KAAKP,SAASO,KAAKM,SAAS;IAC7D;AAEA,QAAI+F,MAAM/F,aAAa+F,MAAM/F,cAAcN,KAAKM,WAAW;AACzD,YAAM,IAAIwE,MAAM,oBAAA;IAClB;AAEA1E,0BAAAA,WAAUJ,KAAKK,UAAU,wBAAA;;;;;;;;;AACzBD,0BAAAA,WAAUiG,MAAMjF,gBAAgB8D,QAAW,8BAAA;;;;;;;;;AAG3C,UAAM,KAAKF,aAAaqB,MAAMlG,EAAE,EAAE+F,MAAMG,MAAMjF,iBAAa+E,4BAAcE,MAAMhF,OAAO,CAAA;AAGtFb,iBAAAA,SAAS2C,IAAInD,KAAKK,UAAUT,KAAKwF,MAAMiB,MAAMjF,cAAcpB,KAAKM,SAAS,GAAG,IAAA;AAG5E,QAAIE,aAAAA,SAASgF,MAAMxF,KAAKK,UAAU,GAAGL,KAAKP,MAAM,IAAIO,KAAKM,aAAaN,KAAKP,QAAQ;AACjFO,WAAK8C,QAAQC,aAAAA,SAASC,MAAMC;IAC9B;AACAjD,SAAKiG,UAAU,oBAAIrI,KAAAA;AAEnB,UAAM,KAAKwI,WAAWC,MAAMlG,IAAIH,IAAAA;AAEhC,WAAOA;EACT;EAEA,MAAcoG,WAAWjG,IAAgBH,MAA+B;AACtE,UAAMwG,cAAUL,4BAAc5B,cAAckC,OAAOzG,IAAAA,CAAAA;AACnD,UAAMZ,OAAOsH,OAAOC,MAAMH,QAAQ/G,SAAS,CAAA;AAC3CL,SAAKwH,cAAcJ,QAAQ/G,QAAQ,CAAA;AACnC+G,YAAQK,KAAKzH,MAAM,CAAA;AAGnB,UAAM,KAAK0H,aAAa3G,EAAAA,EAAI+F,MAAM,GAAG9G,IAAAA;EACvC;EAEA,MAAcuF,SAASxE,IAA+C;AACpE,UAAM4E,OAAO,KAAK+B,aAAa3G,EAAAA;AAC/B,UAAM4G,QAAQ,MAAMhC,KAAKiC,KAAI,GAAID;AACjC,QAAIA,SAAS,GAAG;AACd;IACF;AACA,UAAM3H,OAAO,MAAM2F,KAAKE,KAAK,GAAG8B,IAAAA;AAChC,UAAME,YAAY7H,KAAK8H,aAAa,CAAA;AACpC,WAAO3C,cAAc4C,OAAO/H,KAAKgI,SAAS,GAAGH,YAAY,CAAA,CAAA;EAC3D;EAEQH,aAAa3G,IAAgB;AACnC,WAAO,KAAKuE,WAAW2C,gBAAgBC,iBAAAA,QAAKC,SAAKpB,4BAAchG,EAAAA,EAAIqH,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;EAEQxC,aAAa7E,IAAgB;AACnC,WAAO,KAAKuE,WAAW2C,gBAAgBC,iBAAAA,QAAKC,SAAKpB,4BAAchG,EAAAA,EAAIqH,SAAS,KAAA,GAAQ,MAAA,CAAA;EACtF;AACF;;EA9JGlG,cAAAA;GAHUmD,UAAAA,WAAAA,WAAAA,IAAAA;;EAWVnD,cAAAA;GAXUmD,UAAAA,WAAAA,OAAAA,IAAAA;;EAgDVnD,cAAAA;GAhDUmD,UAAAA,WAAAA,QAAAA,IAAAA;;EAwEVnD,cAAAA;GAxEUmD,UAAAA,WAAAA,OAAAA,IAAAA;;EA6FVnD,cAAAA;GA7FUmD,UAAAA,WAAAA,YAAAA,IAAAA;",
|
|
6
|
+
"names": ["import_async", "import_context", "import_invariant", "import_log", "import_util", "import_keys", "import_protocols", "import_blob", "MIN_WANT_LIST_UPDATE_INTERVAL", "process", "env", "NODE_ENV", "MAX_CONCURRENT_UPLOADS", "BlobSyncExtension", "RpcExtension", "constructor", "_params", "exposed", "BlobSyncService", "schema", "getService", "requested", "timeout", "encodingOptions", "preserveAny", "_ctx", "Context", "onError", "err", "log", "catch", "_lastWantListUpdate", "_localWantList", "blobs", "_updateWantList", "DeferredTask", "Date", "now", "sleep", "disposed", "list", "rpc", "want", "_currentUploads", "_upload", "blobChunks", "_pickBlobChunks", "blobChunk", "push", "RpcClosedError", "warn", "finally", "reconcileUploads", "remoteWantList", "onOpen", "context", "onClose", "dispose", "getHandlers", "wantList", "data", "onPush", "updateWantList", "schedule", "amount", "length", "shuffled", "sort", "Math", "random", "chunks", "header", "meta", "blobStore", "getMeta", "id", "invariant", "bitfield", "chunkSize", "requestBitfield", "BitField", "ones", "presentData", "and", "chunkIndices", "findIndexes", "idx", "chunkData", "get", "offset", "min", "totalLength", "chunkOffset", "payload", "synchronized", "BlobSync", "_lock", "Lock", "_downloadRequests", "ComplexMap", "key", "PublicKey", "from", "toHex", "_extensions", "Set", "open", "close", "download", "ctx", "request", "executeSynchronized", "existingRequest", "counter", "trigger", "Trigger", "Uint8Array", "invert", "state", "BlobMeta", "State", "FULLY_PRESENT", "wake", "set", "_updateExtensionsWantList", "onDispose", "delete", "cancelWithContext", "wait", "createExtension", "extension", "add", "_getWantList", "has", "setChunk", "_reconcileUploads", "notifyBlobAdded", "_id", "Array", "values", "map", "trackLeaks", "DEFAULT_CHUNK_SIZE", "BlobMetaCodec", "getCodecForType", "BlobStore", "_directory", "_getMeta", "options", "metadata", "Error", "file", "_getDataFile", "read", "undefined", "beginChunk", "floor", "endChunk", "ceil", "present", "count", "files", "f", "split", "res", "asUint8Array", "subtleCrypto", "digest", "created", "updated", "write", "arrayToBuffer", "_writeMeta", "chunk", "PARTIALLY_PRESENT", "zeros", "encoded", "encode", "Buffer", "alloc", "writeUInt32LE", "copy", "_getMetaFile", "size", "stat", "protoSize", "readUInt32LE", "decode", "subarray", "getOrCreateFile", "path", "join", "toString"]
|
|
7
7
|
}
|
package/dist/lib/node/meta.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":
|
|
1
|
+
{"inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytes":24384,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytes":17817,"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytes":22281,"imports":[{"path":"node:path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytes":703,"imports":[{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts","kind":"import-statement","original":"./blob-sync-extension"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts","kind":"import-statement","original":"./blob-sync"},{"path":"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts","kind":"import-statement","original":"./blob-store"}],"format":"esm"}},"outputs":{"packages/core/mesh/teleport-extension-object-sync/dist/lib/node/index.cjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":29938},"packages/core/mesh/teleport-extension-object-sync/dist/lib/node/index.cjs":{"imports":[{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/teleport","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/context","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true},{"path":"node:path","kind":"import-statement","external":true},{"path":"@dxos/async","kind":"import-statement","external":true},{"path":"@dxos/crypto","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/keys","kind":"import-statement","external":true},{"path":"@dxos/protocols","kind":"import-statement","external":true},{"path":"@dxos/protocols/proto/dxos/echo/blob","kind":"import-statement","external":true},{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["BlobStore","BlobSync","BlobSyncExtension","DEFAULT_CHUNK_SIZE"],"entryPoint":"packages/core/mesh/teleport-extension-object-sync/src/index.ts","inputs":{"packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts":{"bytesInOutput":7147},"packages/core/mesh/teleport-extension-object-sync/src/index.ts":{"bytesInOutput":0},"packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts":{"bytesInOutput":5176},"packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts":{"bytesInOutput":6537}},"bytes":19197}}}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/teleport-extension-object-sync",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.8-main.152ec6f",
|
|
4
4
|
"description": "Teleport extension to synchronize opaque data objects.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -16,19 +16,19 @@
|
|
|
16
16
|
"src"
|
|
17
17
|
],
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"@dxos/async": "0.3.
|
|
20
|
-
"@dxos/
|
|
21
|
-
"@dxos/
|
|
22
|
-
"@dxos/debug": "0.3.
|
|
23
|
-
"@dxos/invariant": "0.3.
|
|
24
|
-
"@dxos/keys": "0.3.
|
|
25
|
-
"@dxos/log": "0.3.
|
|
26
|
-
"@dxos/node-std": "0.3.
|
|
27
|
-
"@dxos/protocols": "0.3.
|
|
28
|
-
"@dxos/random-access-storage": "0.3.
|
|
29
|
-
"@dxos/rpc": "0.3.
|
|
30
|
-
"@dxos/teleport": "0.3.
|
|
31
|
-
"@dxos/util": "0.3.
|
|
19
|
+
"@dxos/async": "0.3.8-main.152ec6f",
|
|
20
|
+
"@dxos/crypto": "0.3.8-main.152ec6f",
|
|
21
|
+
"@dxos/context": "0.3.8-main.152ec6f",
|
|
22
|
+
"@dxos/debug": "0.3.8-main.152ec6f",
|
|
23
|
+
"@dxos/invariant": "0.3.8-main.152ec6f",
|
|
24
|
+
"@dxos/keys": "0.3.8-main.152ec6f",
|
|
25
|
+
"@dxos/log": "0.3.8-main.152ec6f",
|
|
26
|
+
"@dxos/node-std": "0.3.8-main.152ec6f",
|
|
27
|
+
"@dxos/protocols": "0.3.8-main.152ec6f",
|
|
28
|
+
"@dxos/random-access-storage": "0.3.8-main.152ec6f",
|
|
29
|
+
"@dxos/rpc": "0.3.8-main.152ec6f",
|
|
30
|
+
"@dxos/teleport": "0.3.8-main.152ec6f",
|
|
31
|
+
"@dxos/util": "0.3.8-main.152ec6f"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"typescript": "^5.2.2"
|