@dxos/teleport-extension-object-sync 0.8.4-main.fffef41 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/dist/lib/{node-esm → neutral}/index.mjs +214 -185
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/types/src/blob-store.d.ts +11 -1
- package/dist/types/src/blob-store.d.ts.map +1 -1
- package/dist/types/src/blob-sync-extension.d.ts +4 -4
- package/dist/types/src/blob-sync-extension.d.ts.map +1 -1
- package/dist/types/src/blob-sync.d.ts +4 -4
- package/dist/types/src/blob-sync.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +1 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/sqlite-blob-store.d.ts +31 -0
- package/dist/types/src/sqlite-blob-store.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +23 -19
- package/src/blob-store.ts +12 -1
- package/src/blob-sync-extension.ts +4 -4
- package/src/blob-sync.node.test.ts +0 -1
- package/src/blob-sync.ts +4 -4
- package/src/index.ts +1 -0
- package/src/sqlite-blob-store.ts +228 -0
- package/dist/lib/browser/index.mjs +0 -647
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
package/LICENSE
CHANGED
|
@@ -1,8 +1,105 @@
|
|
|
1
|
-
|
|
2
|
-
Copyright (c) 2022 DXOS
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
## Abbreviation
|
|
5
4
|
|
|
6
|
-
|
|
5
|
+
FSL-1.1-Apache-2.0
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
## Notice
|
|
8
|
+
|
|
9
|
+
Copyright 2026 DXOS
|
|
10
|
+
|
|
11
|
+
## Terms and Conditions
|
|
12
|
+
|
|
13
|
+
### Licensor ("We")
|
|
14
|
+
|
|
15
|
+
The party offering the Software under these Terms and Conditions.
|
|
16
|
+
|
|
17
|
+
### The Software
|
|
18
|
+
|
|
19
|
+
The "Software" is each version of the software that we make available under
|
|
20
|
+
these Terms and Conditions, as indicated by our inclusion of these Terms and
|
|
21
|
+
Conditions with the Software.
|
|
22
|
+
|
|
23
|
+
### License Grant
|
|
24
|
+
|
|
25
|
+
Subject to your compliance with this License Grant and the Patents,
|
|
26
|
+
Redistribution and Trademark clauses below, we hereby grant you the right to
|
|
27
|
+
use, copy, modify, create derivative works, publicly perform, publicly display
|
|
28
|
+
and redistribute the Software for any Permitted Purpose identified below.
|
|
29
|
+
|
|
30
|
+
### Permitted Purpose
|
|
31
|
+
|
|
32
|
+
A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
|
|
33
|
+
means making the Software available to others in a commercial product or
|
|
34
|
+
service that:
|
|
35
|
+
|
|
36
|
+
1. substitutes for the Software;
|
|
37
|
+
|
|
38
|
+
2. substitutes for any other product or service we offer using the Software
|
|
39
|
+
that exists as of the date we make the Software available; or
|
|
40
|
+
|
|
41
|
+
3. offers the same or substantially similar functionality as the Software.
|
|
42
|
+
|
|
43
|
+
Permitted Purposes specifically include using the Software:
|
|
44
|
+
|
|
45
|
+
1. for your internal use and access;
|
|
46
|
+
|
|
47
|
+
2. for non-commercial education;
|
|
48
|
+
|
|
49
|
+
3. for non-commercial research; and
|
|
50
|
+
|
|
51
|
+
4. in connection with professional services that you provide to a licensee
|
|
52
|
+
using the Software in accordance with these Terms and Conditions.
|
|
53
|
+
|
|
54
|
+
### Patents
|
|
55
|
+
|
|
56
|
+
To the extent your use for a Permitted Purpose would necessarily infringe our
|
|
57
|
+
patents, the license grant above includes a license under our patents. If you
|
|
58
|
+
make a claim against any party that the Software infringes or contributes to
|
|
59
|
+
the infringement of any patent, then your patent license to the Software ends
|
|
60
|
+
immediately.
|
|
61
|
+
|
|
62
|
+
### Redistribution
|
|
63
|
+
|
|
64
|
+
The Terms and Conditions apply to all copies, modifications and derivatives of
|
|
65
|
+
the Software.
|
|
66
|
+
|
|
67
|
+
If you redistribute any copies, modifications or derivatives of the Software,
|
|
68
|
+
you must include a copy of or a link to these Terms and Conditions and not
|
|
69
|
+
remove any copyright notices provided in or with the Software.
|
|
70
|
+
|
|
71
|
+
### Disclaimer
|
|
72
|
+
|
|
73
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
|
|
74
|
+
IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
|
|
75
|
+
PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
|
|
76
|
+
|
|
77
|
+
IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
|
|
78
|
+
SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
|
|
79
|
+
EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
|
|
80
|
+
|
|
81
|
+
### Trademarks
|
|
82
|
+
|
|
83
|
+
Except for displaying the License Details and identifying us as the origin of
|
|
84
|
+
the Software, you have no right under these Terms and Conditions to use our
|
|
85
|
+
trademarks, trade names, service marks or product names.
|
|
86
|
+
|
|
87
|
+
## Grant of Future License
|
|
88
|
+
|
|
89
|
+
We hereby irrevocably grant you an additional license to use the Software under
|
|
90
|
+
the Apache License, Version 2.0 that is effective on the second anniversary of
|
|
91
|
+
the date we make the Software available. On or after that date, you may use the
|
|
92
|
+
Software under the Apache License, Version 2.0, in which case the following
|
|
93
|
+
will apply:
|
|
94
|
+
|
|
95
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
96
|
+
this file except in compliance with the License.
|
|
97
|
+
|
|
98
|
+
You may obtain a copy of the License at
|
|
99
|
+
|
|
100
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
101
|
+
|
|
102
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
103
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
104
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
105
|
+
specific language governing permissions and limitations under the License.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "@dxos/node-std/globals";
|
|
2
2
|
|
|
3
3
|
// src/blob-sync-extension.ts
|
|
4
4
|
import { DeferredTask, sleep, synchronized } from "@dxos/async";
|
|
@@ -9,28 +9,20 @@ import { RpcClosedError } from "@dxos/protocols";
|
|
|
9
9
|
import { schema } from "@dxos/protocols/proto";
|
|
10
10
|
import { RpcExtension } from "@dxos/teleport";
|
|
11
11
|
import { BitField } from "@dxos/util";
|
|
12
|
+
var __dxlog_file = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
|
|
12
13
|
function _ts_decorate(decorators, target, key, desc) {
|
|
13
14
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
14
15
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
15
16
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
16
17
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
17
18
|
}
|
|
18
|
-
var __dxlog_file = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync-extension.ts";
|
|
19
19
|
var MIN_WANT_LIST_UPDATE_INTERVAL = process.env.NODE_ENV === "test" ? 5 : 500;
|
|
20
20
|
var MAX_CONCURRENT_UPLOADS = 20;
|
|
21
21
|
var BlobSyncExtension = class extends RpcExtension {
|
|
22
22
|
_params;
|
|
23
23
|
_ctx = new Context({
|
|
24
|
-
onError: (err) => log.catch(err, void 0, {
|
|
25
|
-
|
|
26
|
-
L: 35,
|
|
27
|
-
S: this,
|
|
28
|
-
C: (f, a) => f(...a)
|
|
29
|
-
})
|
|
30
|
-
}, {
|
|
31
|
-
F: __dxlog_file,
|
|
32
|
-
L: 35
|
|
33
|
-
});
|
|
24
|
+
onError: (err) => log.catch(err, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 26, S: this })
|
|
25
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 25 });
|
|
34
26
|
_lastWantListUpdate = 0;
|
|
35
27
|
_localWantList = {
|
|
36
28
|
blobs: []
|
|
@@ -44,12 +36,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
44
36
|
}
|
|
45
37
|
log("want", {
|
|
46
38
|
list: this._localWantList
|
|
47
|
-
}, {
|
|
48
|
-
F: __dxlog_file,
|
|
49
|
-
L: 49,
|
|
50
|
-
S: this,
|
|
51
|
-
C: (f, a) => f(...a)
|
|
52
|
-
});
|
|
39
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 40, S: this });
|
|
53
40
|
await this.rpc.BlobSyncService.want(this._localWantList);
|
|
54
41
|
this._lastWantListUpdate = Date.now();
|
|
55
42
|
});
|
|
@@ -73,12 +60,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
73
60
|
}
|
|
74
61
|
log.warn("push failed", {
|
|
75
62
|
err
|
|
76
|
-
}, {
|
|
77
|
-
F: __dxlog_file,
|
|
78
|
-
L: 76,
|
|
79
|
-
S: this,
|
|
80
|
-
C: (f, a) => f(...a)
|
|
81
|
-
});
|
|
63
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 64, S: this });
|
|
82
64
|
}).finally(() => {
|
|
83
65
|
this._currentUploads--;
|
|
84
66
|
this.reconcileUploads();
|
|
@@ -106,33 +88,18 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
106
88
|
}), this._params = _params;
|
|
107
89
|
}
|
|
108
90
|
async onOpen(context) {
|
|
109
|
-
log("open", void 0, {
|
|
110
|
-
F: __dxlog_file,
|
|
111
|
-
L: 108,
|
|
112
|
-
S: this,
|
|
113
|
-
C: (f, a) => f(...a)
|
|
114
|
-
});
|
|
91
|
+
log("open", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 93, S: this });
|
|
115
92
|
await super.onOpen(context);
|
|
116
93
|
await this._params.onOpen();
|
|
117
94
|
}
|
|
118
95
|
async onClose(err) {
|
|
119
|
-
log("close", void 0, {
|
|
120
|
-
F: __dxlog_file,
|
|
121
|
-
L: 114,
|
|
122
|
-
S: this,
|
|
123
|
-
C: (f, a) => f(...a)
|
|
124
|
-
});
|
|
96
|
+
log("close", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 98, S: this });
|
|
125
97
|
await this._ctx.dispose();
|
|
126
98
|
await this._params.onClose();
|
|
127
99
|
await super.onClose(err);
|
|
128
100
|
}
|
|
129
101
|
async onAbort(err) {
|
|
130
|
-
log("abort", void 0, {
|
|
131
|
-
F: __dxlog_file,
|
|
132
|
-
L: 121,
|
|
133
|
-
S: this,
|
|
134
|
-
C: (f, a) => f(...a)
|
|
135
|
-
});
|
|
102
|
+
log("abort", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 104, S: this });
|
|
136
103
|
await this._ctx.dispose();
|
|
137
104
|
await this._params.onAbort();
|
|
138
105
|
await super.onAbort(err);
|
|
@@ -143,24 +110,14 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
143
110
|
want: async (wantList) => {
|
|
144
111
|
log("remote want", {
|
|
145
112
|
remoteWantList: wantList
|
|
146
|
-
}, {
|
|
147
|
-
F: __dxlog_file,
|
|
148
|
-
L: 131,
|
|
149
|
-
S: this,
|
|
150
|
-
C: (f, a) => f(...a)
|
|
151
|
-
});
|
|
113
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 113, S: this });
|
|
152
114
|
this.remoteWantList = wantList;
|
|
153
115
|
this.reconcileUploads();
|
|
154
116
|
},
|
|
155
117
|
push: async (data) => {
|
|
156
118
|
log("received", {
|
|
157
119
|
data
|
|
158
|
-
}, {
|
|
159
|
-
F: __dxlog_file,
|
|
160
|
-
L: 136,
|
|
161
|
-
S: this,
|
|
162
|
-
C: (f, a) => f(...a)
|
|
163
|
-
});
|
|
120
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 120, S: this });
|
|
164
121
|
await this._params.onPush(data);
|
|
165
122
|
}
|
|
166
123
|
}
|
|
@@ -172,12 +129,7 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
172
129
|
}
|
|
173
130
|
log("push", {
|
|
174
131
|
data
|
|
175
|
-
}, {
|
|
176
|
-
F: __dxlog_file,
|
|
177
|
-
L: 148,
|
|
178
|
-
S: this,
|
|
179
|
-
C: (f, a) => f(...a)
|
|
180
|
-
});
|
|
132
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 132, S: this });
|
|
181
133
|
await this.rpc.BlobSyncService.push(data);
|
|
182
134
|
}
|
|
183
135
|
updateWantList(wantList) {
|
|
@@ -209,43 +161,14 @@ var BlobSyncExtension = class extends RpcExtension {
|
|
|
209
161
|
if (!meta) {
|
|
210
162
|
continue;
|
|
211
163
|
}
|
|
212
|
-
invariant(meta.bitfield, void 0, {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
S: this,
|
|
216
|
-
A: [
|
|
217
|
-
"meta.bitfield",
|
|
218
|
-
""
|
|
219
|
-
]
|
|
220
|
-
});
|
|
221
|
-
invariant(meta.chunkSize, void 0, {
|
|
222
|
-
F: __dxlog_file,
|
|
223
|
-
L: 188,
|
|
224
|
-
S: this,
|
|
225
|
-
A: [
|
|
226
|
-
"meta.chunkSize",
|
|
227
|
-
""
|
|
228
|
-
]
|
|
229
|
-
});
|
|
230
|
-
invariant(meta.length, void 0, {
|
|
231
|
-
F: __dxlog_file,
|
|
232
|
-
L: 189,
|
|
233
|
-
S: this,
|
|
234
|
-
A: [
|
|
235
|
-
"meta.length",
|
|
236
|
-
""
|
|
237
|
-
]
|
|
238
|
-
});
|
|
164
|
+
invariant(meta.bitfield, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 166, S: this, A: ["meta.bitfield", ""] });
|
|
165
|
+
invariant(meta.chunkSize, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 167, S: this, A: ["meta.chunkSize", ""] });
|
|
166
|
+
invariant(meta.length, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 168, S: this, A: ["meta.length", ""] });
|
|
239
167
|
if (header.chunkSize && header.chunkSize !== meta.chunkSize) {
|
|
240
168
|
log.warn("Invalid chunk size", {
|
|
241
169
|
header,
|
|
242
170
|
meta
|
|
243
|
-
}, {
|
|
244
|
-
F: __dxlog_file,
|
|
245
|
-
L: 192,
|
|
246
|
-
S: this,
|
|
247
|
-
C: (f, a) => f(...a)
|
|
248
|
-
});
|
|
171
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 170, S: this });
|
|
249
172
|
continue;
|
|
250
173
|
}
|
|
251
174
|
const requestBitfield = header.bitfield ?? BitField.ones(meta.length / meta.chunkSize);
|
|
@@ -283,19 +206,16 @@ import { PublicKey } from "@dxos/keys";
|
|
|
283
206
|
import { log as log2 } from "@dxos/log";
|
|
284
207
|
import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
285
208
|
import { BitField as BitField2, ComplexMap } from "@dxos/util";
|
|
209
|
+
var __dxlog_file2 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
|
|
286
210
|
function _ts_decorate2(decorators, target, key, desc) {
|
|
287
211
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
288
212
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
289
213
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
290
214
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
291
215
|
}
|
|
292
|
-
var __dxlog_file2 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-sync.ts";
|
|
293
216
|
var BlobSync = class {
|
|
294
217
|
_params;
|
|
295
|
-
_ctx = new Context2(void 0, {
|
|
296
|
-
F: __dxlog_file2,
|
|
297
|
-
L: 30
|
|
298
|
-
});
|
|
218
|
+
_ctx = new Context2(void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 20 });
|
|
299
219
|
_mutex = new Mutex();
|
|
300
220
|
_downloadRequests = new ComplexMap((key) => PublicKey.from(key).toHex());
|
|
301
221
|
_extensions = /* @__PURE__ */ new Set();
|
|
@@ -315,12 +235,7 @@ var BlobSync = class {
|
|
|
315
235
|
async download(ctx, id) {
|
|
316
236
|
log2("download", {
|
|
317
237
|
id
|
|
318
|
-
}, {
|
|
319
|
-
F: __dxlog_file2,
|
|
320
|
-
L: 53,
|
|
321
|
-
S: this,
|
|
322
|
-
C: (f, a) => f(...a)
|
|
323
|
-
});
|
|
238
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 36, S: this });
|
|
324
239
|
const request = await this._mutex.executeSynchronized(async () => {
|
|
325
240
|
const existingRequest = this._downloadRequests.get(id);
|
|
326
241
|
if (existingRequest) {
|
|
@@ -361,31 +276,16 @@ var BlobSync = class {
|
|
|
361
276
|
const extension = new BlobSyncExtension({
|
|
362
277
|
blobStore: this._params.blobStore,
|
|
363
278
|
onOpen: async () => {
|
|
364
|
-
log2("extension opened", void 0, {
|
|
365
|
-
F: __dxlog_file2,
|
|
366
|
-
L: 105,
|
|
367
|
-
S: this,
|
|
368
|
-
C: (f, a) => f(...a)
|
|
369
|
-
});
|
|
279
|
+
log2("extension opened", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 81, S: this });
|
|
370
280
|
this._extensions.add(extension);
|
|
371
281
|
extension.updateWantList(this._getWantList());
|
|
372
282
|
},
|
|
373
283
|
onClose: async () => {
|
|
374
|
-
log2("extension closed", void 0, {
|
|
375
|
-
F: __dxlog_file2,
|
|
376
|
-
L: 110,
|
|
377
|
-
S: this,
|
|
378
|
-
C: (f, a) => f(...a)
|
|
379
|
-
});
|
|
284
|
+
log2("extension closed", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 86, S: this });
|
|
380
285
|
this._extensions.delete(extension);
|
|
381
286
|
},
|
|
382
287
|
onAbort: async () => {
|
|
383
|
-
log2("extension aborted", void 0, {
|
|
384
|
-
F: __dxlog_file2,
|
|
385
|
-
L: 114,
|
|
386
|
-
S: this,
|
|
387
|
-
C: (f, a) => f(...a)
|
|
388
|
-
});
|
|
288
|
+
log2("extension aborted", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 90, S: this });
|
|
389
289
|
this._extensions.delete(extension);
|
|
390
290
|
},
|
|
391
291
|
onPush: async (blobChunk) => {
|
|
@@ -394,26 +294,13 @@ var BlobSync = class {
|
|
|
394
294
|
}
|
|
395
295
|
log2("received", {
|
|
396
296
|
blobChunk
|
|
397
|
-
}, {
|
|
398
|
-
F: __dxlog_file2,
|
|
399
|
-
L: 121,
|
|
400
|
-
S: this,
|
|
401
|
-
C: (f, a) => f(...a)
|
|
402
|
-
});
|
|
297
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 97, S: this });
|
|
403
298
|
const meta = await this._params.blobStore.setChunk(blobChunk);
|
|
404
299
|
if (meta.state === BlobMeta.State.FULLY_PRESENT) {
|
|
405
300
|
this._downloadRequests.get(blobChunk.id)?.trigger.wake();
|
|
406
301
|
this._downloadRequests.delete(blobChunk.id);
|
|
407
302
|
} else {
|
|
408
|
-
invariant2(meta.bitfield, void 0, {
|
|
409
|
-
F: __dxlog_file2,
|
|
410
|
-
L: 127,
|
|
411
|
-
S: this,
|
|
412
|
-
A: [
|
|
413
|
-
"meta.bitfield",
|
|
414
|
-
""
|
|
415
|
-
]
|
|
416
|
-
});
|
|
303
|
+
invariant2(meta.bitfield, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 105, S: this, A: ["meta.bitfield", ""] });
|
|
417
304
|
this._downloadRequests.get(blobChunk.id).want.bitfield = BitField2.invert(meta.bitfield);
|
|
418
305
|
}
|
|
419
306
|
this._updateExtensionsWantList();
|
|
@@ -449,7 +336,7 @@ BlobSync = _ts_decorate2([
|
|
|
449
336
|
], BlobSync);
|
|
450
337
|
|
|
451
338
|
// src/blob-store.ts
|
|
452
|
-
import path from "node
|
|
339
|
+
import path from "@dxos/node-std/path";
|
|
453
340
|
import { synchronized as synchronized2 } from "@dxos/async";
|
|
454
341
|
import { subtleCrypto } from "@dxos/crypto";
|
|
455
342
|
import { invariant as invariant3 } from "@dxos/invariant";
|
|
@@ -457,13 +344,13 @@ import { PublicKey as PublicKey2 } from "@dxos/keys";
|
|
|
457
344
|
import { schema as schema2 } from "@dxos/protocols/proto";
|
|
458
345
|
import { BlobMeta as BlobMeta2 } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
459
346
|
import { BitField as BitField3, arrayToBuffer } from "@dxos/util";
|
|
347
|
+
var __dxlog_file3 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
|
|
460
348
|
function _ts_decorate3(decorators, target, key, desc) {
|
|
461
349
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
462
350
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
463
351
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
464
352
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
465
353
|
}
|
|
466
|
-
var __dxlog_file3 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/blob-store.ts";
|
|
467
354
|
var DEFAULT_CHUNK_SIZE = 4096;
|
|
468
355
|
var BlobMetaCodec = schema2.getCodecForType("dxos.echo.blob.BlobMeta");
|
|
469
356
|
var BlobStore = class {
|
|
@@ -494,24 +381,8 @@ var BlobStore = class {
|
|
|
494
381
|
}
|
|
495
382
|
const beginChunk = Math.floor(offset / metadata.chunkSize);
|
|
496
383
|
const endChunk = Math.ceil((offset + length) / metadata.chunkSize);
|
|
497
|
-
invariant3(metadata.bitfield, "Bitfield not present", {
|
|
498
|
-
|
|
499
|
-
L: 61,
|
|
500
|
-
S: this,
|
|
501
|
-
A: [
|
|
502
|
-
"metadata.bitfield",
|
|
503
|
-
"'Bitfield not present'"
|
|
504
|
-
]
|
|
505
|
-
});
|
|
506
|
-
invariant3(metadata.bitfield.length * 8 >= endChunk, "Invalid bitfield length", {
|
|
507
|
-
F: __dxlog_file3,
|
|
508
|
-
L: 62,
|
|
509
|
-
S: this,
|
|
510
|
-
A: [
|
|
511
|
-
"metadata.bitfield.length * 8 >= endChunk",
|
|
512
|
-
"'Invalid bitfield length'"
|
|
513
|
-
]
|
|
514
|
-
});
|
|
384
|
+
invariant3(metadata.bitfield, "Bitfield not present", { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 47, S: this, A: ["metadata.bitfield", "'Bitfield not present'"] });
|
|
385
|
+
invariant3(metadata.bitfield.length * 8 >= endChunk, "Invalid bitfield length", { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 48, S: this, A: ["metadata.bitfield.length * 8 >= endChunk", "'Invalid bitfield length'"] });
|
|
515
386
|
const present = BitField3.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;
|
|
516
387
|
if (!present) {
|
|
517
388
|
throw new Error("Blob not available");
|
|
@@ -551,15 +422,7 @@ var BlobStore = class {
|
|
|
551
422
|
async setChunk(chunk) {
|
|
552
423
|
let meta = await this._getMeta(chunk.id);
|
|
553
424
|
if (!meta) {
|
|
554
|
-
invariant3(chunk.totalLength, "totalLength is not present", {
|
|
555
|
-
F: __dxlog_file3,
|
|
556
|
-
L: 124,
|
|
557
|
-
S: this,
|
|
558
|
-
A: [
|
|
559
|
-
"chunk.totalLength",
|
|
560
|
-
"'totalLength is not present'"
|
|
561
|
-
]
|
|
562
|
-
});
|
|
425
|
+
invariant3(chunk.totalLength, "totalLength is not present", { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 95, S: this, A: ["chunk.totalLength", "'totalLength is not present'"] });
|
|
563
426
|
meta = {
|
|
564
427
|
id: chunk.id,
|
|
565
428
|
state: BlobMeta2.State.PARTIALLY_PRESENT,
|
|
@@ -572,24 +435,8 @@ var BlobStore = class {
|
|
|
572
435
|
if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {
|
|
573
436
|
throw new Error("Invalid chunk size");
|
|
574
437
|
}
|
|
575
|
-
invariant3(meta.bitfield, "Bitfield not present", {
|
|
576
|
-
|
|
577
|
-
L: 139,
|
|
578
|
-
S: this,
|
|
579
|
-
A: [
|
|
580
|
-
"meta.bitfield",
|
|
581
|
-
"'Bitfield not present'"
|
|
582
|
-
]
|
|
583
|
-
});
|
|
584
|
-
invariant3(chunk.chunkOffset !== void 0, "chunkOffset is not present", {
|
|
585
|
-
F: __dxlog_file3,
|
|
586
|
-
L: 140,
|
|
587
|
-
S: this,
|
|
588
|
-
A: [
|
|
589
|
-
"chunk.chunkOffset !== undefined",
|
|
590
|
-
"'chunkOffset is not present'"
|
|
591
|
-
]
|
|
592
|
-
});
|
|
438
|
+
invariant3(meta.bitfield, "Bitfield not present", { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 108, S: this, A: ["meta.bitfield", "'Bitfield not present'"] });
|
|
439
|
+
invariant3(chunk.chunkOffset !== void 0, "chunkOffset is not present", { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 109, S: this, A: ["chunk.chunkOffset !== undefined", "'chunkOffset is not present'"] });
|
|
593
440
|
await this._getDataFile(chunk.id).write(chunk.chunkOffset, arrayToBuffer(chunk.payload));
|
|
594
441
|
BitField3.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);
|
|
595
442
|
if (BitField3.count(meta.bitfield, 0, meta.length) * meta.chunkSize >= meta.length) {
|
|
@@ -638,10 +485,192 @@ _ts_decorate3([
|
|
|
638
485
|
_ts_decorate3([
|
|
639
486
|
synchronized2
|
|
640
487
|
], BlobStore.prototype, "setChunk", null);
|
|
488
|
+
|
|
489
|
+
// src/sqlite-blob-store.ts
|
|
490
|
+
import * as SqlClient from "@effect/sql/SqlClient";
|
|
491
|
+
import * as Effect from "effect/Effect";
|
|
492
|
+
import { synchronized as synchronized3 } from "@dxos/async";
|
|
493
|
+
import { subtleCrypto as subtleCrypto2 } from "@dxos/crypto";
|
|
494
|
+
import { RuntimeProvider } from "@dxos/effect";
|
|
495
|
+
import { invariant as invariant4 } from "@dxos/invariant";
|
|
496
|
+
import { log as log3 } from "@dxos/log";
|
|
497
|
+
import { schema as schema3 } from "@dxos/protocols/proto";
|
|
498
|
+
import { BlobMeta as BlobMeta3 } from "@dxos/protocols/proto/dxos/echo/blob";
|
|
499
|
+
import { BitField as BitField4, arrayToBuffer as arrayToBuffer2 } from "@dxos/util";
|
|
500
|
+
var __dxlog_file4 = "/__w/dxos/dxos/packages/core/mesh/teleport-extension-object-sync/src/sqlite-blob-store.ts";
|
|
501
|
+
function _ts_decorate4(decorators, target, key, desc) {
|
|
502
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
503
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
504
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
505
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
506
|
+
}
|
|
507
|
+
var BlobMetaCodec2 = schema3.getCodecForType("dxos.echo.blob.BlobMeta");
|
|
508
|
+
var SqliteBlobStore = class {
|
|
509
|
+
#runtime;
|
|
510
|
+
constructor({ runtime }) {
|
|
511
|
+
this.#runtime = runtime;
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Creates the blobs_meta and blobs_data tables if they do not exist.
|
|
515
|
+
*/
|
|
516
|
+
migrate = Effect.fn("SqliteBlobStore.migrate")(() => Effect.gen(function* () {
|
|
517
|
+
const sql = yield* SqlClient.SqlClient;
|
|
518
|
+
yield* sql`CREATE TABLE IF NOT EXISTS blobs_meta (
|
|
519
|
+
id TEXT PRIMARY KEY,
|
|
520
|
+
meta BLOB NOT NULL
|
|
521
|
+
)`;
|
|
522
|
+
yield* sql`CREATE TABLE IF NOT EXISTS blobs_data (
|
|
523
|
+
id TEXT PRIMARY KEY,
|
|
524
|
+
data BLOB NOT NULL
|
|
525
|
+
)`;
|
|
526
|
+
log3("blobs tables ready", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 42, S: this });
|
|
527
|
+
}).pipe(Effect.withSpan("SqliteBlobStore.migrate")))();
|
|
528
|
+
async getMeta(id) {
|
|
529
|
+
return this.#getMeta(id);
|
|
530
|
+
}
|
|
531
|
+
async get(id, options = {}) {
|
|
532
|
+
const metadata = await this.#getMeta(id);
|
|
533
|
+
if (!metadata) {
|
|
534
|
+
throw new Error("Blob not available");
|
|
535
|
+
}
|
|
536
|
+
const { offset = 0, length = metadata.length } = options;
|
|
537
|
+
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0 || offset + length > metadata.length) {
|
|
538
|
+
throw new Error("Invalid range");
|
|
539
|
+
}
|
|
540
|
+
if (metadata.state === BlobMeta3.State.FULLY_PRESENT) {
|
|
541
|
+
const data2 = await this.#getData(id);
|
|
542
|
+
if (!data2) {
|
|
543
|
+
throw new Error("Blob data missing");
|
|
544
|
+
}
|
|
545
|
+
return data2.subarray(offset, offset + length);
|
|
546
|
+
} else if (options.offset === void 0 && options.length === void 0) {
|
|
547
|
+
throw new Error("Blob not available");
|
|
548
|
+
}
|
|
549
|
+
const beginChunk = Math.floor(offset / metadata.chunkSize);
|
|
550
|
+
const endChunk = Math.ceil((offset + length) / metadata.chunkSize);
|
|
551
|
+
invariant4(metadata.bitfield, "Bitfield not present", { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 67, S: this, A: ["metadata.bitfield", "'Bitfield not present'"] });
|
|
552
|
+
invariant4(metadata.bitfield.length * 8 >= endChunk, "Invalid bitfield length", { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 68, S: this, A: ["metadata.bitfield.length * 8 >= endChunk", "'Invalid bitfield length'"] });
|
|
553
|
+
const present = BitField4.count(metadata.bitfield, beginChunk, endChunk) === endChunk - beginChunk;
|
|
554
|
+
if (!present) {
|
|
555
|
+
throw new Error("Blob not available");
|
|
556
|
+
}
|
|
557
|
+
const data = await this.#getData(id);
|
|
558
|
+
if (!data) {
|
|
559
|
+
throw new Error("Blob data missing");
|
|
560
|
+
}
|
|
561
|
+
return data.subarray(offset, offset + length);
|
|
562
|
+
}
|
|
563
|
+
async list() {
|
|
564
|
+
const rows = await RuntimeProvider.runPromise(this.#runtime)(Effect.gen(function* () {
|
|
565
|
+
const sql = yield* SqlClient.SqlClient;
|
|
566
|
+
return yield* sql`SELECT id, meta FROM blobs_meta`;
|
|
567
|
+
}));
|
|
568
|
+
return rows.map((row) => BlobMetaCodec2.decode(row.meta));
|
|
569
|
+
}
|
|
570
|
+
async set(data) {
|
|
571
|
+
const id = new Uint8Array(await subtleCrypto2.digest("SHA-256", data));
|
|
572
|
+
const bitfield = BitField4.ones(Math.ceil(data.length / DEFAULT_CHUNK_SIZE));
|
|
573
|
+
const meta = {
|
|
574
|
+
id,
|
|
575
|
+
state: BlobMeta3.State.FULLY_PRESENT,
|
|
576
|
+
length: data.length,
|
|
577
|
+
chunkSize: DEFAULT_CHUNK_SIZE,
|
|
578
|
+
bitfield,
|
|
579
|
+
created: /* @__PURE__ */ new Date(),
|
|
580
|
+
updated: /* @__PURE__ */ new Date()
|
|
581
|
+
};
|
|
582
|
+
const idHex = arrayToBuffer2(id).toString("hex");
|
|
583
|
+
const encodedMeta = arrayToBuffer2(BlobMetaCodec2.encode(meta));
|
|
584
|
+
await RuntimeProvider.runPromise(this.#runtime)(Effect.gen(function* () {
|
|
585
|
+
const sql = yield* SqlClient.SqlClient;
|
|
586
|
+
yield* sql`INSERT OR REPLACE INTO blobs_meta (id, meta) VALUES (${idHex}, ${encodedMeta})`;
|
|
587
|
+
yield* sql`INSERT OR REPLACE INTO blobs_data (id, data) VALUES (${idHex}, ${data})`;
|
|
588
|
+
}));
|
|
589
|
+
return meta;
|
|
590
|
+
}
|
|
591
|
+
async setChunk(chunk) {
|
|
592
|
+
const idHex = arrayToBuffer2(chunk.id).toString("hex");
|
|
593
|
+
let meta = await this.#getMeta(chunk.id);
|
|
594
|
+
if (!meta) {
|
|
595
|
+
invariant4(chunk.totalLength, "totalLength is not present", { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 111, S: this, A: ["chunk.totalLength", "'totalLength is not present'"] });
|
|
596
|
+
meta = {
|
|
597
|
+
id: chunk.id,
|
|
598
|
+
state: BlobMeta3.State.PARTIALLY_PRESENT,
|
|
599
|
+
length: chunk.totalLength,
|
|
600
|
+
chunkSize: chunk.chunkSize ?? DEFAULT_CHUNK_SIZE,
|
|
601
|
+
created: /* @__PURE__ */ new Date()
|
|
602
|
+
};
|
|
603
|
+
meta.bitfield = BitField4.zeros(Math.ceil(meta.length / meta.chunkSize));
|
|
604
|
+
}
|
|
605
|
+
if (chunk.chunkSize && chunk.chunkSize !== meta.chunkSize) {
|
|
606
|
+
throw new Error("Invalid chunk size");
|
|
607
|
+
}
|
|
608
|
+
invariant4(meta.bitfield, "Bitfield not present", { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 124, S: this, A: ["meta.bitfield", "'Bitfield not present'"] });
|
|
609
|
+
invariant4(chunk.chunkOffset !== void 0, "chunkOffset is not present", { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 125, S: this, A: ["chunk.chunkOffset !== undefined", "'chunkOffset is not present'"] });
|
|
610
|
+
if (chunk.chunkOffset < 0 || chunk.chunkOffset + chunk.payload.length > meta.length) {
|
|
611
|
+
throw new Error("Invalid chunk range");
|
|
612
|
+
}
|
|
613
|
+
const existingData = await this.#getData(chunk.id) ?? new Uint8Array(meta.length);
|
|
614
|
+
const newData = Buffer.from(existingData);
|
|
615
|
+
Buffer.from(chunk.payload).copy(newData, chunk.chunkOffset);
|
|
616
|
+
BitField4.set(meta.bitfield, Math.floor(chunk.chunkOffset / meta.chunkSize), true);
|
|
617
|
+
const totalChunks = Math.ceil(meta.length / meta.chunkSize);
|
|
618
|
+
if (BitField4.count(meta.bitfield, 0, totalChunks) === totalChunks) {
|
|
619
|
+
meta.state = BlobMeta3.State.FULLY_PRESENT;
|
|
620
|
+
}
|
|
621
|
+
meta.updated = /* @__PURE__ */ new Date();
|
|
622
|
+
const encodedMeta = arrayToBuffer2(BlobMetaCodec2.encode(meta));
|
|
623
|
+
const id = chunk.id;
|
|
624
|
+
await RuntimeProvider.runPromise(this.#runtime)(Effect.gen(function* () {
|
|
625
|
+
const sql = yield* SqlClient.SqlClient;
|
|
626
|
+
yield* sql`INSERT OR REPLACE INTO blobs_meta (id, meta) VALUES (${idHex}, ${encodedMeta})`;
|
|
627
|
+
yield* sql`INSERT OR REPLACE INTO blobs_data (id, data) VALUES (${idHex}, ${newData})`;
|
|
628
|
+
}));
|
|
629
|
+
return meta;
|
|
630
|
+
}
|
|
631
|
+
async #getMeta(id) {
|
|
632
|
+
const idHex = arrayToBuffer2(id).toString("hex");
|
|
633
|
+
const rows = await RuntimeProvider.runPromise(this.#runtime)(Effect.gen(function* () {
|
|
634
|
+
const sql = yield* SqlClient.SqlClient;
|
|
635
|
+
return yield* sql`SELECT meta FROM blobs_meta WHERE id = ${idHex}`;
|
|
636
|
+
}));
|
|
637
|
+
if (rows.length === 0) {
|
|
638
|
+
return void 0;
|
|
639
|
+
}
|
|
640
|
+
return BlobMetaCodec2.decode(rows[0].meta);
|
|
641
|
+
}
|
|
642
|
+
async #getData(id) {
|
|
643
|
+
const idHex = arrayToBuffer2(id).toString("hex");
|
|
644
|
+
const rows = await RuntimeProvider.runPromise(this.#runtime)(Effect.gen(function* () {
|
|
645
|
+
const sql = yield* SqlClient.SqlClient;
|
|
646
|
+
return yield* sql`SELECT data FROM blobs_data WHERE id = ${idHex}`;
|
|
647
|
+
}));
|
|
648
|
+
if (rows.length === 0) {
|
|
649
|
+
return void 0;
|
|
650
|
+
}
|
|
651
|
+
return rows[0].data;
|
|
652
|
+
}
|
|
653
|
+
};
|
|
654
|
+
_ts_decorate4([
|
|
655
|
+
synchronized3
|
|
656
|
+
], SqliteBlobStore.prototype, "getMeta", null);
|
|
657
|
+
_ts_decorate4([
|
|
658
|
+
synchronized3
|
|
659
|
+
], SqliteBlobStore.prototype, "get", null);
|
|
660
|
+
_ts_decorate4([
|
|
661
|
+
synchronized3
|
|
662
|
+
], SqliteBlobStore.prototype, "list", null);
|
|
663
|
+
_ts_decorate4([
|
|
664
|
+
synchronized3
|
|
665
|
+
], SqliteBlobStore.prototype, "set", null);
|
|
666
|
+
_ts_decorate4([
|
|
667
|
+
synchronized3
|
|
668
|
+
], SqliteBlobStore.prototype, "setChunk", null);
|
|
641
669
|
export {
|
|
642
670
|
BlobStore,
|
|
643
671
|
BlobSync,
|
|
644
672
|
BlobSyncExtension,
|
|
645
|
-
DEFAULT_CHUNK_SIZE
|
|
673
|
+
DEFAULT_CHUNK_SIZE,
|
|
674
|
+
SqliteBlobStore
|
|
646
675
|
};
|
|
647
676
|
//# sourceMappingURL=index.mjs.map
|