@fireproof/core 0.19.100 → 0.19.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
- package/chunk-3EB3ENHT.js.map +1 -0
- package/chunk-HQ7D3PEU.js +61 -0
- package/chunk-HQ7D3PEU.js.map +1 -0
- package/chunk-PZ5AY32C.js +10 -0
- package/deno-filesystem-Q2IJ7YDR.js +57 -0
- package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
- package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
- package/gateway-GK5QZ6KP.js.map +1 -0
- package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
- package/gateway-TQTGDRCN.js.map +1 -0
- package/index.cjs +2158 -1718
- package/index.cjs.map +1 -1
- package/index.d.cts +261 -117
- package/index.d.ts +261 -117
- package/index.global.js +11834 -11354
- package/index.global.js.map +1 -1
- package/index.js +1865 -1519
- package/index.js.map +1 -1
- package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
- package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
- package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
- package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
- package/key-bag-memory-LWE6ARPX.js +29 -0
- package/key-bag-memory-LWE6ARPX.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
- package/node-filesystem-CFRXFSO7.js.map +1 -0
- package/package.json +1 -1
- package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
- package/tests/blockstore/keyed-crypto.test.ts +63 -227
- package/tests/blockstore/loader.test.ts +19 -11
- package/tests/blockstore/store.test.ts +23 -19
- package/tests/blockstore/transaction.test.ts +12 -12
- package/tests/fireproof/all-gateway.test.ts +201 -193
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
- package/tests/fireproof/crdt.test.ts +67 -16
- package/tests/fireproof/database.test.ts +183 -21
- package/tests/fireproof/fireproof.test.ts +83 -74
- package/tests/fireproof/hello.test.ts +18 -14
- package/tests/fireproof/indexer.test.ts +53 -43
- package/tests/fireproof/utils.test.ts +18 -6
- package/tests/gateway/file/loader-config.test.ts +303 -0
- package/tests/gateway/indexdb/loader-config.test.ts +75 -0
- package/tests/helpers.ts +27 -9
- package/tests/react/useFireproof.test.tsx +1 -1
- package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
- package/utils-STA2C35G.js.map +1 -0
- package/chunk-OFGPKRCM.js.map +0 -1
- package/chunk-WS3YRPIA.js +0 -75
- package/chunk-WS3YRPIA.js.map +0 -1
- package/gateway-5FCWPX5W.js.map +0 -1
- package/gateway-H7UD6TNB.js.map +0 -1
- package/mem-filesystem-YPPJV7Q2.js +0 -41
- package/mem-filesystem-YPPJV7Q2.js.map +0 -1
- package/node-filesystem-INX4ZTHE.js.map +0 -1
- package/tests/fireproof/config.test.ts +0 -172
- /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
- /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.cjs
CHANGED
@@ -30,6 +30,47 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
30
30
|
));
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
32
32
|
|
33
|
+
// src/types.ts
|
34
|
+
function isFalsy(value) {
|
35
|
+
return value === false && value === null && value === void 0;
|
36
|
+
}
|
37
|
+
function throwFalsy(value) {
|
38
|
+
if (isFalsy(value)) {
|
39
|
+
throw new Error("value is Falsy");
|
40
|
+
}
|
41
|
+
return value;
|
42
|
+
}
|
43
|
+
function falsyToUndef(value) {
|
44
|
+
if (isFalsy(value)) {
|
45
|
+
return void 0;
|
46
|
+
}
|
47
|
+
return value;
|
48
|
+
}
|
49
|
+
var PARAM;
|
50
|
+
var init_types = __esm({
|
51
|
+
"src/types.ts"() {
|
52
|
+
"use strict";
|
53
|
+
PARAM = /* @__PURE__ */ ((PARAM2) => {
|
54
|
+
PARAM2["SUFFIX"] = "suffix";
|
55
|
+
PARAM2["URL_GEN"] = "urlGen";
|
56
|
+
PARAM2["STORE_KEY"] = "storekey";
|
57
|
+
PARAM2["STORE"] = "store";
|
58
|
+
PARAM2["KEY"] = "key";
|
59
|
+
PARAM2["INDEX"] = "index";
|
60
|
+
PARAM2["NAME"] = "name";
|
61
|
+
PARAM2["VERSION"] = "version";
|
62
|
+
PARAM2["FRAG_SIZE"] = "fragSize";
|
63
|
+
PARAM2["IV_VERIFY"] = "ivVerify";
|
64
|
+
PARAM2["IV_HASH"] = "ivHash";
|
65
|
+
PARAM2["FRAG_FID"] = "fid";
|
66
|
+
PARAM2["FRAG_OFS"] = "ofs";
|
67
|
+
PARAM2["FRAG_LEN"] = "len";
|
68
|
+
PARAM2["FRAG_HEAD"] = "headerSize";
|
69
|
+
return PARAM2;
|
70
|
+
})(PARAM || {});
|
71
|
+
}
|
72
|
+
});
|
73
|
+
|
33
74
|
// src/utils.ts
|
34
75
|
function presetEnv() {
|
35
76
|
const penv = new Map([
|
@@ -87,6 +128,10 @@ function ensureLogger(sthis, componentName, ctx) {
|
|
87
128
|
exposeStack = true;
|
88
129
|
delete ctx.exposeStack;
|
89
130
|
}
|
131
|
+
if ("exposeStack" in ctx) {
|
132
|
+
exposeStack = true;
|
133
|
+
delete ctx.exposeStack;
|
134
|
+
}
|
90
135
|
if ("this" in ctx) {
|
91
136
|
cLogger.Str("this", sthis.nextId(4).str);
|
92
137
|
delete ctx.this;
|
@@ -153,10 +198,13 @@ function ensureLogger(sthis, componentName, ctx) {
|
|
153
198
|
logger.SetExposeStack(true);
|
154
199
|
}
|
155
200
|
const out = cLogger.Logger();
|
201
|
+
if (sthis.env.get("FP_CONSTRUCTOR_DEBUG")) {
|
202
|
+
out.Debug().Msg("constructor");
|
203
|
+
}
|
156
204
|
return out;
|
157
205
|
}
|
158
206
|
function getStore(url, sthis, joiner) {
|
159
|
-
const store = url.getParam("store");
|
207
|
+
const store = url.getParam("store" /* STORE */);
|
160
208
|
switch (store) {
|
161
209
|
case "data":
|
162
210
|
case "wal":
|
@@ -164,20 +212,21 @@ function getStore(url, sthis, joiner) {
|
|
164
212
|
break;
|
165
213
|
default:
|
166
214
|
throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
|
215
|
+
throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
|
167
216
|
}
|
168
217
|
let name = store;
|
169
218
|
if (url.hasParam("index")) {
|
170
|
-
name = joiner(url.getParam("index") || "idx", name);
|
219
|
+
name = joiner(url.getParam("index" /* INDEX */) || "idx", name);
|
171
220
|
}
|
172
221
|
return { store, name };
|
173
222
|
}
|
174
223
|
function getKey(url, logger) {
|
175
|
-
const result = url.getParam("key");
|
224
|
+
const result = url.getParam("key" /* KEY */);
|
176
225
|
if (!result) throw logger.Error().Str("url", url.toString()).Msg(`key not found`).AsError();
|
177
226
|
return result;
|
178
227
|
}
|
179
228
|
function getName(sthis, url) {
|
180
|
-
let result = url.getParam("name");
|
229
|
+
let result = url.getParam("name" /* NAME */);
|
181
230
|
if (!result) {
|
182
231
|
result = sthis.pathOps.dirname(url.pathname);
|
183
232
|
if (result.length === 0) {
|
@@ -197,17 +246,6 @@ function isNotFoundError(e) {
|
|
197
246
|
if (e.code === "ENOENT") return true;
|
198
247
|
return false;
|
199
248
|
}
|
200
|
-
function dataDir(sthis, name, base) {
|
201
|
-
if (!base) {
|
202
|
-
if (!(0, import_cement.runtimeFn)().isBrowser) {
|
203
|
-
const home = sthis.env.get("HOME") || "./";
|
204
|
-
base = sthis.env.get("FP_STORAGE_URL") || `file://${sthis.pathOps.join(home, ".fireproof")}`;
|
205
|
-
} else {
|
206
|
-
base = sthis.env.get("FP_STORAGE_URL") || `indexdb://fp`;
|
207
|
-
}
|
208
|
-
}
|
209
|
-
return import_cement.URI.from(base.toString()).build().setParam("name", name || "").URI();
|
210
|
-
}
|
211
249
|
function UInt8ArrayEqual(a, b) {
|
212
250
|
if (a.length !== b.length) {
|
213
251
|
return false;
|
@@ -224,6 +262,7 @@ var init_utils = __esm({
|
|
224
262
|
"src/utils.ts"() {
|
225
263
|
"use strict";
|
226
264
|
import_cement = require("@adviser/cement");
|
265
|
+
init_types();
|
227
266
|
import_base58 = require("multiformats/bases/base58");
|
228
267
|
globalLogger = new import_cement.LoggerImpl();
|
229
268
|
registerFP_DEBUG = new import_cement.ResolveOnce();
|
@@ -274,6 +313,9 @@ var init_utils = __esm({
|
|
274
313
|
dirname(path) {
|
275
314
|
return path.split("/").slice(0, -1).join("/");
|
276
315
|
}
|
316
|
+
basename(path) {
|
317
|
+
return path.split("/").pop() || "";
|
318
|
+
}
|
277
319
|
// homedir() {
|
278
320
|
// throw new Error("SysContainer:homedir is not available in seeded state");
|
279
321
|
// }
|
@@ -292,45 +334,63 @@ var init_utils = __esm({
|
|
292
334
|
}
|
293
335
|
});
|
294
336
|
|
295
|
-
// src/runtime/gateways/file/
|
296
|
-
var
|
297
|
-
__export(
|
298
|
-
|
337
|
+
// src/runtime/gateways/file/deno-filesystem.ts
|
338
|
+
var deno_filesystem_exports = {};
|
339
|
+
__export(deno_filesystem_exports, {
|
340
|
+
DenoFileSystem: () => DenoFileSystem
|
299
341
|
});
|
300
|
-
var
|
301
|
-
var
|
302
|
-
"src/runtime/gateways/file/
|
342
|
+
var DenoFileSystem;
|
343
|
+
var init_deno_filesystem = __esm({
|
344
|
+
"src/runtime/gateways/file/deno-filesystem.ts"() {
|
303
345
|
"use strict";
|
304
|
-
|
305
|
-
init_utils2();
|
306
|
-
MemFileSystem = class {
|
346
|
+
DenoFileSystem = class {
|
307
347
|
async start() {
|
348
|
+
this.fs = Deno;
|
308
349
|
return this;
|
309
350
|
}
|
310
|
-
mkdir(path, options) {
|
311
|
-
return
|
351
|
+
async mkdir(path, options) {
|
352
|
+
return this.fs?.mkdir(path, options).then(() => path);
|
312
353
|
}
|
313
|
-
readdir(path
|
314
|
-
|
354
|
+
async readdir(path) {
|
355
|
+
const ret = [];
|
356
|
+
for await (const dirEntry of this.fs.readdir(path)) {
|
357
|
+
ret.push(dirEntry.name);
|
358
|
+
}
|
359
|
+
return ret;
|
315
360
|
}
|
316
|
-
rm(path, options) {
|
317
|
-
return
|
361
|
+
async rm(path, options) {
|
362
|
+
return this.fs?.rm(path, options);
|
318
363
|
}
|
319
|
-
copyFile(source, destination) {
|
320
|
-
return
|
364
|
+
async copyFile(source, destination) {
|
365
|
+
return this.fs?.copyFile(source, destination);
|
321
366
|
}
|
322
|
-
async readfile(path
|
323
|
-
|
324
|
-
return toArrayBuffer(ret);
|
367
|
+
async readfile(path) {
|
368
|
+
return this.fs.readFile(path);
|
325
369
|
}
|
326
|
-
stat(path) {
|
327
|
-
|
370
|
+
async stat(path) {
|
371
|
+
const x = await this.fs.stat(path);
|
372
|
+
return {
|
373
|
+
isFile: () => x.isFile,
|
374
|
+
isDirectory: () => x.isDirectory,
|
375
|
+
isBlockDevice: () => !!x.isBlockDevice,
|
376
|
+
isCharacterDevice: () => !!x.isCharDevice,
|
377
|
+
isSymbolicLink: () => !!x.isSymlink,
|
378
|
+
isFIFO: () => !!x.isFifo,
|
379
|
+
isSocket: () => !!x.isSocket,
|
380
|
+
uid: x.uid,
|
381
|
+
gid: x.gid,
|
382
|
+
size: x.size,
|
383
|
+
atime: x.atime,
|
384
|
+
mtime: x.mtime,
|
385
|
+
ctime: x.birthtime,
|
386
|
+
birthtime: x.birthtime
|
387
|
+
};
|
328
388
|
}
|
329
|
-
unlink(path) {
|
330
|
-
return
|
389
|
+
async unlink(path) {
|
390
|
+
return this.fs?.unlink(path);
|
331
391
|
}
|
332
|
-
writefile(path, data) {
|
333
|
-
return
|
392
|
+
async writefile(path, data) {
|
393
|
+
return this.fs?.writeFile(path, Buffer.from(data));
|
334
394
|
}
|
335
395
|
};
|
336
396
|
}
|
@@ -341,12 +401,11 @@ var node_filesystem_exports = {};
|
|
341
401
|
__export(node_filesystem_exports, {
|
342
402
|
NodeFileSystem: () => NodeFileSystem
|
343
403
|
});
|
344
|
-
var
|
404
|
+
var NodeFileSystem;
|
345
405
|
var init_node_filesystem = __esm({
|
346
406
|
"src/runtime/gateways/file/node-filesystem.ts"() {
|
347
407
|
"use strict";
|
348
408
|
init_utils2();
|
349
|
-
import_cement4 = require("@adviser/cement");
|
350
409
|
NodeFileSystem = class {
|
351
410
|
async start() {
|
352
411
|
this.fs = await import("fs/promises");
|
@@ -369,15 +428,12 @@ var init_node_filesystem = __esm({
|
|
369
428
|
return toArrayBuffer(ret);
|
370
429
|
}
|
371
430
|
stat(path) {
|
372
|
-
return this.fs
|
431
|
+
return this.fs.stat(path);
|
373
432
|
}
|
374
433
|
async unlink(path) {
|
375
434
|
return this.fs?.unlink(path);
|
376
435
|
}
|
377
436
|
async writefile(path, data) {
|
378
|
-
if ((0, import_cement4.runtimeFn)().isDeno) {
|
379
|
-
return this.fs?.writeFile(path, data);
|
380
|
-
}
|
381
437
|
return this.fs?.writeFile(path, Buffer.from(data));
|
382
438
|
}
|
383
439
|
};
|
@@ -393,49 +449,34 @@ __export(utils_exports, {
|
|
393
449
|
toArrayBuffer: () => toArrayBuffer
|
394
450
|
});
|
395
451
|
async function getFileSystem(url) {
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
// case 'deno': {
|
406
|
-
// const { DenoFileSystem } = await import("./deno-filesystem.js");
|
407
|
-
// fs = new DenoFileSystem();
|
408
|
-
// break;
|
409
|
-
// }
|
410
|
-
case "node": {
|
411
|
-
const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
|
412
|
-
fs2 = new NodeFileSystem2();
|
413
|
-
break;
|
414
|
-
}
|
415
|
-
case "sys":
|
416
|
-
default: {
|
417
|
-
return getFileSystem(url.build().setParam("fs", "node").URI());
|
418
|
-
}
|
452
|
+
let fs;
|
453
|
+
if ((0, import_cement4.runtimeFn)().isDeno) {
|
454
|
+
const { DenoFileSystem: DenoFileSystem2 } = await Promise.resolve().then(() => (init_deno_filesystem(), deno_filesystem_exports));
|
455
|
+
fs = new DenoFileSystem2();
|
456
|
+
} else if ((0, import_cement4.runtimeFn)().isNodeIsh) {
|
457
|
+
const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
|
458
|
+
fs = new NodeFileSystem2();
|
459
|
+
} else {
|
460
|
+
throw new Error("unsupported runtime");
|
419
461
|
}
|
420
|
-
return
|
462
|
+
return fs.start();
|
421
463
|
}
|
422
464
|
function getPath(url, sthis) {
|
423
465
|
const basePath = url.pathname;
|
424
|
-
const name = url.getParam("name");
|
466
|
+
const name = url.getParam("name" /* NAME */);
|
425
467
|
if (name) {
|
426
|
-
|
427
|
-
if (!version) throw sthis.logger.Error().Url(url).Msg(`version not found`).AsError();
|
428
|
-
return sthis.pathOps.join(basePath, version, name);
|
468
|
+
return sthis.pathOps.join(basePath, name);
|
429
469
|
}
|
430
470
|
return sthis.pathOps.join(basePath);
|
431
471
|
}
|
432
472
|
function getFileName(url, sthis) {
|
433
|
-
const key = url.getParam("key");
|
473
|
+
const key = url.getParam("key" /* KEY */);
|
434
474
|
if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
|
435
475
|
const res = getStore(url, sthis, (...a) => a.join("-"));
|
436
476
|
switch (res.store) {
|
437
|
-
case "data":
|
438
|
-
return sthis.pathOps.join(res.name, key +
|
477
|
+
case "data": {
|
478
|
+
return sthis.pathOps.join(res.name, key + (url.getParam("suffix" /* SUFFIX */) || ""));
|
479
|
+
}
|
439
480
|
case "wal":
|
440
481
|
case "meta":
|
441
482
|
return sthis.pathOps.join(res.name, key + ".json");
|
@@ -454,10 +495,13 @@ function toArrayBuffer(buffer) {
|
|
454
495
|
}
|
455
496
|
return view;
|
456
497
|
}
|
498
|
+
var import_cement4;
|
457
499
|
var init_utils2 = __esm({
|
458
500
|
"src/runtime/gateways/file/utils.ts"() {
|
459
501
|
"use strict";
|
502
|
+
import_cement4 = require("@adviser/cement");
|
460
503
|
init_utils();
|
504
|
+
init_types();
|
461
505
|
}
|
462
506
|
});
|
463
507
|
|
@@ -572,6 +616,40 @@ var init_key_bag_indexdb = __esm({
|
|
572
616
|
}
|
573
617
|
});
|
574
618
|
|
619
|
+
// src/runtime/key-bag-memory.ts
|
620
|
+
var key_bag_memory_exports = {};
|
621
|
+
__export(key_bag_memory_exports, {
|
622
|
+
KeyBagProviderMemory: () => KeyBagProviderMemory
|
623
|
+
});
|
624
|
+
var memoryKeyBag, KeyBagProviderMemory;
|
625
|
+
var init_key_bag_memory = __esm({
|
626
|
+
"src/runtime/key-bag-memory.ts"() {
|
627
|
+
"use strict";
|
628
|
+
memoryKeyBag = /* @__PURE__ */ new Map();
|
629
|
+
KeyBagProviderMemory = class {
|
630
|
+
constructor(url, sthis) {
|
631
|
+
this.url = url;
|
632
|
+
this.sthis = sthis;
|
633
|
+
}
|
634
|
+
key(id) {
|
635
|
+
return `${this.url.pathname}/${id}`;
|
636
|
+
}
|
637
|
+
async get(id) {
|
638
|
+
const binKeyItem = memoryKeyBag.get(this.key(id));
|
639
|
+
if (binKeyItem) {
|
640
|
+
const ki = JSON.parse(this.sthis.txt.decode(binKeyItem));
|
641
|
+
return ki;
|
642
|
+
}
|
643
|
+
return void 0;
|
644
|
+
}
|
645
|
+
async set(id, item) {
|
646
|
+
const p = this.sthis.txt.encode(JSON.stringify(item, null, 2));
|
647
|
+
memoryKeyBag.set(this.key(id), p);
|
648
|
+
}
|
649
|
+
};
|
650
|
+
}
|
651
|
+
});
|
652
|
+
|
575
653
|
// src/runtime/gateways/file/version.ts
|
576
654
|
var FILESTORE_VERSION;
|
577
655
|
var init_version = __esm({
|
@@ -587,23 +665,24 @@ __export(gateway_exports, {
|
|
587
665
|
FileGateway: () => FileGateway,
|
588
666
|
FileTestStore: () => FileTestStore
|
589
667
|
});
|
590
|
-
var
|
668
|
+
var import_cement12, versionFiles, FileGateway, FileTestStore;
|
591
669
|
var init_gateway = __esm({
|
592
670
|
"src/runtime/gateways/file/gateway.ts"() {
|
593
671
|
"use strict";
|
594
672
|
init_version();
|
595
|
-
|
673
|
+
import_cement12 = require("@adviser/cement");
|
596
674
|
init_utils();
|
597
675
|
init_utils2();
|
598
|
-
|
676
|
+
init_types();
|
677
|
+
versionFiles = new import_cement12.KeyedResolvOnce();
|
599
678
|
FileGateway = class {
|
600
679
|
get fs() {
|
601
680
|
if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
|
602
681
|
return this._fs;
|
603
682
|
}
|
604
683
|
constructor(sthis) {
|
605
|
-
this.sthis = sthis;
|
606
|
-
this.logger = sthis.logger;
|
684
|
+
this.sthis = ensureSuperLog(sthis, "FileGateway", { this: 1 });
|
685
|
+
this.logger = this.sthis.logger;
|
607
686
|
}
|
608
687
|
async getVersionFromFile(path, logger) {
|
609
688
|
return versionFiles.get(path).once(async () => {
|
@@ -625,34 +704,33 @@ var init_gateway = __esm({
|
|
625
704
|
});
|
626
705
|
}
|
627
706
|
start(baseURL) {
|
628
|
-
return (0,
|
707
|
+
return (0, import_cement12.exception2Result)(async () => {
|
629
708
|
this._fs = await getFileSystem(baseURL);
|
630
|
-
await this.fs.start();
|
631
709
|
const url = baseURL.build();
|
632
|
-
url.defParam("version"
|
710
|
+
url.defParam("version" /* VERSION */, FILESTORE_VERSION);
|
633
711
|
const dbUrl = await this.buildUrl(url.URI(), "dummy");
|
634
712
|
const dbdirFile = this.getFilePath(dbUrl.Ok());
|
635
713
|
await this.fs.mkdir(this.sthis.pathOps.dirname(dbdirFile), { recursive: true });
|
636
714
|
const dbroot = this.sthis.pathOps.dirname(dbdirFile);
|
637
715
|
this.logger.Debug().Url(url.URI()).Str("dbroot", dbroot).Msg("start");
|
638
|
-
url.setParam("version"
|
716
|
+
url.setParam("version" /* VERSION */, await this.getVersionFromFile(dbroot, this.logger));
|
639
717
|
return url.URI();
|
640
718
|
});
|
641
719
|
}
|
642
720
|
async buildUrl(baseUrl, key) {
|
643
|
-
return
|
721
|
+
return import_cement12.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI());
|
644
722
|
}
|
645
723
|
async close() {
|
646
|
-
return
|
724
|
+
return import_cement12.Result.Ok(void 0);
|
647
725
|
}
|
648
726
|
// abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
|
649
727
|
getFilePath(url) {
|
650
|
-
const key = url.getParam("key");
|
728
|
+
const key = url.getParam("key" /* KEY */);
|
651
729
|
if (!key) throw this.logger.Error().Url(url).Msg(`key not found`).AsError();
|
652
730
|
return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
|
653
731
|
}
|
654
732
|
async put(url, body) {
|
655
|
-
return (0,
|
733
|
+
return (0, import_cement12.exception2Result)(async () => {
|
656
734
|
const file = await this.getFilePath(url);
|
657
735
|
this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
|
658
736
|
await this.fs.writefile(file, body);
|
@@ -662,19 +740,19 @@ var init_gateway = __esm({
|
|
662
740
|
return exceptionWrapper(async () => {
|
663
741
|
const file = this.getFilePath(url);
|
664
742
|
try {
|
743
|
+
this.logger.Debug().Url(url).Str("file", file).Msg("get");
|
665
744
|
const res = await this.fs.readfile(file);
|
666
|
-
|
667
|
-
return import_cement11.Result.Ok(new Uint8Array(res));
|
745
|
+
return import_cement12.Result.Ok(new Uint8Array(res));
|
668
746
|
} catch (e) {
|
669
747
|
if (isNotFoundError(e)) {
|
670
|
-
return
|
748
|
+
return import_cement12.Result.Err(new NotFoundError(`file not found: ${file}`));
|
671
749
|
}
|
672
|
-
return
|
750
|
+
return import_cement12.Result.Err(e);
|
673
751
|
}
|
674
752
|
});
|
675
753
|
}
|
676
754
|
async delete(url) {
|
677
|
-
return (0,
|
755
|
+
return (0, import_cement12.exception2Result)(async () => {
|
678
756
|
await this.fs.unlink(this.getFilePath(url));
|
679
757
|
});
|
680
758
|
}
|
@@ -700,7 +778,7 @@ var init_gateway = __esm({
|
|
700
778
|
}
|
701
779
|
}
|
702
780
|
}
|
703
|
-
return
|
781
|
+
return import_cement12.Result.Ok(void 0);
|
704
782
|
}
|
705
783
|
};
|
706
784
|
FileTestStore = class {
|
@@ -709,7 +787,7 @@ var init_gateway = __esm({
|
|
709
787
|
this.sthis = sthis;
|
710
788
|
}
|
711
789
|
async get(iurl, key) {
|
712
|
-
const url = iurl.build().setParam("key"
|
790
|
+
const url = iurl.build().setParam("key" /* KEY */, key).URI();
|
713
791
|
const dbFile = this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
|
714
792
|
this.logger.Debug().Url(url).Str("dbFile", dbFile).Msg("get");
|
715
793
|
const buffer = await (await getFileSystem(url)).readfile(dbFile);
|
@@ -737,7 +815,7 @@ __export(gateway_exports2, {
|
|
737
815
|
getIndexDBName: () => getIndexDBName
|
738
816
|
});
|
739
817
|
function ensureVersion(url) {
|
740
|
-
return url.build().defParam("version"
|
818
|
+
return url.build().defParam("version" /* VERSION */, INDEXDB_VERSION).URI();
|
741
819
|
}
|
742
820
|
function sanitzeKey(key) {
|
743
821
|
if (key.length === 1) {
|
@@ -758,17 +836,17 @@ async function connectIdb(url, sthis) {
|
|
758
836
|
}
|
759
837
|
});
|
760
838
|
const found = await db.get("version", "version");
|
761
|
-
const version = ensureVersion(url).getParam("version");
|
839
|
+
const version = ensureVersion(url).getParam("version" /* VERSION */);
|
762
840
|
if (!found) {
|
763
841
|
await db.put("version", { version }, "version");
|
764
842
|
} else if (found.version !== version) {
|
765
|
-
sthis.logger.Warn().
|
843
|
+
sthis.logger.Warn().Url(url).Str("version", version).Str("found", found.version).Msg("version mismatch");
|
766
844
|
}
|
767
845
|
return { db, dbName, version, url };
|
768
846
|
});
|
769
847
|
return {
|
770
848
|
...once,
|
771
|
-
url: url.build().setParam("version"
|
849
|
+
url: url.build().setParam("version" /* VERSION */, once.version).URI()
|
772
850
|
};
|
773
851
|
}
|
774
852
|
function joinDBName(...names) {
|
@@ -777,7 +855,7 @@ function joinDBName(...names) {
|
|
777
855
|
function getIndexDBName(iurl, sthis) {
|
778
856
|
const url = ensureVersion(iurl);
|
779
857
|
const fullDb = url.pathname.replace(/^\/+/, "").replace(/\?.*$/, "");
|
780
|
-
const dbName = url.getParam("name");
|
858
|
+
const dbName = url.getParam("name" /* NAME */);
|
781
859
|
if (!dbName) throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
|
782
860
|
const result = joinDBName(fullDb, dbName);
|
783
861
|
const objStore = getStore(url, sthis, joinDBName).name;
|
@@ -789,15 +867,16 @@ function getIndexDBName(iurl, sthis) {
|
|
789
867
|
dbName
|
790
868
|
};
|
791
869
|
}
|
792
|
-
var import_idb2,
|
870
|
+
var import_idb2, import_cement13, onceIndexDB, IndexDBGateway, IndexDBTestStore;
|
793
871
|
var init_gateway2 = __esm({
|
794
872
|
"src/runtime/gateways/indexdb/gateway.ts"() {
|
795
873
|
"use strict";
|
796
874
|
import_idb2 = require("idb");
|
797
|
-
|
875
|
+
import_cement13 = require("@adviser/cement");
|
798
876
|
init_version2();
|
799
877
|
init_utils();
|
800
|
-
|
878
|
+
init_types();
|
879
|
+
onceIndexDB = new import_cement13.KeyedResolvOnce();
|
801
880
|
IndexDBGateway = class {
|
802
881
|
constructor(sthis) {
|
803
882
|
this._db = {};
|
@@ -805,7 +884,7 @@ var init_gateway2 = __esm({
|
|
805
884
|
this.sthis = sthis;
|
806
885
|
}
|
807
886
|
async start(baseURL) {
|
808
|
-
return (0,
|
887
|
+
return (0, import_cement13.exception2Result)(async () => {
|
809
888
|
this.logger.Debug().Url(baseURL).Msg("starting");
|
810
889
|
await this.sthis.start();
|
811
890
|
const ic = await connectIdb(baseURL, this.sthis);
|
@@ -815,10 +894,10 @@ var init_gateway2 = __esm({
|
|
815
894
|
});
|
816
895
|
}
|
817
896
|
async close() {
|
818
|
-
return
|
897
|
+
return import_cement13.Result.Ok(void 0);
|
819
898
|
}
|
820
899
|
async destroy(baseUrl) {
|
821
|
-
return (0,
|
900
|
+
return (0, import_cement13.exception2Result)(async () => {
|
822
901
|
const type = getStore(baseUrl, this.sthis, joinDBName).name;
|
823
902
|
const idb = this._db;
|
824
903
|
const trans = idb.transaction(type, "readwrite");
|
@@ -834,7 +913,7 @@ var init_gateway2 = __esm({
|
|
834
913
|
});
|
835
914
|
}
|
836
915
|
buildUrl(baseUrl, key) {
|
837
|
-
return Promise.resolve(
|
916
|
+
return Promise.resolve(import_cement13.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
|
838
917
|
}
|
839
918
|
async get(url) {
|
840
919
|
return exceptionWrapper(async () => {
|
@@ -845,13 +924,13 @@ var init_gateway2 = __esm({
|
|
845
924
|
const bytes = await tx.objectStore(store).get(sanitzeKey(key));
|
846
925
|
await tx.done;
|
847
926
|
if (!bytes) {
|
848
|
-
return
|
927
|
+
return import_cement13.Result.Err(new NotFoundError(`missing ${key}`));
|
849
928
|
}
|
850
|
-
return
|
929
|
+
return import_cement13.Result.Ok(bytes);
|
851
930
|
});
|
852
931
|
}
|
853
932
|
async put(url, value) {
|
854
|
-
return (0,
|
933
|
+
return (0, import_cement13.exception2Result)(async () => {
|
855
934
|
const key = getKey(url, this.logger);
|
856
935
|
const store = getStore(url, this.sthis, joinDBName).name;
|
857
936
|
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
|
@@ -861,14 +940,14 @@ var init_gateway2 = __esm({
|
|
861
940
|
});
|
862
941
|
}
|
863
942
|
async delete(url) {
|
864
|
-
return (0,
|
943
|
+
return (0, import_cement13.exception2Result)(async () => {
|
865
944
|
const key = getKey(url, this.logger);
|
866
945
|
const store = getStore(url, this.sthis, joinDBName).name;
|
867
946
|
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
|
868
947
|
const tx = this._db.transaction([store], "readwrite");
|
869
948
|
await tx.objectStore(store).delete(sanitzeKey(key));
|
870
949
|
await tx.done;
|
871
|
-
return
|
950
|
+
return import_cement13.Result.Ok(void 0);
|
872
951
|
});
|
873
952
|
}
|
874
953
|
};
|
@@ -896,15 +975,16 @@ var init_gateway2 = __esm({
|
|
896
975
|
var src_exports = {};
|
897
976
|
__export(src_exports, {
|
898
977
|
CRDT: () => CRDT,
|
899
|
-
|
978
|
+
DatabaseFactory: () => DatabaseFactory,
|
979
|
+
DatabaseShell: () => DatabaseShell,
|
900
980
|
Index: () => Index,
|
901
981
|
NotFoundError: () => NotFoundError,
|
902
982
|
PACKAGE_VERSION: () => PACKAGE_VERSION,
|
983
|
+
PARAM: () => PARAM,
|
903
984
|
Result: () => import_cement.Result,
|
904
985
|
UInt8ArrayEqual: () => UInt8ArrayEqual,
|
905
986
|
blockstore: () => blockstore_exports,
|
906
987
|
bs: () => blockstore_exports,
|
907
|
-
dataDir: () => dataDir,
|
908
988
|
ensureLogger: () => ensureLogger,
|
909
989
|
ensureSuperLog: () => ensureSuperLog,
|
910
990
|
ensureSuperThis: () => ensureSuperThis,
|
@@ -915,16 +995,19 @@ __export(src_exports, {
|
|
915
995
|
getName: () => getName,
|
916
996
|
getStore: () => getStore,
|
917
997
|
index: () => index,
|
998
|
+
isDatabase: () => isDatabase,
|
918
999
|
isFalsy: () => isFalsy,
|
919
1000
|
isNotFoundError: () => isNotFoundError,
|
1001
|
+
keyConfigOpts: () => keyConfigOpts,
|
920
1002
|
rt: () => runtime_exports,
|
921
1003
|
runtime: () => runtime_exports,
|
922
|
-
throwFalsy: () => throwFalsy
|
1004
|
+
throwFalsy: () => throwFalsy,
|
1005
|
+
toStoreURIRuntime: () => toStoreURIRuntime
|
923
1006
|
});
|
924
1007
|
module.exports = __toCommonJS(src_exports);
|
925
1008
|
|
926
1009
|
// src/database.ts
|
927
|
-
var
|
1010
|
+
var import_cement18 = require("@adviser/cement");
|
928
1011
|
|
929
1012
|
// src/write-queue.ts
|
930
1013
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -967,86 +1050,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
967
1050
|
}
|
968
1051
|
|
969
1052
|
// src/crdt.ts
|
970
|
-
var
|
971
|
-
|
972
|
-
// src/runtime/wait-pr-multiformats/block.ts
|
973
|
-
var block_exports = {};
|
974
|
-
__export(block_exports, {
|
975
|
-
Block: () => Block,
|
976
|
-
create: () => create,
|
977
|
-
createUnsafe: () => createUnsafe,
|
978
|
-
decode: () => decode,
|
979
|
-
encode: () => encode
|
980
|
-
});
|
981
|
-
var import_multiformats = require("multiformats");
|
982
|
-
var import_block = require("multiformats/block");
|
983
|
-
var Block = import_block.Block;
|
984
|
-
async function decode({
|
985
|
-
bytes,
|
986
|
-
codec: codec3,
|
987
|
-
hasher: hasher7
|
988
|
-
}) {
|
989
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
990
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
991
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
992
|
-
const hash = await hasher7.digest(bytes);
|
993
|
-
const cid = import_multiformats.CID.create(1, codec3.code, hash);
|
994
|
-
return new import_block.Block({ value, bytes, cid });
|
995
|
-
}
|
996
|
-
async function encode({
|
997
|
-
value,
|
998
|
-
codec: codec3,
|
999
|
-
hasher: hasher7
|
1000
|
-
}) {
|
1001
|
-
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
1002
|
-
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
1003
|
-
const bytes = await Promise.resolve(codec3.encode(value));
|
1004
|
-
const hash = await hasher7.digest(bytes);
|
1005
|
-
const cid = import_multiformats.CID.create(1, codec3.code, hash);
|
1006
|
-
return new import_block.Block({ value, bytes, cid });
|
1007
|
-
}
|
1008
|
-
async function create({
|
1009
|
-
bytes,
|
1010
|
-
cid,
|
1011
|
-
hasher: hasher7,
|
1012
|
-
codec: codec3
|
1013
|
-
}) {
|
1014
|
-
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
1015
|
-
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
1016
|
-
const value = await Promise.resolve(codec3.decode(bytes));
|
1017
|
-
const hash = await hasher7.digest(bytes);
|
1018
|
-
if (!import_multiformats.bytes.equals(cid.multihash.bytes, hash.bytes)) {
|
1019
|
-
throw new Error("CID hash does not match bytes");
|
1020
|
-
}
|
1021
|
-
return createUnsafe({
|
1022
|
-
bytes,
|
1023
|
-
cid,
|
1024
|
-
value,
|
1025
|
-
codec: codec3
|
1026
|
-
});
|
1027
|
-
}
|
1028
|
-
async function createUnsafe({
|
1029
|
-
bytes,
|
1030
|
-
cid,
|
1031
|
-
value: maybeValue,
|
1032
|
-
codec: codec3
|
1033
|
-
}) {
|
1034
|
-
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
1035
|
-
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
1036
|
-
return new Block({
|
1037
|
-
cid,
|
1038
|
-
bytes,
|
1039
|
-
value
|
1040
|
-
});
|
1041
|
-
}
|
1042
|
-
|
1043
|
-
// src/crdt-helpers.ts
|
1044
|
-
var import_link = require("multiformats/link");
|
1045
|
-
var import_sha25 = require("multiformats/hashes/sha2");
|
1046
|
-
var codec = __toESM(require("@ipld/dag-cbor"), 1);
|
1047
|
-
var import_crdt = require("@web3-storage/pail/crdt");
|
1048
|
-
var import_clock2 = require("@web3-storage/pail/clock");
|
1049
|
-
var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
|
1053
|
+
var import_cement17 = require("@adviser/cement");
|
1050
1054
|
|
1051
1055
|
// src/blockstore/index.ts
|
1052
1056
|
var blockstore_exports = {};
|
@@ -1059,8 +1063,11 @@ __export(blockstore_exports, {
|
|
1059
1063
|
FragmentGateway: () => FragmentGateway,
|
1060
1064
|
Loader: () => Loader,
|
1061
1065
|
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
1062
|
-
|
1063
|
-
|
1066
|
+
ensureStoreEnDeFile: () => ensureStoreEnDeFile,
|
1067
|
+
fileGatewayFactoryItem: () => fileGatewayFactoryItem,
|
1068
|
+
getDefaultURI: () => getDefaultURI,
|
1069
|
+
getGatewayFactoryItem: () => getGatewayFactoryItem,
|
1070
|
+
getStartedGateway: () => getStartedGateway,
|
1064
1071
|
parseCarFile: () => parseCarFile,
|
1065
1072
|
registerStoreProtocol: () => registerStoreProtocol,
|
1066
1073
|
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
@@ -1075,7 +1082,7 @@ function toCIDBlock(block) {
|
|
1075
1082
|
}
|
1076
1083
|
|
1077
1084
|
// src/blockstore/store-factory.ts
|
1078
|
-
var
|
1085
|
+
var import_cement15 = require("@adviser/cement");
|
1079
1086
|
|
1080
1087
|
// src/runtime/files.ts
|
1081
1088
|
var files_exports = {};
|
@@ -1151,258 +1158,24 @@ var UnixFSFileBuilder = class {
|
|
1151
1158
|
// src/blockstore/store.ts
|
1152
1159
|
var import_dag_json2 = require("@ipld/dag-json");
|
1153
1160
|
var import_cement10 = require("@adviser/cement");
|
1154
|
-
|
1155
|
-
// src/types.ts
|
1156
|
-
function isFalsy(value) {
|
1157
|
-
return value === false && value === null && value === void 0;
|
1158
|
-
}
|
1159
|
-
function throwFalsy(value) {
|
1160
|
-
if (isFalsy(value)) {
|
1161
|
-
throw new Error("value is Falsy");
|
1162
|
-
}
|
1163
|
-
return value;
|
1164
|
-
}
|
1165
|
-
function falsyToUndef(value) {
|
1166
|
-
if (isFalsy(value)) {
|
1167
|
-
return void 0;
|
1168
|
-
}
|
1169
|
-
return value;
|
1170
|
-
}
|
1171
|
-
|
1172
|
-
// src/blockstore/store.ts
|
1161
|
+
init_types();
|
1173
1162
|
init_utils();
|
1174
1163
|
|
1175
|
-
// src/blockstore/
|
1176
|
-
var import_p_limit = __toESM(require("p-limit"), 1);
|
1177
|
-
var import_car = require("@ipld/car");
|
1178
|
-
var import_cement7 = require("@adviser/cement");
|
1179
|
-
|
1180
|
-
// src/blockstore/loader-helpers.ts
|
1181
|
-
var import_sha2 = require("multiformats/hashes/sha2");
|
1182
|
-
var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
|
1183
|
-
async function parseCarFile(reader, logger) {
|
1184
|
-
const roots = await reader.getRoots();
|
1185
|
-
const header = await reader.get(roots[0]);
|
1186
|
-
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
1187
|
-
const dec = await decode({ bytes: header.bytes, hasher: import_sha2.sha256, codec: dagCodec });
|
1188
|
-
const fpvalue = dec.value;
|
1189
|
-
if (fpvalue && !fpvalue.fp) {
|
1190
|
-
throw logger.Error().Msg("missing fp").AsError();
|
1191
|
-
}
|
1192
|
-
return fpvalue.fp;
|
1193
|
-
}
|
1194
|
-
|
1195
|
-
// src/blockstore/transaction.ts
|
1196
|
-
var import_block3 = require("@web3-storage/pail/block");
|
1164
|
+
// src/blockstore/commit-queue.ts
|
1197
1165
|
var import_cement2 = require("@adviser/cement");
|
1198
|
-
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
parent.transactions.add(this);
|
1204
|
-
}
|
1205
|
-
this.parent = parent;
|
1206
|
-
}
|
1207
|
-
async get(cid) {
|
1208
|
-
return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
|
1209
|
-
}
|
1210
|
-
async superGet(cid) {
|
1211
|
-
return super.get(cid);
|
1166
|
+
var CommitQueue = class {
|
1167
|
+
constructor() {
|
1168
|
+
this.queue = [];
|
1169
|
+
this.processing = false;
|
1170
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
1212
1171
|
}
|
1213
|
-
|
1214
|
-
|
1215
|
-
const logger = ensureLogger(sthis, component, ctx);
|
1216
|
-
const store = opts.store || {};
|
1217
|
-
return {
|
1218
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1219
|
-
applyMeta: (meta, snap) => {
|
1172
|
+
waitIdle() {
|
1173
|
+
if (this.queue.length === 0 && !this.processing) {
|
1220
1174
|
return Promise.resolve();
|
1221
|
-
}
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
},
|
1226
|
-
autoCompact: 100,
|
1227
|
-
public: false,
|
1228
|
-
name: void 0,
|
1229
|
-
threshold: 1e3 * 1e3,
|
1230
|
-
...opts,
|
1231
|
-
logger,
|
1232
|
-
keyBag: opts.keyBag || {},
|
1233
|
-
crypto: (0, import_cement2.toCryptoRuntime)(opts.crypto),
|
1234
|
-
store,
|
1235
|
-
storeRuntime: toStoreRuntime(store, sthis)
|
1236
|
-
};
|
1237
|
-
}
|
1238
|
-
function blockstoreFactory(sthis, opts) {
|
1239
|
-
if (opts.name) {
|
1240
|
-
return new EncryptedBlockstore(sthis, opts);
|
1241
|
-
} else {
|
1242
|
-
return new BaseBlockstore(opts);
|
1243
|
-
}
|
1244
|
-
}
|
1245
|
-
var BaseBlockstore = class {
|
1246
|
-
constructor(ebOpts = {}) {
|
1247
|
-
this.transactions = /* @__PURE__ */ new Set();
|
1248
|
-
this.sthis = ensureSuperThis(ebOpts);
|
1249
|
-
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1250
|
-
this.logger = this.ebOpts.logger;
|
1251
|
-
}
|
1252
|
-
// ready: Promise<void>;
|
1253
|
-
ready() {
|
1254
|
-
return Promise.resolve();
|
1255
|
-
}
|
1256
|
-
async close() {
|
1257
|
-
}
|
1258
|
-
async destroy() {
|
1259
|
-
}
|
1260
|
-
async get(cid) {
|
1261
|
-
if (!cid) throw this.logger.Error().Msg("required cid").AsError();
|
1262
|
-
for (const f of this.transactions) {
|
1263
|
-
const v = await f.superGet(cid);
|
1264
|
-
if (v) return v;
|
1265
|
-
}
|
1266
|
-
}
|
1267
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1268
|
-
async put(cid, block) {
|
1269
|
-
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
1270
|
-
}
|
1271
|
-
// TransactionMeta
|
1272
|
-
async transaction(fn, _opts) {
|
1273
|
-
const t = new CarTransaction(this, _opts);
|
1274
|
-
const done = await fn(t);
|
1275
|
-
this.lastTxMeta = done;
|
1276
|
-
return { t, meta: done };
|
1277
|
-
}
|
1278
|
-
async *entries() {
|
1279
|
-
const seen = /* @__PURE__ */ new Set();
|
1280
|
-
for (const t of this.transactions) {
|
1281
|
-
for await (const blk of t.entries()) {
|
1282
|
-
if (seen.has(blk.cid.toString())) continue;
|
1283
|
-
seen.add(blk.cid.toString());
|
1284
|
-
yield blk;
|
1285
|
-
}
|
1286
|
-
}
|
1287
|
-
}
|
1288
|
-
};
|
1289
|
-
var EncryptedBlockstore = class extends BaseBlockstore {
|
1290
|
-
constructor(sthis, ebOpts) {
|
1291
|
-
super(ebOpts);
|
1292
|
-
this.compacting = false;
|
1293
|
-
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
1294
|
-
const { name } = ebOpts;
|
1295
|
-
if (!name) {
|
1296
|
-
throw this.logger.Error().Msg("name required").AsError();
|
1297
|
-
}
|
1298
|
-
this.name = name;
|
1299
|
-
this.loader = new Loader(this.name, ebOpts, sthis);
|
1300
|
-
}
|
1301
|
-
ready() {
|
1302
|
-
return this.loader.ready();
|
1303
|
-
}
|
1304
|
-
close() {
|
1305
|
-
return this.loader.close();
|
1306
|
-
}
|
1307
|
-
destroy() {
|
1308
|
-
return this.loader.destroy();
|
1309
|
-
}
|
1310
|
-
async get(cid) {
|
1311
|
-
const got = await super.get(cid);
|
1312
|
-
if (got) return got;
|
1313
|
-
if (!this.loader) {
|
1314
|
-
return;
|
1315
|
-
}
|
1316
|
-
return falsyToUndef(await this.loader.getBlock(cid));
|
1317
|
-
}
|
1318
|
-
async transaction(fn, opts = { noLoader: false }) {
|
1319
|
-
const { t, meta: done } = await super.transaction(fn);
|
1320
|
-
const cars = await this.loader.commit(t, done, opts);
|
1321
|
-
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
1322
|
-
setTimeout(() => void this.compact(), 10);
|
1323
|
-
}
|
1324
|
-
if (cars) {
|
1325
|
-
this.transactions.delete(t);
|
1326
|
-
return { meta: done, cars, t };
|
1327
|
-
}
|
1328
|
-
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1329
|
-
}
|
1330
|
-
async getFile(car, cid) {
|
1331
|
-
await this.ready();
|
1332
|
-
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
1333
|
-
const reader = await this.loader.loadFileCar(
|
1334
|
-
car
|
1335
|
-
/*, isPublic */
|
1336
|
-
);
|
1337
|
-
const block = await reader.get(cid);
|
1338
|
-
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
1339
|
-
return block.bytes;
|
1340
|
-
}
|
1341
|
-
async compact() {
|
1342
|
-
await this.ready();
|
1343
|
-
if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
|
1344
|
-
if (this.loader.carLog.length < 2) return;
|
1345
|
-
const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
|
1346
|
-
if (!compactFn || this.compacting) return;
|
1347
|
-
const blockLog = new CompactionFetcher(this);
|
1348
|
-
this.compacting = true;
|
1349
|
-
const meta = await compactFn(blockLog);
|
1350
|
-
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
1351
|
-
compact: true,
|
1352
|
-
noLoader: true
|
1353
|
-
});
|
1354
|
-
this.compacting = false;
|
1355
|
-
}
|
1356
|
-
async defaultCompact(blocks, logger) {
|
1357
|
-
if (!this.loader) {
|
1358
|
-
throw logger.Error().Msg("no loader").AsError();
|
1359
|
-
}
|
1360
|
-
if (!this.lastTxMeta) {
|
1361
|
-
throw logger.Error().Msg("no lastTxMeta").AsError();
|
1362
|
-
}
|
1363
|
-
for await (const blk of this.loader.entries(false)) {
|
1364
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1365
|
-
}
|
1366
|
-
for (const t of this.transactions) {
|
1367
|
-
for await (const blk of t.entries()) {
|
1368
|
-
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
1369
|
-
}
|
1370
|
-
}
|
1371
|
-
return this.lastTxMeta;
|
1372
|
-
}
|
1373
|
-
async *entries() {
|
1374
|
-
for await (const blk of this.loader.entries()) {
|
1375
|
-
yield blk;
|
1376
|
-
}
|
1377
|
-
}
|
1378
|
-
};
|
1379
|
-
var CompactionFetcher = class {
|
1380
|
-
constructor(blocks) {
|
1381
|
-
this.blockstore = blocks;
|
1382
|
-
this.loggedBlocks = new CarTransaction(blocks);
|
1383
|
-
}
|
1384
|
-
async get(cid) {
|
1385
|
-
const block = await this.blockstore.get(cid);
|
1386
|
-
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
1387
|
-
return falsyToUndef(block);
|
1388
|
-
}
|
1389
|
-
};
|
1390
|
-
|
1391
|
-
// src/blockstore/commit-queue.ts
|
1392
|
-
var import_cement3 = require("@adviser/cement");
|
1393
|
-
var CommitQueue = class {
|
1394
|
-
constructor() {
|
1395
|
-
this.queue = [];
|
1396
|
-
this.processing = false;
|
1397
|
-
this._waitIdleItems = /* @__PURE__ */ new Set();
|
1398
|
-
}
|
1399
|
-
waitIdle() {
|
1400
|
-
if (this.queue.length === 0 && !this.processing) {
|
1401
|
-
return Promise.resolve();
|
1402
|
-
}
|
1403
|
-
const fn = new import_cement3.Future();
|
1404
|
-
this._waitIdleItems.add(fn);
|
1405
|
-
return fn.asPromise();
|
1175
|
+
}
|
1176
|
+
const fn = new import_cement2.Future();
|
1177
|
+
this._waitIdleItems.add(fn);
|
1178
|
+
return fn.asPromise();
|
1406
1179
|
}
|
1407
1180
|
async enqueue(fn) {
|
1408
1181
|
return new Promise((resolve, reject) => {
|
@@ -1439,1132 +1212,1501 @@ var CommitQueue = class {
|
|
1439
1212
|
}
|
1440
1213
|
};
|
1441
1214
|
|
1442
|
-
// src/runtime/
|
1443
|
-
var
|
1444
|
-
__export(
|
1445
|
-
|
1446
|
-
|
1447
|
-
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
1215
|
+
// src/runtime/keyed-crypto.ts
|
1216
|
+
var keyed_crypto_exports = {};
|
1217
|
+
__export(keyed_crypto_exports, {
|
1218
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
1219
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
1448
1220
|
});
|
1449
|
-
var import_cement6 = require("@adviser/cement");
|
1450
1221
|
init_utils();
|
1451
1222
|
var import_base582 = require("multiformats/bases/base58");
|
1452
|
-
var
|
1453
|
-
|
1454
|
-
|
1455
|
-
|
1456
|
-
|
1457
|
-
|
1458
|
-
|
1459
|
-
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1463
|
-
|
1464
|
-
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
1465
|
-
);
|
1466
|
-
}
|
1467
|
-
return await this.rt.crypto.importKey(
|
1468
|
-
"raw",
|
1469
|
-
// raw or jwk
|
1470
|
-
import_base582.base58btc.decode(key),
|
1471
|
-
// hexStringToUint8Array(key), // raw data
|
1472
|
-
"AES-GCM",
|
1473
|
-
extractable,
|
1474
|
-
["encrypt", "decrypt"]
|
1475
|
-
);
|
1476
|
-
}
|
1477
|
-
async ensureKeyFromUrl(url, keyFactory) {
|
1478
|
-
const storeKey = url.getParam("storekey");
|
1479
|
-
if (storeKey === "insecure") {
|
1480
|
-
return import_cement6.Result.Ok(url);
|
1481
|
-
}
|
1482
|
-
if (!storeKey) {
|
1483
|
-
const keyName = `@${keyFactory()}@`;
|
1484
|
-
const ret = await this.getNamedKey(keyName);
|
1485
|
-
if (ret.isErr()) {
|
1486
|
-
return ret;
|
1487
|
-
}
|
1488
|
-
const urb = url.build().setParam("storekey", keyName);
|
1489
|
-
return import_cement6.Result.Ok(urb.URI());
|
1223
|
+
var import_sha2 = require("multiformats/hashes/sha2");
|
1224
|
+
var CBOR = __toESM(require("cborg"), 1);
|
1225
|
+
init_types();
|
1226
|
+
var generateIV = {
|
1227
|
+
random: {
|
1228
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1229
|
+
calc: async (ko, crypto, data) => {
|
1230
|
+
return crypto.randomBytes(ko.ivLength);
|
1231
|
+
},
|
1232
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1233
|
+
verify: async (ko, crypto, iv, data) => {
|
1234
|
+
return true;
|
1490
1235
|
}
|
1491
|
-
|
1492
|
-
|
1493
|
-
|
1494
|
-
|
1236
|
+
},
|
1237
|
+
hash: {
|
1238
|
+
calc: async (ko, crypto, data) => {
|
1239
|
+
const hash = await import_sha2.sha256.digest(data);
|
1240
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
1241
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
1242
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
1243
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
1495
1244
|
}
|
1245
|
+
return hashArray;
|
1246
|
+
},
|
1247
|
+
verify: async function(ko, crypto, iv, data) {
|
1248
|
+
return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
1496
1249
|
}
|
1497
|
-
return import_cement6.Result.Ok(url);
|
1498
1250
|
}
|
1499
|
-
|
1500
|
-
|
1501
|
-
|
1502
|
-
|
1503
|
-
|
1504
|
-
|
1505
|
-
|
1251
|
+
};
|
1252
|
+
function getGenerateIVFn(url, opts) {
|
1253
|
+
const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
|
1254
|
+
return generateIV[ivhash] || generateIV["hash"];
|
1255
|
+
}
|
1256
|
+
var BlockIvKeyIdCodec = class {
|
1257
|
+
constructor(ko, iv, opts) {
|
1258
|
+
this.code = 3147065;
|
1259
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
1260
|
+
this.ko = ko;
|
1261
|
+
this.iv = iv;
|
1262
|
+
this.opts = opts || {};
|
1263
|
+
}
|
1264
|
+
async encode(data) {
|
1265
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
1266
|
+
const { iv } = this.ko.algo(calcIv);
|
1267
|
+
const fprt = await this.ko.fingerPrint();
|
1268
|
+
const keyId = import_base582.base58btc.decode(fprt);
|
1269
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
1270
|
+
return CBOR.encode({
|
1271
|
+
iv,
|
1272
|
+
keyId,
|
1273
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
1506
1274
|
});
|
1507
1275
|
}
|
1508
|
-
async
|
1509
|
-
|
1276
|
+
async decode(abytes) {
|
1277
|
+
let bytes;
|
1278
|
+
if (abytes instanceof Uint8Array) {
|
1279
|
+
bytes = abytes;
|
1280
|
+
} else {
|
1281
|
+
bytes = new Uint8Array(abytes);
|
1282
|
+
}
|
1283
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
1284
|
+
const fprt = await this.ko.fingerPrint();
|
1285
|
+
this.ko.logger.Debug().Str("fp", import_base582.base58btc.encode(keyId)).Msg("decode");
|
1286
|
+
if (import_base582.base58btc.encode(keyId) !== fprt) {
|
1287
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base582.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
|
1288
|
+
}
|
1289
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
1290
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
1291
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
1292
|
+
}
|
1293
|
+
return result;
|
1510
1294
|
}
|
1511
|
-
|
1512
|
-
|
1513
|
-
|
1514
|
-
|
1515
|
-
|
1295
|
+
};
|
1296
|
+
var keyedCrypto = class {
|
1297
|
+
constructor(url, key, cyopt, sthis) {
|
1298
|
+
this.ivLength = 12;
|
1299
|
+
this.isEncrypting = true;
|
1300
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
1301
|
+
this.crypto = cyopt;
|
1302
|
+
this.key = key;
|
1303
|
+
this.url = url;
|
1304
|
+
}
|
1305
|
+
fingerPrint() {
|
1306
|
+
return Promise.resolve(this.key.fingerPrint);
|
1307
|
+
}
|
1308
|
+
codec(iv, opts) {
|
1309
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
1310
|
+
}
|
1311
|
+
algo(iv) {
|
1312
|
+
return {
|
1313
|
+
name: "AES-GCM",
|
1314
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
1315
|
+
tagLength: 128
|
1516
1316
|
};
|
1517
|
-
const bag = await this.rt.getBag();
|
1518
|
-
this.logger.Debug().Str("name", name).Msg("setNamedKey");
|
1519
|
-
await bag.set(name, item);
|
1520
|
-
return await this.toKeyWithFingerPrint(item.key);
|
1521
1317
|
}
|
1522
|
-
async
|
1523
|
-
|
1524
|
-
|
1525
|
-
return ret;
|
1526
|
-
}
|
1527
|
-
const named = ret.Ok();
|
1528
|
-
return import_cement6.Result.Ok({
|
1529
|
-
...named,
|
1530
|
-
extract: async () => {
|
1531
|
-
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
1532
|
-
return {
|
1533
|
-
key: ext,
|
1534
|
-
keyStr: import_base582.base58btc.encode(ext)
|
1535
|
-
};
|
1536
|
-
}
|
1537
|
-
});
|
1318
|
+
async _decrypt(data) {
|
1319
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1320
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
1538
1321
|
}
|
1539
|
-
async
|
1540
|
-
|
1541
|
-
|
1542
|
-
|
1543
|
-
const named = await bag.get(name);
|
1544
|
-
if (named) {
|
1545
|
-
const fpr = await this.toKeyWithFingerPrint(named.key);
|
1546
|
-
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
|
1547
|
-
return fpr;
|
1548
|
-
}
|
1549
|
-
if (failIfNotFound) {
|
1550
|
-
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
1551
|
-
return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
|
1552
|
-
}
|
1553
|
-
const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
1554
|
-
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
1555
|
-
return ret;
|
1556
|
-
});
|
1322
|
+
async _encrypt(data) {
|
1323
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
1324
|
+
const a = this.algo(data.iv);
|
1325
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
1557
1326
|
}
|
1558
1327
|
};
|
1559
|
-
var
|
1560
|
-
|
1561
|
-
|
1562
|
-
|
1563
|
-
factory: async (url, sthis) => {
|
1564
|
-
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1565
|
-
return new KeyBagProviderFile2(url, sthis);
|
1566
|
-
}
|
1567
|
-
},
|
1568
|
-
{
|
1569
|
-
protocol: "indexdb:",
|
1570
|
-
factory: async (url, sthis) => {
|
1571
|
-
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1572
|
-
return new KeyBagProviderIndexDB2(url, sthis);
|
1573
|
-
}
|
1574
|
-
}
|
1575
|
-
].map((i) => [i.protocol, i])
|
1576
|
-
);
|
1577
|
-
function registerKeyBagProviderFactory(item) {
|
1578
|
-
const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
|
1579
|
-
keyBagProviderFactories.set(protocol, {
|
1580
|
-
...item,
|
1581
|
-
protocol
|
1582
|
-
});
|
1583
|
-
}
|
1584
|
-
function defaultKeyBagOpts(sthis, kbo) {
|
1585
|
-
if (kbo.keyRuntime) {
|
1586
|
-
return kbo.keyRuntime;
|
1328
|
+
var nullCodec = class {
|
1329
|
+
constructor() {
|
1330
|
+
this.code = 0;
|
1331
|
+
this.name = "Fireproof@unencrypted-block";
|
1587
1332
|
}
|
1588
|
-
|
1589
|
-
|
1590
|
-
if (kbo.url) {
|
1591
|
-
url = import_cement6.URI.from(kbo.url);
|
1592
|
-
logger.Debug().Url(url).Msg("from opts");
|
1593
|
-
} else {
|
1594
|
-
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
1595
|
-
if ((0, import_cement6.runtimeFn)().isBrowser) {
|
1596
|
-
url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
1597
|
-
} else {
|
1598
|
-
if (!bagFnameOrUrl) {
|
1599
|
-
const home = sthis.env.get("HOME");
|
1600
|
-
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
1601
|
-
url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
|
1602
|
-
} else {
|
1603
|
-
url = import_cement6.URI.from(bagFnameOrUrl);
|
1604
|
-
}
|
1605
|
-
}
|
1606
|
-
logger.Debug().Url(url).Msg("from env");
|
1333
|
+
encode(data) {
|
1334
|
+
return data;
|
1607
1335
|
}
|
1608
|
-
|
1609
|
-
|
1610
|
-
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
1336
|
+
decode(data) {
|
1337
|
+
return data;
|
1611
1338
|
}
|
1612
|
-
|
1613
|
-
|
1614
|
-
|
1339
|
+
};
|
1340
|
+
var noCrypto = class {
|
1341
|
+
constructor(url, cyrt, sthis) {
|
1342
|
+
this.ivLength = 0;
|
1343
|
+
this.code = 0;
|
1344
|
+
this.name = "Fireproof@unencrypted-block";
|
1345
|
+
this.isEncrypting = false;
|
1346
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
1347
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
1348
|
+
this.crypto = cyrt;
|
1349
|
+
this.url = url;
|
1615
1350
|
}
|
1616
|
-
|
1617
|
-
|
1618
|
-
crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
|
1619
|
-
sthis,
|
1620
|
-
logger,
|
1621
|
-
keyLength: kbo.keyLength || 16,
|
1622
|
-
getBag,
|
1623
|
-
id: () => {
|
1624
|
-
return url.toString();
|
1625
|
-
}
|
1626
|
-
};
|
1627
|
-
}
|
1628
|
-
var _keyBags = new import_cement6.KeyedResolvOnce();
|
1629
|
-
async function getKeyBag(sthis, kbo = {}) {
|
1630
|
-
await sthis.start();
|
1631
|
-
const rt = defaultKeyBagOpts(sthis, kbo);
|
1632
|
-
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
1633
|
-
}
|
1634
|
-
|
1635
|
-
// src/blockstore/commitor.ts
|
1636
|
-
var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
|
1637
|
-
var import_sha22 = require("multiformats/hashes/sha2");
|
1638
|
-
var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
1639
|
-
async function encodeCarFile(roots, t, codec3) {
|
1640
|
-
let size = 0;
|
1641
|
-
const headerSize = CBW.headerLength({ roots });
|
1642
|
-
size += headerSize;
|
1643
|
-
for (const { cid, bytes } of t.entries()) {
|
1644
|
-
size += CBW.blockLength({ cid, bytes });
|
1351
|
+
fingerPrint() {
|
1352
|
+
return Promise.resolve(this._fingerPrint);
|
1645
1353
|
}
|
1646
|
-
|
1647
|
-
|
1648
|
-
|
1649
|
-
writer.addRoot(r);
|
1354
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1355
|
+
codec(iv) {
|
1356
|
+
return new nullCodec();
|
1650
1357
|
}
|
1651
|
-
|
1652
|
-
|
1358
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1359
|
+
algo(iv) {
|
1360
|
+
return {
|
1361
|
+
name: "noCrypto",
|
1362
|
+
iv: new Uint8Array(),
|
1363
|
+
tagLength: 0
|
1364
|
+
};
|
1653
1365
|
}
|
1654
|
-
|
1655
|
-
|
1656
|
-
}
|
1657
|
-
async function createCarFile(encoder, cid, t) {
|
1658
|
-
return encodeCarFile([cid], t, encoder);
|
1659
|
-
}
|
1660
|
-
async function commitFiles(fileStore, walStore, t, done) {
|
1661
|
-
const { files: roots } = makeFileCarHeader(done);
|
1662
|
-
const cids = [];
|
1663
|
-
const codec3 = (await fileStore.keyedCrypto()).codec();
|
1664
|
-
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
1665
|
-
for (const car of cars) {
|
1666
|
-
const { cid, bytes } = car;
|
1667
|
-
await fileStore.save({ cid, bytes });
|
1668
|
-
await walStore.enqueueFile(
|
1669
|
-
cid
|
1670
|
-
/*, !!opts.public*/
|
1671
|
-
);
|
1672
|
-
cids.push(cid);
|
1366
|
+
_decrypt() {
|
1367
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
1673
1368
|
}
|
1674
|
-
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
|
1679
|
-
|
1680
|
-
|
1369
|
+
_encrypt() {
|
1370
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
1371
|
+
}
|
1372
|
+
};
|
1373
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
1374
|
+
const storekey = url.getParam("storekey" /* STORE_KEY */);
|
1375
|
+
if (storekey && storekey !== "insecure") {
|
1376
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
1377
|
+
if (rkey.isErr()) {
|
1378
|
+
try {
|
1379
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
1380
|
+
} catch (e) {
|
1381
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
1382
|
+
}
|
1681
1383
|
}
|
1384
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
1682
1385
|
}
|
1683
|
-
return
|
1684
|
-
}
|
1685
|
-
async function prepareCarFilesFiles(encoder, roots, t) {
|
1686
|
-
return [await encodeCarFile(roots, t, encoder)];
|
1687
|
-
}
|
1688
|
-
function makeCarHeader(meta, cars, compact = false) {
|
1689
|
-
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
1690
|
-
return { ...coreHeader, meta };
|
1691
|
-
}
|
1692
|
-
async function encodeCarHeader(fp) {
|
1693
|
-
return await encode({
|
1694
|
-
value: { fp },
|
1695
|
-
hasher: import_sha22.sha256,
|
1696
|
-
codec: dagCodec2
|
1697
|
-
});
|
1386
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
1698
1387
|
}
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
|
1703
|
-
|
1704
|
-
|
1705
|
-
|
1706
|
-
|
1707
|
-
|
1388
|
+
|
1389
|
+
// src/blockstore/fragment-gateway.ts
|
1390
|
+
var import_cement3 = require("@adviser/cement");
|
1391
|
+
var import_base583 = require("multiformats/bases/base58");
|
1392
|
+
var import_cborg = require("cborg");
|
1393
|
+
init_types();
|
1394
|
+
init_utils();
|
1395
|
+
function getFragSize(url) {
|
1396
|
+
const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
|
1397
|
+
let ret = 0;
|
1398
|
+
if (fragSize) {
|
1399
|
+
ret = parseInt(fragSize);
|
1708
1400
|
}
|
1709
|
-
|
1710
|
-
|
1711
|
-
|
1712
|
-
return
|
1401
|
+
if (isNaN(ret) || ret <= 0) {
|
1402
|
+
ret = 0;
|
1403
|
+
}
|
1404
|
+
return ret;
|
1713
1405
|
}
|
1714
|
-
async function
|
1715
|
-
const
|
1716
|
-
|
1717
|
-
|
1718
|
-
|
1719
|
-
|
1720
|
-
let cidRootBlock = rootBlock;
|
1721
|
-
for (const { cid, bytes } of t.entries()) {
|
1722
|
-
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1723
|
-
if (newsize >= threshold) {
|
1724
|
-
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1725
|
-
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1726
|
-
clonedt.putSync(cid, bytes);
|
1727
|
-
cidRootBlock = { cid, bytes };
|
1728
|
-
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1729
|
-
} else {
|
1730
|
-
clonedt.putSync(cid, bytes);
|
1406
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
1407
|
+
const fragSize = getFragSize(url);
|
1408
|
+
if (!fragSize) {
|
1409
|
+
const res = await innerGW.get(url);
|
1410
|
+
if (res.isErr()) {
|
1411
|
+
return [res];
|
1731
1412
|
}
|
1413
|
+
const data = res.unwrap();
|
1414
|
+
return [
|
1415
|
+
import_cement3.Result.Ok({
|
1416
|
+
fid: new Uint8Array(0),
|
1417
|
+
ofs: 0,
|
1418
|
+
len: data.length,
|
1419
|
+
data
|
1420
|
+
})
|
1421
|
+
];
|
1732
1422
|
}
|
1733
|
-
|
1734
|
-
|
1423
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
1424
|
+
if (firstRaw.isErr()) {
|
1425
|
+
return [firstRaw];
|
1426
|
+
}
|
1427
|
+
const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
|
1428
|
+
const blockSize = firstFragment.data.length;
|
1429
|
+
const ops = [Promise.resolve(import_cement3.Result.Ok(firstFragment))];
|
1430
|
+
const fidStr = import_base583.base58btc.encode(firstFragment.fid);
|
1431
|
+
const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
|
1432
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
1433
|
+
ops.push(
|
1434
|
+
(async (furl, ofs2) => {
|
1435
|
+
const raw2 = await innerGW.get(furl);
|
1436
|
+
if (raw2.isErr()) {
|
1437
|
+
return raw2;
|
1438
|
+
}
|
1439
|
+
const fragment = (0, import_cborg.decode)(raw2.unwrap());
|
1440
|
+
if (import_base583.base58btc.encode(fragment.fid) !== fidStr) {
|
1441
|
+
return import_cement3.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1442
|
+
}
|
1443
|
+
if (fragment.ofs !== ofs2) {
|
1444
|
+
return import_cement3.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1445
|
+
}
|
1446
|
+
return import_cement3.Result.Ok(fragment);
|
1447
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1448
|
+
);
|
1449
|
+
}
|
1450
|
+
return Promise.all(ops);
|
1735
1451
|
}
|
1736
|
-
|
1737
|
-
|
1738
|
-
|
1739
|
-
|
1740
|
-
|
1741
|
-
|
1742
|
-
|
1743
|
-
constructor(sthis, callback) {
|
1744
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
1745
|
-
this.queue = [];
|
1746
|
-
this.isProcessing = false;
|
1747
|
-
this.logger = ensureLogger(sthis, "TaskManager");
|
1748
|
-
this.callback = callback;
|
1452
|
+
var FragmentGateway = class {
|
1453
|
+
constructor(sthis, innerGW) {
|
1454
|
+
this.fidLength = 4;
|
1455
|
+
this.headerSize = 32;
|
1456
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
1457
|
+
this.logger = this.sthis.logger;
|
1458
|
+
this.innerGW = innerGW;
|
1749
1459
|
}
|
1750
|
-
|
1751
|
-
|
1752
|
-
|
1460
|
+
slicer(url, body) {
|
1461
|
+
const fragSize = getFragSize(url);
|
1462
|
+
if (!fragSize) {
|
1463
|
+
return [this.innerGW.put(url, body)];
|
1753
1464
|
}
|
1754
|
-
|
1755
|
-
|
1756
|
-
|
1757
|
-
}
|
1758
|
-
async processQueue() {
|
1759
|
-
if (this.isProcessing) return;
|
1760
|
-
this.isProcessing = true;
|
1761
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1762
|
-
const first = filteredQueue[0];
|
1763
|
-
if (!first) {
|
1764
|
-
return;
|
1465
|
+
const blocksize = fragSize - this.headerSize;
|
1466
|
+
if (blocksize <= 0) {
|
1467
|
+
throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
|
1765
1468
|
}
|
1766
|
-
|
1767
|
-
|
1768
|
-
|
1769
|
-
|
1770
|
-
|
1771
|
-
|
1772
|
-
|
1773
|
-
|
1774
|
-
|
1775
|
-
|
1776
|
-
|
1777
|
-
|
1778
|
-
this.isProcessing = false;
|
1779
|
-
if (this.queue.length > 0) {
|
1780
|
-
void this.processQueue();
|
1469
|
+
const ops = [];
|
1470
|
+
const fid = this.sthis.nextId(this.fidLength);
|
1471
|
+
const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
|
1472
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
1473
|
+
const block = (0, import_cborg.encode)({
|
1474
|
+
fid: fid.bin,
|
1475
|
+
ofs,
|
1476
|
+
len: body.length,
|
1477
|
+
data: body.slice(ofs, ofs + blocksize)
|
1478
|
+
});
|
1479
|
+
if (block.length > fragSize) {
|
1480
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
1781
1481
|
}
|
1482
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
1782
1483
|
}
|
1484
|
+
return ops;
|
1783
1485
|
}
|
1784
|
-
|
1785
|
-
|
1786
|
-
// src/blockstore/loader.ts
|
1787
|
-
function carLogIncludesGroup(list, cids) {
|
1788
|
-
return list.some((arr) => {
|
1789
|
-
return arr.toString() === cids.toString();
|
1790
|
-
});
|
1791
|
-
}
|
1792
|
-
function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
1793
|
-
const byString = /* @__PURE__ */ new Map();
|
1794
|
-
for (const cid of list) {
|
1795
|
-
if (remove.has(cid.toString())) continue;
|
1796
|
-
byString.set(cid.toString(), cid);
|
1797
|
-
}
|
1798
|
-
return [...byString.values()];
|
1799
|
-
}
|
1800
|
-
var Loader = class {
|
1801
|
-
constructor(name, ebOpts, sthis) {
|
1802
|
-
this.commitQueue = new CommitQueue();
|
1803
|
-
this.isCompacting = false;
|
1804
|
-
this.carReaders = /* @__PURE__ */ new Map();
|
1805
|
-
this.seenCompacted = /* @__PURE__ */ new Set();
|
1806
|
-
this.processedCars = /* @__PURE__ */ new Set();
|
1807
|
-
this.carLog = [];
|
1808
|
-
this.getBlockCache = /* @__PURE__ */ new Map();
|
1809
|
-
this.seenMeta = /* @__PURE__ */ new Set();
|
1810
|
-
this.writeLimit = (0, import_p_limit.default)(1);
|
1811
|
-
this.onceReady = new import_cement7.ResolveOnce();
|
1812
|
-
this.name = name;
|
1813
|
-
this.sthis = sthis;
|
1814
|
-
this.ebOpts = defaultedBlockstoreRuntime(
|
1815
|
-
sthis,
|
1816
|
-
{
|
1817
|
-
...ebOpts,
|
1818
|
-
name
|
1819
|
-
},
|
1820
|
-
"Loader"
|
1821
|
-
);
|
1822
|
-
this.logger = this.ebOpts.logger;
|
1823
|
-
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
1824
|
-
await this.handleDbMetasFromStore([dbMeta]);
|
1825
|
-
});
|
1826
|
-
}
|
1827
|
-
// readonly id = uuidv4();
|
1828
|
-
async keyBag() {
|
1829
|
-
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
1486
|
+
buildUrl(baseUrl, key) {
|
1487
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
1830
1488
|
}
|
1831
|
-
async
|
1832
|
-
return this.
|
1489
|
+
async destroy(iurl) {
|
1490
|
+
return this.innerGW.destroy(iurl);
|
1833
1491
|
}
|
1834
|
-
async
|
1835
|
-
|
1492
|
+
async start(url) {
|
1493
|
+
this.headerSize = (0, import_cborg.encode)({
|
1494
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
1495
|
+
ofs: 1024 * 1024,
|
1496
|
+
// 32bit
|
1497
|
+
len: 16 * 1024 * 1024,
|
1498
|
+
// 32bit
|
1499
|
+
data: new Uint8Array(1024)
|
1500
|
+
}).length - 1024;
|
1501
|
+
return this.innerGW.start(url);
|
1836
1502
|
}
|
1837
|
-
async
|
1838
|
-
return this.
|
1503
|
+
async close(url) {
|
1504
|
+
return this.innerGW.close(url);
|
1839
1505
|
}
|
1840
|
-
async
|
1841
|
-
|
1506
|
+
async put(url, body) {
|
1507
|
+
await Promise.all(this.slicer(url, body));
|
1508
|
+
return import_cement3.Result.Ok(void 0);
|
1842
1509
|
}
|
1843
|
-
async
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1848
|
-
|
1849
|
-
await this.handleDbMetasFromStore(metas);
|
1510
|
+
async get(url) {
|
1511
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1512
|
+
let buffer = void 0;
|
1513
|
+
for (const rfrag of rfrags) {
|
1514
|
+
if (rfrag.isErr()) {
|
1515
|
+
return import_cement3.Result.Err(rfrag.Err());
|
1850
1516
|
}
|
1851
|
-
|
1852
|
-
|
1853
|
-
|
1854
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1855
|
-
await Promise.all(toClose.map((store) => store.close()));
|
1856
|
-
}
|
1857
|
-
async destroy() {
|
1858
|
-
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1859
|
-
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1860
|
-
}
|
1861
|
-
// async snapToCar(carCid: AnyLink | string) {
|
1862
|
-
// await this.ready
|
1863
|
-
// if (typeof carCid === 'string') {
|
1864
|
-
// carCid = CID.parse(carCid)
|
1865
|
-
// }
|
1866
|
-
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
1867
|
-
// this.carLog = [carCid, ...carHeader.cars]
|
1868
|
-
// await this.getMoreReaders(carHeader.cars)
|
1869
|
-
// await this._applyCarHeader(carHeader, true)
|
1870
|
-
// }
|
1871
|
-
async handleDbMetasFromStore(metas) {
|
1872
|
-
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
1873
|
-
for (const meta of metas) {
|
1874
|
-
await this.writeLimit(async () => {
|
1875
|
-
await this.mergeDbMetaIntoClock(meta);
|
1876
|
-
});
|
1517
|
+
const frag = rfrag.Ok();
|
1518
|
+
buffer = buffer || new Uint8Array(frag.len);
|
1519
|
+
buffer.set(frag.data, frag.ofs);
|
1877
1520
|
}
|
1521
|
+
return import_cement3.Result.Ok(buffer || new Uint8Array(0));
|
1878
1522
|
}
|
1879
|
-
async
|
1880
|
-
if (this.
|
1881
|
-
|
1523
|
+
async subscribe(url, callback) {
|
1524
|
+
if (this.innerGW.subscribe) {
|
1525
|
+
return this.innerGW.subscribe(url, callback);
|
1526
|
+
} else {
|
1527
|
+
return import_cement3.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
|
1882
1528
|
}
|
1883
|
-
if (this.seenMeta.has(meta.cars.toString())) return;
|
1884
|
-
this.seenMeta.add(meta.cars.toString());
|
1885
|
-
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1886
|
-
return;
|
1887
|
-
}
|
1888
|
-
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
1889
|
-
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1890
|
-
await this.getMoreReaders(carHeader.cars.flat());
|
1891
|
-
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1892
|
-
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1893
|
-
}
|
1894
|
-
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1895
|
-
// const { key } = meta;
|
1896
|
-
// if (key) {
|
1897
|
-
// await this.setKey(key);
|
1898
|
-
// }
|
1899
|
-
// }
|
1900
|
-
async loadCarHeaderFromMeta({ cars: cids }) {
|
1901
|
-
const reader = await this.loadCar(cids[0]);
|
1902
|
-
return await parseCarFile(reader, this.logger);
|
1903
|
-
}
|
1904
|
-
// async _getKey(): Promise<string | undefined> {
|
1905
|
-
// if (this.key) return this.key;
|
1906
|
-
// // generate a random key
|
1907
|
-
// if (!this.ebOpts.public) {
|
1908
|
-
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1909
|
-
// }
|
1910
|
-
// return this.key || undefined;
|
1911
|
-
// }
|
1912
|
-
async commitFiles(t, done) {
|
1913
|
-
await this.ready();
|
1914
|
-
const fstore = await this.fileStore();
|
1915
|
-
const wstore = await this.WALStore();
|
1916
|
-
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1917
1529
|
}
|
1918
|
-
async
|
1919
|
-
|
1530
|
+
async delete(url) {
|
1531
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1532
|
+
for (const rfrag of rfrags) {
|
1533
|
+
if (rfrag.isErr()) {
|
1534
|
+
return import_cement3.Result.Err(rfrag.Err());
|
1535
|
+
}
|
1536
|
+
const frag = rfrag.Ok();
|
1537
|
+
let fragUrl;
|
1538
|
+
if (rfrags.length > 1) {
|
1539
|
+
const fidStr = import_base583.base58btc.encode(frag.fid);
|
1540
|
+
fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
|
1541
|
+
} else {
|
1542
|
+
fragUrl = url;
|
1543
|
+
}
|
1544
|
+
await this.innerGW.delete(fragUrl);
|
1545
|
+
}
|
1546
|
+
return import_cement3.Result.Ok(void 0);
|
1920
1547
|
}
|
1921
|
-
|
1922
|
-
|
1923
|
-
|
1924
|
-
|
1925
|
-
|
1926
|
-
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
|
1932
|
-
|
1933
|
-
|
1934
|
-
|
1935
|
-
|
1936
|
-
|
1548
|
+
};
|
1549
|
+
|
1550
|
+
// src/blockstore/meta-key-helper.ts
|
1551
|
+
var import_dag_json = require("@ipld/dag-json");
|
1552
|
+
var import_clock = require("@web3-storage/pail/clock");
|
1553
|
+
var import_multiformats = require("multiformats");
|
1554
|
+
var import_base64 = require("multiformats/bases/base64");
|
1555
|
+
var import_cement7 = require("@adviser/cement");
|
1556
|
+
|
1557
|
+
// src/runtime/key-bag.ts
|
1558
|
+
var key_bag_exports = {};
|
1559
|
+
__export(key_bag_exports, {
|
1560
|
+
KeyBag: () => KeyBag,
|
1561
|
+
defaultKeyBagOpts: () => defaultKeyBagOpts,
|
1562
|
+
getKeyBag: () => getKeyBag,
|
1563
|
+
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
1564
|
+
});
|
1565
|
+
var import_cement6 = require("@adviser/cement");
|
1566
|
+
init_utils();
|
1567
|
+
var import_base584 = require("multiformats/bases/base58");
|
1568
|
+
init_types();
|
1569
|
+
var KeyBag = class {
|
1570
|
+
constructor(rt) {
|
1571
|
+
this.rt = rt;
|
1572
|
+
this._warnOnce = new import_cement6.ResolveOnce();
|
1573
|
+
this._seq = new import_cement6.ResolveSeq();
|
1574
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag", {
|
1575
|
+
id: rt.id()
|
1937
1576
|
});
|
1577
|
+
this.logger.Debug().Msg("KeyBag created");
|
1938
1578
|
}
|
1939
|
-
async
|
1940
|
-
|
1941
|
-
|
1942
|
-
|
1943
|
-
|
1944
|
-
|
1945
|
-
} else {
|
1946
|
-
this.carLog.unshift(cids);
|
1579
|
+
async subtleKey(key) {
|
1580
|
+
const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
|
1581
|
+
if (extractable) {
|
1582
|
+
this._warnOnce.once(
|
1583
|
+
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
1584
|
+
);
|
1947
1585
|
}
|
1586
|
+
return await this.rt.crypto.importKey(
|
1587
|
+
"raw",
|
1588
|
+
// raw or jwk
|
1589
|
+
import_base584.base58btc.decode(key),
|
1590
|
+
// hexStringToUint8Array(key), // raw data
|
1591
|
+
"AES-GCM",
|
1592
|
+
extractable,
|
1593
|
+
["encrypt", "decrypt"]
|
1594
|
+
);
|
1948
1595
|
}
|
1949
|
-
async
|
1950
|
-
|
1951
|
-
|
1952
|
-
|
1953
|
-
|
1596
|
+
async ensureKeyFromUrl(url, keyFactory) {
|
1597
|
+
const storeKey = url.getParam("storekey" /* STORE_KEY */);
|
1598
|
+
if (storeKey === "insecure") {
|
1599
|
+
return import_cement6.Result.Ok(url);
|
1600
|
+
}
|
1601
|
+
if (!storeKey) {
|
1602
|
+
const keyName = `@${keyFactory()}@`;
|
1603
|
+
const ret = await this.getNamedKey(keyName);
|
1604
|
+
if (ret.isErr()) {
|
1605
|
+
return ret;
|
1954
1606
|
}
|
1607
|
+
const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
|
1608
|
+
return import_cement6.Result.Ok(urb.URI());
|
1955
1609
|
}
|
1956
|
-
|
1957
|
-
|
1958
|
-
|
1959
|
-
|
1960
|
-
for await (const block of reader.blocks()) {
|
1961
|
-
const sBlock = block.cid.toString();
|
1962
|
-
if (!this.getBlockCache.has(sBlock)) {
|
1963
|
-
this.getBlockCache.set(sBlock, block);
|
1610
|
+
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
1611
|
+
const ret = await this.getNamedKey(storeKey);
|
1612
|
+
if (ret.isErr()) {
|
1613
|
+
return ret;
|
1964
1614
|
}
|
1965
1615
|
}
|
1616
|
+
return import_cement6.Result.Ok(url);
|
1966
1617
|
}
|
1967
|
-
async
|
1968
|
-
const
|
1969
|
-
|
1618
|
+
async toKeyWithFingerPrint(keyStr) {
|
1619
|
+
const material = import_base584.base58btc.decode(keyStr);
|
1620
|
+
const key = await this.subtleKey(keyStr);
|
1621
|
+
const fpr = await this.rt.crypto.digestSHA256(material);
|
1622
|
+
return import_cement6.Result.Ok({
|
1623
|
+
key,
|
1624
|
+
fingerPrint: import_base584.base58btc.encode(new Uint8Array(fpr))
|
1970
1625
|
});
|
1971
|
-
for (const cids of carHeader.compact) {
|
1972
|
-
for (const cid2 of cids) {
|
1973
|
-
await (await this.carStore()).remove(cid2);
|
1974
|
-
}
|
1975
|
-
}
|
1976
1626
|
}
|
1977
|
-
|
1978
|
-
|
1979
|
-
|
1980
|
-
//
|
1981
|
-
|
1982
|
-
|
1983
|
-
|
1984
|
-
|
1985
|
-
|
1986
|
-
await this.
|
1987
|
-
|
1988
|
-
|
1989
|
-
|
1627
|
+
async setNamedKey(name, key) {
|
1628
|
+
return this._seq.add(() => this._setNamedKey(name, key));
|
1629
|
+
}
|
1630
|
+
// avoid deadlock
|
1631
|
+
async _setNamedKey(name, key) {
|
1632
|
+
const item = {
|
1633
|
+
name,
|
1634
|
+
key
|
1635
|
+
};
|
1636
|
+
const bag = await this.rt.getBag();
|
1637
|
+
this.logger.Debug().Str("name", name).Msg("setNamedKey");
|
1638
|
+
await bag.set(name, item);
|
1639
|
+
return await this.toKeyWithFingerPrint(item.key);
|
1640
|
+
}
|
1641
|
+
async getNamedExtractableKey(name, failIfNotFound = false) {
|
1642
|
+
const ret = await this.getNamedKey(name, failIfNotFound);
|
1643
|
+
if (ret.isErr()) {
|
1644
|
+
return ret;
|
1645
|
+
}
|
1646
|
+
const named = ret.Ok();
|
1647
|
+
return import_cement6.Result.Ok({
|
1648
|
+
...named,
|
1649
|
+
extract: async () => {
|
1650
|
+
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
1651
|
+
return {
|
1652
|
+
key: ext,
|
1653
|
+
keyStr: import_base584.base58btc.encode(ext)
|
1654
|
+
};
|
1990
1655
|
}
|
1991
|
-
}
|
1992
|
-
|
1993
|
-
|
1656
|
+
});
|
1657
|
+
}
|
1658
|
+
async getNamedKey(name, failIfNotFound = false) {
|
1659
|
+
const id = this.rt.sthis.nextId(4).str;
|
1660
|
+
return this._seq.add(async () => {
|
1661
|
+
const bag = await this.rt.getBag();
|
1662
|
+
const named = await bag.get(name);
|
1663
|
+
if (named) {
|
1664
|
+
const fpr = await this.toKeyWithFingerPrint(named.key);
|
1665
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
|
1666
|
+
return fpr;
|
1994
1667
|
}
|
1995
|
-
|
1996
|
-
|
1997
|
-
|
1998
|
-
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
1999
|
-
for await (const block of reader.blocks()) {
|
2000
|
-
const sCid = block.cid.toString();
|
2001
|
-
if (!this.getBlockCache.has(sCid)) {
|
2002
|
-
yield block;
|
2003
|
-
}
|
2004
|
-
}
|
2005
|
-
}
|
1668
|
+
if (failIfNotFound) {
|
1669
|
+
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
1670
|
+
return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
|
2006
1671
|
}
|
2007
|
-
|
1672
|
+
const ret = await this._setNamedKey(name, import_base584.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
1673
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
1674
|
+
return ret;
|
1675
|
+
});
|
2008
1676
|
}
|
2009
|
-
|
2010
|
-
|
2011
|
-
|
2012
|
-
|
2013
|
-
|
2014
|
-
|
2015
|
-
|
2016
|
-
|
2017
|
-
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
1677
|
+
};
|
1678
|
+
var keyBagProviderFactories = new Map(
|
1679
|
+
[
|
1680
|
+
{
|
1681
|
+
protocol: "file:",
|
1682
|
+
factory: async (url, sthis) => {
|
1683
|
+
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1684
|
+
return new KeyBagProviderFile2(url, sthis);
|
2018
1685
|
}
|
2019
|
-
|
2020
|
-
|
2021
|
-
|
2022
|
-
|
2023
|
-
|
2024
|
-
|
2025
|
-
const getCompactCarCids = async (carCid) => {
|
2026
|
-
const reader = await this.loadCar(carCid);
|
2027
|
-
if (!reader) {
|
2028
|
-
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
1686
|
+
},
|
1687
|
+
{
|
1688
|
+
protocol: "indexdb:",
|
1689
|
+
factory: async (url, sthis) => {
|
1690
|
+
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1691
|
+
return new KeyBagProviderIndexDB2(url, sthis);
|
2029
1692
|
}
|
2030
|
-
const header = await parseCarFile(reader, this.logger);
|
2031
|
-
const compacts = header.compact;
|
2032
|
-
let got2;
|
2033
|
-
const batchSize2 = 5;
|
2034
|
-
for (let i = 0; i < compacts.length; i += batchSize2) {
|
2035
|
-
const promises = [];
|
2036
|
-
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
2037
|
-
for (const cid2 of compacts[j]) {
|
2038
|
-
promises.push(getCarCid(cid2));
|
2039
|
-
}
|
2040
|
-
}
|
2041
|
-
try {
|
2042
|
-
got2 = await Promise.any(promises);
|
2043
|
-
} catch {
|
2044
|
-
}
|
2045
|
-
if (got2) break;
|
2046
|
-
}
|
2047
|
-
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
2048
|
-
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
2049
|
-
};
|
2050
|
-
let got;
|
2051
|
-
const batchSize = 5;
|
2052
|
-
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
2053
|
-
const batch = this.carLog.slice(i, i + batchSize);
|
2054
|
-
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
2055
|
-
try {
|
2056
|
-
got = await Promise.any(promises);
|
2057
|
-
} catch {
|
2058
|
-
}
|
2059
|
-
if (got) break;
|
2060
1693
|
}
|
2061
|
-
|
2062
|
-
|
2063
|
-
|
2064
|
-
|
1694
|
+
].map((i) => [i.protocol, i])
|
1695
|
+
);
|
1696
|
+
function registerKeyBagProviderFactory(item) {
|
1697
|
+
const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
|
1698
|
+
keyBagProviderFactories.set(protocol, {
|
1699
|
+
...item,
|
1700
|
+
protocol
|
1701
|
+
});
|
1702
|
+
}
|
1703
|
+
function defaultKeyBagOpts(sthis, kbo) {
|
1704
|
+
kbo = kbo || {};
|
1705
|
+
if (kbo.keyRuntime) {
|
1706
|
+
return kbo.keyRuntime;
|
1707
|
+
}
|
1708
|
+
const logger = ensureLogger(sthis, "KeyBag");
|
1709
|
+
let url;
|
1710
|
+
if (kbo.url) {
|
1711
|
+
url = import_cement6.URI.from(kbo.url);
|
1712
|
+
logger.Debug().Url(url).Msg("from opts");
|
1713
|
+
} else {
|
1714
|
+
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
1715
|
+
if ((0, import_cement6.runtimeFn)().isBrowser) {
|
1716
|
+
url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
1717
|
+
} else {
|
1718
|
+
if (!bagFnameOrUrl) {
|
1719
|
+
const home = sthis.env.get("HOME");
|
1720
|
+
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
1721
|
+
url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
|
1722
|
+
} else {
|
1723
|
+
url = import_cement6.URI.from(bagFnameOrUrl);
|
2065
1724
|
}
|
2066
1725
|
}
|
2067
|
-
|
1726
|
+
logger.Debug().Url(url).Msg("from env");
|
2068
1727
|
}
|
2069
|
-
|
2070
|
-
|
2071
|
-
|
1728
|
+
let keyProviderFactory;
|
1729
|
+
switch (url.protocol) {
|
1730
|
+
case "file:":
|
1731
|
+
keyProviderFactory = async () => {
|
1732
|
+
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1733
|
+
return new KeyBagProviderFile2(url, sthis);
|
1734
|
+
};
|
1735
|
+
break;
|
1736
|
+
case "indexdb:":
|
1737
|
+
keyProviderFactory = async () => {
|
1738
|
+
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1739
|
+
return new KeyBagProviderIndexDB2(url, sthis);
|
1740
|
+
};
|
1741
|
+
break;
|
1742
|
+
case "memory:":
|
1743
|
+
keyProviderFactory = async () => {
|
1744
|
+
const { KeyBagProviderMemory: KeyBagProviderMemory2 } = await Promise.resolve().then(() => (init_key_bag_memory(), key_bag_memory_exports));
|
1745
|
+
return new KeyBagProviderMemory2(url, sthis);
|
1746
|
+
};
|
1747
|
+
break;
|
1748
|
+
default:
|
1749
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
1750
|
+
}
|
1751
|
+
if (url.hasParam("masterkey")) {
|
1752
|
+
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
1753
|
+
}
|
1754
|
+
return {
|
1755
|
+
url,
|
1756
|
+
crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
|
1757
|
+
sthis,
|
1758
|
+
logger,
|
1759
|
+
keyLength: kbo.keyLength || 16,
|
1760
|
+
getBag: keyProviderFactory,
|
1761
|
+
id: () => {
|
1762
|
+
return url.toString();
|
2072
1763
|
}
|
2073
|
-
|
2074
|
-
|
1764
|
+
};
|
1765
|
+
}
|
1766
|
+
var _keyBags = new import_cement6.KeyedResolvOnce();
|
1767
|
+
async function getKeyBag(sthis, kbo = {}) {
|
1768
|
+
await sthis.start();
|
1769
|
+
const rt = defaultKeyBagOpts(sthis, kbo);
|
1770
|
+
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
1771
|
+
}
|
1772
|
+
|
1773
|
+
// src/blockstore/meta-key-helper.ts
|
1774
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1775
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1776
|
+
if (!crdtEntries.length) {
|
1777
|
+
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
1778
|
+
return [];
|
2075
1779
|
}
|
2076
|
-
|
2077
|
-
|
2078
|
-
|
2079
|
-
|
2080
|
-
|
2081
|
-
|
2082
|
-
|
2083
|
-
|
2084
|
-
|
2085
|
-
|
2086
|
-
|
2087
|
-
|
2088
|
-
|
2089
|
-
|
2090
|
-
|
2091
|
-
|
1780
|
+
if (!crdtEntries.map) {
|
1781
|
+
sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
|
1782
|
+
return [];
|
1783
|
+
}
|
1784
|
+
return Promise.all(
|
1785
|
+
crdtEntries.map(async (crdtEntry) => {
|
1786
|
+
const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
|
1787
|
+
const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
1788
|
+
return {
|
1789
|
+
eventCid: eventBlock.cid,
|
1790
|
+
parents: crdtEntry.parents,
|
1791
|
+
dbMeta
|
1792
|
+
};
|
1793
|
+
})
|
1794
|
+
);
|
1795
|
+
}
|
1796
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
1797
|
+
try {
|
1798
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
1799
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
1800
|
+
if (keyInfo.length) {
|
1801
|
+
const dbMeta = keyInfo[0].dbMeta;
|
1802
|
+
if (dbMeta.key) {
|
1803
|
+
const kb = await getKeyBag(sthis);
|
1804
|
+
const keyName = getStoreKeyName(uri);
|
1805
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1806
|
+
if (res.isErr()) {
|
1807
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1808
|
+
throw res.Err();
|
2092
1809
|
}
|
2093
|
-
} else {
|
2094
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
2095
1810
|
}
|
1811
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1812
|
+
return import_cement7.Result.Ok(dbMeta);
|
2096
1813
|
}
|
2097
|
-
|
2098
|
-
|
2099
|
-
|
2100
|
-
|
2101
|
-
|
2102
|
-
const readerP = Promise.resolve(rawReader);
|
2103
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
2104
|
-
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
2105
|
-
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
2106
|
-
return;
|
2107
|
-
});
|
2108
|
-
return reader;
|
2109
|
-
});
|
2110
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
2111
|
-
return readerP;
|
1814
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
1815
|
+
return import_cement7.Result.Ok(void 0);
|
1816
|
+
} catch (error) {
|
1817
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1818
|
+
return import_cement7.Result.Err(error);
|
2112
1819
|
}
|
2113
|
-
|
2114
|
-
|
2115
|
-
|
2116
|
-
|
2117
|
-
|
2118
|
-
|
2119
|
-
|
1820
|
+
}
|
1821
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1822
|
+
try {
|
1823
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1824
|
+
const keyName = getStoreKeyName(uri);
|
1825
|
+
const kb = await getKeyBag(sthis);
|
1826
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
1827
|
+
if (res.isErr()) {
|
1828
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1829
|
+
throw res.Err();
|
2120
1830
|
}
|
2121
|
-
|
1831
|
+
const keyData = await res.Ok().extract();
|
1832
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
1833
|
+
const { dbMeta, parents } = dbMetas[0];
|
1834
|
+
const parentLinks = parents.map((p) => import_multiformats.CID.parse(p));
|
1835
|
+
dbMeta.key = keyData.keyStr;
|
1836
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
1837
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
1838
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
1839
|
+
return import_cement7.Result.Ok(encoded);
|
1840
|
+
} catch (error) {
|
1841
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
1842
|
+
return import_cement7.Result.Err(error);
|
2122
1843
|
}
|
2123
|
-
|
2124
|
-
|
2125
|
-
|
2126
|
-
|
1844
|
+
}
|
1845
|
+
function getStoreKeyName(url) {
|
1846
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
1847
|
+
const idx = url.getParam("index");
|
1848
|
+
if (idx) {
|
1849
|
+
storeKeyName.push(idx);
|
2127
1850
|
}
|
2128
|
-
|
1851
|
+
storeKeyName.push("data");
|
1852
|
+
return `@${storeKeyName.join(":")}@`;
|
1853
|
+
}
|
1854
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
1855
|
+
const event = await import_clock.EventBlock.create(
|
1856
|
+
{
|
1857
|
+
dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
|
1858
|
+
},
|
1859
|
+
parents
|
1860
|
+
);
|
1861
|
+
return event;
|
1862
|
+
}
|
1863
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
1864
|
+
const crdtEntries = events.map((event) => {
|
1865
|
+
const base64String = import_base64.base64pad.encode(event.bytes);
|
1866
|
+
return {
|
1867
|
+
cid: event.cid.toString(),
|
1868
|
+
data: base64String,
|
1869
|
+
parents: parents.map((p) => p.toString())
|
1870
|
+
};
|
1871
|
+
});
|
1872
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
1873
|
+
}
|
2129
1874
|
|
2130
|
-
// src/
|
2131
|
-
var
|
2132
|
-
|
2133
|
-
|
2134
|
-
|
1875
|
+
// src/blockstore/store.ts
|
1876
|
+
var import_p_retry = __toESM(require("p-retry"), 1);
|
1877
|
+
var import_p_map = __toESM(require("p-map"), 1);
|
1878
|
+
|
1879
|
+
// src/blockstore/loader.ts
|
1880
|
+
var import_p_limit = __toESM(require("p-limit"), 1);
|
1881
|
+
var import_car = require("@ipld/car");
|
1882
|
+
var import_cement9 = require("@adviser/cement");
|
1883
|
+
|
1884
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
1885
|
+
var block_exports = {};
|
1886
|
+
__export(block_exports, {
|
1887
|
+
Block: () => Block,
|
1888
|
+
create: () => create,
|
1889
|
+
createUnsafe: () => createUnsafe,
|
1890
|
+
decode: () => decode3,
|
1891
|
+
encode: () => encode3
|
2135
1892
|
});
|
1893
|
+
var import_multiformats2 = require("multiformats");
|
1894
|
+
var import_block = require("multiformats/block");
|
1895
|
+
var Block = import_block.Block;
|
1896
|
+
async function decode3({
|
1897
|
+
bytes,
|
1898
|
+
codec: codec3,
|
1899
|
+
hasher: hasher7
|
1900
|
+
}) {
|
1901
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
1902
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
1903
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
1904
|
+
const hash = await hasher7.digest(bytes);
|
1905
|
+
const cid = import_multiformats2.CID.create(1, codec3.code, hash);
|
1906
|
+
return new import_block.Block({ value, bytes, cid });
|
1907
|
+
}
|
1908
|
+
async function encode3({
|
1909
|
+
value,
|
1910
|
+
codec: codec3,
|
1911
|
+
hasher: hasher7
|
1912
|
+
}) {
|
1913
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
1914
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
1915
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
1916
|
+
const hash = await hasher7.digest(bytes);
|
1917
|
+
const cid = import_multiformats2.CID.create(1, codec3.code, hash);
|
1918
|
+
return new import_block.Block({ value, bytes, cid });
|
1919
|
+
}
|
1920
|
+
async function create({
|
1921
|
+
bytes,
|
1922
|
+
cid,
|
1923
|
+
hasher: hasher7,
|
1924
|
+
codec: codec3
|
1925
|
+
}) {
|
1926
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
1927
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
1928
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
1929
|
+
const hash = await hasher7.digest(bytes);
|
1930
|
+
if (!import_multiformats2.bytes.equals(cid.multihash.bytes, hash.bytes)) {
|
1931
|
+
throw new Error("CID hash does not match bytes");
|
1932
|
+
}
|
1933
|
+
return createUnsafe({
|
1934
|
+
bytes,
|
1935
|
+
cid,
|
1936
|
+
value,
|
1937
|
+
codec: codec3
|
1938
|
+
});
|
1939
|
+
}
|
1940
|
+
async function createUnsafe({
|
1941
|
+
bytes,
|
1942
|
+
cid,
|
1943
|
+
value: maybeValue,
|
1944
|
+
codec: codec3
|
1945
|
+
}) {
|
1946
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
1947
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
1948
|
+
return new Block({
|
1949
|
+
cid,
|
1950
|
+
bytes,
|
1951
|
+
value
|
1952
|
+
});
|
1953
|
+
}
|
1954
|
+
|
1955
|
+
// src/blockstore/loader-helpers.ts
|
1956
|
+
var import_sha22 = require("multiformats/hashes/sha2");
|
1957
|
+
var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
|
1958
|
+
async function parseCarFile(reader, logger) {
|
1959
|
+
const roots = await reader.getRoots();
|
1960
|
+
const header = await reader.get(roots[0]);
|
1961
|
+
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
1962
|
+
const dec = await decode3({ bytes: header.bytes, hasher: import_sha22.sha256, codec: dagCodec });
|
1963
|
+
const fpvalue = dec.value;
|
1964
|
+
if (fpvalue && !fpvalue.fp) {
|
1965
|
+
throw logger.Error().Msg("missing fp").AsError();
|
1966
|
+
}
|
1967
|
+
return fpvalue.fp;
|
1968
|
+
}
|
1969
|
+
|
1970
|
+
// src/blockstore/transaction.ts
|
1971
|
+
var import_block3 = require("@web3-storage/pail/block");
|
1972
|
+
init_types();
|
1973
|
+
var import_cement8 = require("@adviser/cement");
|
2136
1974
|
init_utils();
|
2137
|
-
var
|
2138
|
-
|
2139
|
-
|
2140
|
-
|
2141
|
-
|
1975
|
+
var CarTransaction = class extends import_block3.MemoryBlockstore {
|
1976
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
1977
|
+
super();
|
1978
|
+
if (opts.add) {
|
1979
|
+
parent.transactions.add(this);
|
1980
|
+
}
|
1981
|
+
this.parent = parent;
|
1982
|
+
}
|
1983
|
+
async get(cid) {
|
1984
|
+
return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
|
1985
|
+
}
|
1986
|
+
async superGet(cid) {
|
1987
|
+
return super.get(cid);
|
1988
|
+
}
|
1989
|
+
};
|
1990
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
1991
|
+
const logger = ensureLogger(sthis, component, ctx);
|
1992
|
+
return {
|
2142
1993
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2143
|
-
|
2144
|
-
return
|
1994
|
+
applyMeta: (meta, snap) => {
|
1995
|
+
return Promise.resolve();
|
2145
1996
|
},
|
2146
1997
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2147
|
-
|
2148
|
-
return
|
1998
|
+
compact: async (blocks) => {
|
1999
|
+
return {};
|
2000
|
+
},
|
2001
|
+
autoCompact: 100,
|
2002
|
+
public: false,
|
2003
|
+
// name: undefined,
|
2004
|
+
threshold: 1e3 * 1e3,
|
2005
|
+
...opts,
|
2006
|
+
logger,
|
2007
|
+
keyBag: opts.keyBag || {},
|
2008
|
+
crypto: (0, import_cement8.toCryptoRuntime)(opts.crypto),
|
2009
|
+
storeUrls: opts.storeUrls,
|
2010
|
+
// storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
|
2011
|
+
// store,
|
2012
|
+
storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
|
2013
|
+
};
|
2014
|
+
}
|
2015
|
+
function blockstoreFactory(sthis, opts) {
|
2016
|
+
return new EncryptedBlockstore(sthis, opts);
|
2017
|
+
}
|
2018
|
+
var BaseBlockstore = class {
|
2019
|
+
constructor(ebOpts) {
|
2020
|
+
this.transactions = /* @__PURE__ */ new Set();
|
2021
|
+
this.sthis = ensureSuperThis(ebOpts);
|
2022
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
2023
|
+
this.sthis = ensureSuperThis(ebOpts);
|
2024
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
2025
|
+
this.logger = this.ebOpts.logger;
|
2026
|
+
}
|
2027
|
+
// readonly name?: string;
|
2028
|
+
// ready: Promise<void>;
|
2029
|
+
ready() {
|
2030
|
+
return Promise.resolve();
|
2031
|
+
}
|
2032
|
+
async close() {
|
2033
|
+
}
|
2034
|
+
async destroy() {
|
2035
|
+
}
|
2036
|
+
async compact() {
|
2037
|
+
}
|
2038
|
+
async get(cid) {
|
2039
|
+
if (!cid) throw this.logger.Error().Msg("required cid").AsError();
|
2040
|
+
for (const f of this.transactions) {
|
2041
|
+
const v = await f.superGet(cid);
|
2042
|
+
if (v) return v;
|
2149
2043
|
}
|
2150
|
-
}
|
2151
|
-
|
2152
|
-
|
2153
|
-
|
2154
|
-
|
2155
|
-
|
2156
|
-
|
2157
|
-
|
2044
|
+
}
|
2045
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2046
|
+
async put(cid, block) {
|
2047
|
+
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
2048
|
+
}
|
2049
|
+
// TransactionMeta
|
2050
|
+
async transaction(fn, _opts) {
|
2051
|
+
this.logger.Debug().Msg("enter transaction");
|
2052
|
+
const t = new CarTransaction(this, _opts);
|
2053
|
+
this.logger.Debug().Msg("post CarTransaction");
|
2054
|
+
const done = await fn(t);
|
2055
|
+
this.logger.Debug().Msg("post fn");
|
2056
|
+
this.lastTxMeta = done;
|
2057
|
+
return { t, meta: done };
|
2058
|
+
}
|
2059
|
+
openTransaction(opts = { add: true, noLoader: false }) {
|
2060
|
+
return new CarTransaction(this, opts);
|
2061
|
+
}
|
2062
|
+
async commitTransaction(t, done, opts) {
|
2063
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
|
2064
|
+
const cars = await this.loader?.commit(t, done, opts);
|
2065
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
2066
|
+
setTimeout(() => void this.compact(), 10);
|
2067
|
+
}
|
2068
|
+
if (cars) {
|
2069
|
+
this.transactions.delete(t);
|
2070
|
+
return { meta: done, cars, t };
|
2071
|
+
}
|
2072
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
2073
|
+
}
|
2074
|
+
async *entries() {
|
2075
|
+
const seen = /* @__PURE__ */ new Set();
|
2076
|
+
for (const t of this.transactions) {
|
2077
|
+
for await (const blk of t.entries()) {
|
2078
|
+
if (seen.has(blk.cid.toString())) continue;
|
2079
|
+
seen.add(blk.cid.toString());
|
2080
|
+
yield blk;
|
2158
2081
|
}
|
2159
|
-
return hashArray;
|
2160
|
-
},
|
2161
|
-
verify: async function(ko, crypto, iv, data) {
|
2162
|
-
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
2163
2082
|
}
|
2164
2083
|
}
|
2165
2084
|
};
|
2166
|
-
|
2167
|
-
|
2168
|
-
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
this.code = 3147065;
|
2173
|
-
this.name = "Fireproof@encrypted-block:aes-gcm";
|
2174
|
-
this.ko = ko;
|
2175
|
-
this.iv = iv;
|
2176
|
-
this.opts = opts || {};
|
2177
|
-
}
|
2178
|
-
async encode(data) {
|
2179
|
-
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
2180
|
-
const { iv } = this.ko.algo(calcIv);
|
2181
|
-
const fprt = await this.ko.fingerPrint();
|
2182
|
-
const keyId = import_base583.base58btc.decode(fprt);
|
2183
|
-
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
2184
|
-
return CBOR.encode({
|
2185
|
-
iv,
|
2186
|
-
keyId,
|
2187
|
-
data: await this.ko._encrypt({ iv, bytes: data })
|
2085
|
+
var EncryptedBlockstore = class extends BaseBlockstore {
|
2086
|
+
constructor(sthis, ebOpts) {
|
2087
|
+
super(ebOpts);
|
2088
|
+
this.compacting = false;
|
2089
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
|
2090
|
+
this: 1
|
2188
2091
|
});
|
2092
|
+
this.loader = new Loader(sthis, ebOpts);
|
2189
2093
|
}
|
2190
|
-
|
2191
|
-
|
2192
|
-
|
2193
|
-
|
2194
|
-
|
2195
|
-
|
2094
|
+
ready() {
|
2095
|
+
return this.loader.ready();
|
2096
|
+
}
|
2097
|
+
close() {
|
2098
|
+
return this.loader.close();
|
2099
|
+
}
|
2100
|
+
destroy() {
|
2101
|
+
return this.loader.destroy();
|
2102
|
+
}
|
2103
|
+
async get(cid) {
|
2104
|
+
const got = await super.get(cid);
|
2105
|
+
if (got) return got;
|
2106
|
+
if (!this.loader) {
|
2107
|
+
return;
|
2196
2108
|
}
|
2197
|
-
|
2198
|
-
|
2199
|
-
|
2200
|
-
|
2201
|
-
|
2109
|
+
return falsyToUndef(await this.loader.getBlock(cid));
|
2110
|
+
}
|
2111
|
+
async transaction(fn, opts = { noLoader: false }) {
|
2112
|
+
this.logger.Debug().Msg("enter transaction");
|
2113
|
+
const { t, meta: done } = await super.transaction(fn);
|
2114
|
+
this.logger.Debug().Msg("post super.transaction");
|
2115
|
+
const cars = await this.loader.commit(t, done, opts);
|
2116
|
+
this.logger.Debug().Msg("post this.loader.commit");
|
2117
|
+
if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
|
2118
|
+
setTimeout(() => void this.compact(), 10);
|
2202
2119
|
}
|
2203
|
-
|
2204
|
-
|
2205
|
-
|
2120
|
+
if (cars) {
|
2121
|
+
this.transactions.delete(t);
|
2122
|
+
return { meta: done, cars, t };
|
2206
2123
|
}
|
2207
|
-
|
2124
|
+
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
2208
2125
|
}
|
2209
|
-
|
2210
|
-
|
2211
|
-
|
2212
|
-
|
2213
|
-
|
2214
|
-
|
2215
|
-
|
2216
|
-
|
2217
|
-
this.
|
2126
|
+
async getFile(car, cid) {
|
2127
|
+
await this.ready();
|
2128
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
2129
|
+
const reader = await this.loader.loadFileCar(
|
2130
|
+
car
|
2131
|
+
/*, isPublic */
|
2132
|
+
);
|
2133
|
+
const block = await reader.get(cid);
|
2134
|
+
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
2135
|
+
return block.bytes;
|
2218
2136
|
}
|
2219
|
-
|
2220
|
-
|
2137
|
+
async compact() {
|
2138
|
+
await this.ready();
|
2139
|
+
if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
|
2140
|
+
if (this.loader.carLog.length < 2) return;
|
2141
|
+
const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
|
2142
|
+
if (!compactFn || this.compacting) return;
|
2143
|
+
const blockLog = new CompactionFetcher(this);
|
2144
|
+
this.compacting = true;
|
2145
|
+
const meta = await compactFn(blockLog);
|
2146
|
+
await this.loader?.commit(blockLog.loggedBlocks, meta, {
|
2147
|
+
compact: true,
|
2148
|
+
noLoader: true
|
2149
|
+
});
|
2150
|
+
this.compacting = false;
|
2221
2151
|
}
|
2222
|
-
|
2223
|
-
|
2152
|
+
async defaultCompact(blocks, logger) {
|
2153
|
+
if (!this.loader) {
|
2154
|
+
throw logger.Error().Msg("no loader").AsError();
|
2155
|
+
}
|
2156
|
+
if (!this.lastTxMeta) {
|
2157
|
+
throw logger.Error().Msg("no lastTxMeta").AsError();
|
2158
|
+
}
|
2159
|
+
for await (const blk of this.loader.entries(false)) {
|
2160
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
2161
|
+
}
|
2162
|
+
for (const t of this.transactions) {
|
2163
|
+
for await (const blk of t.entries()) {
|
2164
|
+
blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
|
2165
|
+
}
|
2166
|
+
}
|
2167
|
+
return this.lastTxMeta;
|
2224
2168
|
}
|
2225
|
-
|
2226
|
-
|
2227
|
-
|
2228
|
-
|
2229
|
-
tagLength: 128
|
2230
|
-
};
|
2169
|
+
async *entries() {
|
2170
|
+
for await (const blk of this.loader.entries()) {
|
2171
|
+
yield blk;
|
2172
|
+
}
|
2231
2173
|
}
|
2232
|
-
|
2233
|
-
|
2234
|
-
|
2174
|
+
};
|
2175
|
+
var CompactionFetcher = class {
|
2176
|
+
constructor(blocks) {
|
2177
|
+
this.blockstore = blocks;
|
2178
|
+
this.loggedBlocks = new CarTransaction(blocks);
|
2235
2179
|
}
|
2236
|
-
async
|
2237
|
-
|
2238
|
-
|
2239
|
-
return
|
2180
|
+
async get(cid) {
|
2181
|
+
const block = await this.blockstore.get(cid);
|
2182
|
+
if (block) this.loggedBlocks.putSync(cid, block.bytes);
|
2183
|
+
return falsyToUndef(block);
|
2240
2184
|
}
|
2241
2185
|
};
|
2242
|
-
|
2243
|
-
|
2244
|
-
|
2245
|
-
|
2186
|
+
|
2187
|
+
// src/blockstore/commitor.ts
|
2188
|
+
var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
|
2189
|
+
var import_sha23 = require("multiformats/hashes/sha2");
|
2190
|
+
var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
2191
|
+
async function encodeCarFile(roots, t, codec3) {
|
2192
|
+
let size = 0;
|
2193
|
+
const headerSize = CBW.headerLength({ roots });
|
2194
|
+
size += headerSize;
|
2195
|
+
for (const { cid, bytes } of t.entries()) {
|
2196
|
+
size += CBW.blockLength({ cid, bytes });
|
2246
2197
|
}
|
2247
|
-
|
2248
|
-
|
2198
|
+
const buffer = new Uint8Array(size);
|
2199
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
2200
|
+
for (const r of roots) {
|
2201
|
+
writer.addRoot(r);
|
2249
2202
|
}
|
2250
|
-
|
2251
|
-
|
2203
|
+
for (const { cid, bytes } of t.entries()) {
|
2204
|
+
writer.write({ cid, bytes });
|
2252
2205
|
}
|
2253
|
-
|
2254
|
-
|
2255
|
-
|
2256
|
-
|
2257
|
-
|
2258
|
-
|
2259
|
-
|
2260
|
-
|
2261
|
-
|
2262
|
-
|
2263
|
-
|
2206
|
+
writer.close();
|
2207
|
+
return await encode3({ value: writer.bytes, hasher: import_sha23.sha256, codec: codec3 });
|
2208
|
+
}
|
2209
|
+
async function createCarFile(encoder, cid, t) {
|
2210
|
+
return encodeCarFile([cid], t, encoder);
|
2211
|
+
}
|
2212
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
2213
|
+
const { files: roots } = makeFileCarHeader(done);
|
2214
|
+
const cids = [];
|
2215
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
2216
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
2217
|
+
for (const car of cars) {
|
2218
|
+
const { cid, bytes } = car;
|
2219
|
+
await fileStore.save({ cid, bytes });
|
2220
|
+
await walStore.enqueueFile(
|
2221
|
+
cid
|
2222
|
+
/*, !!opts.public*/
|
2223
|
+
);
|
2224
|
+
cids.push(cid);
|
2264
2225
|
}
|
2265
|
-
|
2266
|
-
|
2226
|
+
return cids;
|
2227
|
+
}
|
2228
|
+
function makeFileCarHeader(result) {
|
2229
|
+
const files = [];
|
2230
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
2231
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
2232
|
+
files.push(meta.cid);
|
2233
|
+
}
|
2267
2234
|
}
|
2268
|
-
|
2269
|
-
|
2270
|
-
|
2235
|
+
return { ...result, files };
|
2236
|
+
}
|
2237
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
2238
|
+
return [await encodeCarFile(roots, t, encoder)];
|
2239
|
+
}
|
2240
|
+
function makeCarHeader(meta, cars, compact = false) {
|
2241
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
2242
|
+
return { ...coreHeader, meta };
|
2243
|
+
}
|
2244
|
+
async function encodeCarHeader(fp) {
|
2245
|
+
return await encode3({
|
2246
|
+
value: { fp },
|
2247
|
+
hasher: import_sha23.sha256,
|
2248
|
+
codec: dagCodec2
|
2249
|
+
});
|
2250
|
+
}
|
2251
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
2252
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
2253
|
+
const rootBlock = await encodeCarHeader(fp);
|
2254
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
2255
|
+
const cids = [];
|
2256
|
+
for (const car of cars) {
|
2257
|
+
const { cid, bytes } = car;
|
2258
|
+
await params.carStore.save({ cid, bytes });
|
2259
|
+
cids.push(cid);
|
2271
2260
|
}
|
2272
|
-
|
2273
|
-
|
2274
|
-
|
2275
|
-
|
2276
|
-
|
2277
|
-
|
2278
|
-
|
2261
|
+
const newDbMeta = { cars: cids };
|
2262
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
2263
|
+
await params.metaStore.save(newDbMeta);
|
2264
|
+
return { cgrp: cids, header: fp };
|
2265
|
+
}
|
2266
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
2267
|
+
const carFiles = [];
|
2268
|
+
threshold = threshold || 128e3 * 8;
|
2269
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
2270
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
2271
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
2272
|
+
let cidRootBlock = rootBlock;
|
2273
|
+
for (const { cid, bytes } of t.entries()) {
|
2274
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
2275
|
+
if (newsize >= threshold) {
|
2276
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
2277
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
2278
|
+
clonedt.putSync(cid, bytes);
|
2279
|
+
cidRootBlock = { cid, bytes };
|
2280
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
2281
|
+
} else {
|
2282
|
+
clonedt.putSync(cid, bytes);
|
2283
|
+
}
|
2279
2284
|
}
|
2280
|
-
|
2281
|
-
|
2285
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
2286
|
+
return carFiles;
|
2287
|
+
}
|
2288
|
+
|
2289
|
+
// src/blockstore/loader.ts
|
2290
|
+
var import_sha24 = require("multiformats/hashes/sha2");
|
2291
|
+
|
2292
|
+
// src/blockstore/task-manager.ts
|
2293
|
+
init_utils();
|
2294
|
+
var TaskManager = class {
|
2295
|
+
constructor(sthis, callback) {
|
2296
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
2297
|
+
this.queue = [];
|
2298
|
+
this.isProcessing = false;
|
2299
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
2300
|
+
this.callback = callback;
|
2282
2301
|
}
|
2283
|
-
|
2284
|
-
|
2302
|
+
async handleEvent(cid, parents, dbMeta) {
|
2303
|
+
for (const parent of parents) {
|
2304
|
+
this.eventsWeHandled.add(parent.toString());
|
2305
|
+
}
|
2306
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
2307
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
2308
|
+
void this.processQueue();
|
2285
2309
|
}
|
2286
|
-
|
2287
|
-
|
2288
|
-
|
2289
|
-
|
2290
|
-
|
2291
|
-
if (
|
2292
|
-
|
2293
|
-
|
2294
|
-
|
2295
|
-
|
2310
|
+
async processQueue() {
|
2311
|
+
if (this.isProcessing) return;
|
2312
|
+
this.isProcessing = true;
|
2313
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2314
|
+
const first = filteredQueue[0];
|
2315
|
+
if (!first) {
|
2316
|
+
return;
|
2317
|
+
}
|
2318
|
+
try {
|
2319
|
+
await this.callback(first.dbMeta);
|
2320
|
+
this.eventsWeHandled.add(first.cid);
|
2321
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2322
|
+
} catch (err) {
|
2323
|
+
if (first.retries++ > 3) {
|
2324
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
2325
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
2326
|
+
}
|
2327
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
2328
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
2329
|
+
} finally {
|
2330
|
+
this.isProcessing = false;
|
2331
|
+
if (this.queue.length > 0) {
|
2332
|
+
void this.processQueue();
|
2296
2333
|
}
|
2297
2334
|
}
|
2298
|
-
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
2299
2335
|
}
|
2300
|
-
|
2301
|
-
}
|
2336
|
+
};
|
2302
2337
|
|
2303
|
-
// src/blockstore/
|
2304
|
-
|
2305
|
-
|
2306
|
-
|
2307
|
-
|
2308
|
-
|
2309
|
-
|
2310
|
-
|
2311
|
-
|
2312
|
-
|
2313
|
-
|
2314
|
-
if (isNaN(ret) || ret <= 0) {
|
2315
|
-
ret = 0;
|
2338
|
+
// src/blockstore/loader.ts
|
2339
|
+
function carLogIncludesGroup(list, cids) {
|
2340
|
+
return list.some((arr) => {
|
2341
|
+
return arr.toString() === cids.toString();
|
2342
|
+
});
|
2343
|
+
}
|
2344
|
+
function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
2345
|
+
const byString = /* @__PURE__ */ new Map();
|
2346
|
+
for (const cid of list) {
|
2347
|
+
if (remove.has(cid.toString())) continue;
|
2348
|
+
byString.set(cid.toString(), cid);
|
2316
2349
|
}
|
2317
|
-
return
|
2350
|
+
return [...byString.values()];
|
2318
2351
|
}
|
2319
|
-
|
2320
|
-
|
2321
|
-
|
2322
|
-
|
2323
|
-
|
2324
|
-
|
2325
|
-
|
2326
|
-
|
2327
|
-
|
2328
|
-
|
2329
|
-
|
2330
|
-
|
2331
|
-
|
2332
|
-
|
2352
|
+
var Loader = class {
|
2353
|
+
constructor(sthis, ebOpts) {
|
2354
|
+
this.commitQueue = new CommitQueue();
|
2355
|
+
this.isCompacting = false;
|
2356
|
+
this.carReaders = /* @__PURE__ */ new Map();
|
2357
|
+
this.seenCompacted = /* @__PURE__ */ new Set();
|
2358
|
+
this.processedCars = /* @__PURE__ */ new Set();
|
2359
|
+
this.carLog = [];
|
2360
|
+
this.getBlockCache = /* @__PURE__ */ new Map();
|
2361
|
+
this.seenMeta = /* @__PURE__ */ new Set();
|
2362
|
+
this.writeLimit = (0, import_p_limit.default)(1);
|
2363
|
+
this._carStore = new import_cement9.ResolveOnce();
|
2364
|
+
this._fileStore = new import_cement9.ResolveOnce();
|
2365
|
+
this._WALStore = new import_cement9.ResolveOnce();
|
2366
|
+
this._metaStore = new import_cement9.ResolveOnce();
|
2367
|
+
this.onceReady = new import_cement9.ResolveOnce();
|
2368
|
+
this.sthis = sthis;
|
2369
|
+
this.ebOpts = defaultedBlockstoreRuntime(
|
2370
|
+
sthis,
|
2371
|
+
{
|
2372
|
+
...ebOpts
|
2373
|
+
// name,
|
2374
|
+
},
|
2375
|
+
"Loader"
|
2376
|
+
);
|
2377
|
+
this.logger = this.ebOpts.logger;
|
2378
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
2379
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
2380
|
+
});
|
2381
|
+
}
|
2382
|
+
async carStore() {
|
2383
|
+
return this._carStore.once(
|
2384
|
+
async () => this.ebOpts.storeRuntime.makeDataStore({
|
2385
|
+
sthis: this.sthis,
|
2386
|
+
url: this.ebOpts.storeUrls.data,
|
2387
|
+
keybag: await this.keyBag()
|
2333
2388
|
})
|
2334
|
-
|
2389
|
+
);
|
2335
2390
|
}
|
2336
|
-
|
2337
|
-
|
2338
|
-
|
2391
|
+
async fileStore() {
|
2392
|
+
return this._fileStore.once(
|
2393
|
+
async () => this.ebOpts.storeRuntime.makeDataStore({
|
2394
|
+
sthis: this.sthis,
|
2395
|
+
url: this.ebOpts.storeUrls.file,
|
2396
|
+
keybag: await this.keyBag()
|
2397
|
+
})
|
2398
|
+
);
|
2339
2399
|
}
|
2340
|
-
|
2341
|
-
|
2342
|
-
|
2343
|
-
|
2344
|
-
|
2345
|
-
|
2346
|
-
|
2347
|
-
(async (furl, ofs2) => {
|
2348
|
-
const raw2 = await innerGW.get(furl);
|
2349
|
-
if (raw2.isErr()) {
|
2350
|
-
return raw2;
|
2351
|
-
}
|
2352
|
-
const fragment = (0, import_cborg.decode)(raw2.unwrap());
|
2353
|
-
if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
|
2354
|
-
return import_cement8.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
2355
|
-
}
|
2356
|
-
if (fragment.ofs !== ofs2) {
|
2357
|
-
return import_cement8.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
2358
|
-
}
|
2359
|
-
return import_cement8.Result.Ok(fragment);
|
2360
|
-
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
2400
|
+
async WALStore() {
|
2401
|
+
return this._WALStore.once(
|
2402
|
+
async () => this.ebOpts.storeRuntime.makeWALStore({
|
2403
|
+
sthis: this.sthis,
|
2404
|
+
url: this.ebOpts.storeUrls.wal,
|
2405
|
+
keybag: await this.keyBag()
|
2406
|
+
})
|
2361
2407
|
);
|
2362
2408
|
}
|
2363
|
-
|
2364
|
-
|
2365
|
-
|
2366
|
-
|
2367
|
-
|
2368
|
-
|
2369
|
-
|
2370
|
-
|
2371
|
-
this.innerGW = innerGW;
|
2409
|
+
async metaStore() {
|
2410
|
+
return this._metaStore.once(
|
2411
|
+
async () => this.ebOpts.storeRuntime.makeMetaStore({
|
2412
|
+
sthis: this.sthis,
|
2413
|
+
url: this.ebOpts.storeUrls.meta,
|
2414
|
+
keybag: await this.keyBag()
|
2415
|
+
})
|
2416
|
+
);
|
2372
2417
|
}
|
2373
|
-
|
2374
|
-
|
2375
|
-
|
2376
|
-
|
2377
|
-
|
2378
|
-
|
2379
|
-
|
2380
|
-
|
2381
|
-
|
2382
|
-
|
2383
|
-
const fid = this.sthis.nextId(this.fidLength);
|
2384
|
-
const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
|
2385
|
-
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
2386
|
-
const block = (0, import_cborg.encode)({
|
2387
|
-
fid: fid.bin,
|
2388
|
-
ofs,
|
2389
|
-
len: body.length,
|
2390
|
-
data: body.slice(ofs, ofs + blocksize)
|
2391
|
-
});
|
2392
|
-
if (block.length > fragSize) {
|
2393
|
-
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
2418
|
+
keyBag() {
|
2419
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
2420
|
+
}
|
2421
|
+
async ready() {
|
2422
|
+
return this.onceReady.once(async () => {
|
2423
|
+
const metas = await (await this.metaStore()).load();
|
2424
|
+
if (this.ebOpts.meta) {
|
2425
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
2426
|
+
} else if (metas) {
|
2427
|
+
await this.handleDbMetasFromStore(metas);
|
2394
2428
|
}
|
2395
|
-
|
2429
|
+
});
|
2430
|
+
}
|
2431
|
+
async close() {
|
2432
|
+
await this.commitQueue.waitIdle();
|
2433
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
2434
|
+
await Promise.all(toClose.map((store) => store.close()));
|
2435
|
+
}
|
2436
|
+
async destroy() {
|
2437
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
2438
|
+
await Promise.all(toDestroy.map((store) => store.destroy()));
|
2439
|
+
}
|
2440
|
+
// async snapToCar(carCid: AnyLink | string) {
|
2441
|
+
// await this.ready
|
2442
|
+
// if (typeof carCid === 'string') {
|
2443
|
+
// carCid = CID.parse(carCid)
|
2444
|
+
// }
|
2445
|
+
// const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
|
2446
|
+
// this.carLog = [carCid, ...carHeader.cars]
|
2447
|
+
// await this.getMoreReaders(carHeader.cars)
|
2448
|
+
// await this._applyCarHeader(carHeader, true)
|
2449
|
+
// }
|
2450
|
+
async handleDbMetasFromStore(metas) {
|
2451
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
2452
|
+
for (const meta of metas) {
|
2453
|
+
await this.writeLimit(async () => {
|
2454
|
+
await this.mergeDbMetaIntoClock(meta);
|
2455
|
+
});
|
2396
2456
|
}
|
2397
|
-
return ops;
|
2398
2457
|
}
|
2399
|
-
|
2400
|
-
|
2458
|
+
async mergeDbMetaIntoClock(meta) {
|
2459
|
+
if (this.isCompacting) {
|
2460
|
+
throw this.logger.Error().Msg("cannot merge while compacting").AsError();
|
2461
|
+
}
|
2462
|
+
if (this.seenMeta.has(meta.cars.toString())) return;
|
2463
|
+
this.seenMeta.add(meta.cars.toString());
|
2464
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
2465
|
+
return;
|
2466
|
+
}
|
2467
|
+
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
2468
|
+
carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
2469
|
+
await this.getMoreReaders(carHeader.cars.flat());
|
2470
|
+
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
2471
|
+
await this.ebOpts.applyMeta?.(carHeader.meta);
|
2401
2472
|
}
|
2402
|
-
async
|
2403
|
-
|
2473
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
2474
|
+
// const { key } = meta;
|
2475
|
+
// if (key) {
|
2476
|
+
// await this.setKey(key);
|
2477
|
+
// }
|
2478
|
+
// }
|
2479
|
+
async loadCarHeaderFromMeta({ cars: cids }) {
|
2480
|
+
const reader = await this.loadCar(cids[0]);
|
2481
|
+
return await parseCarFile(reader, this.logger);
|
2404
2482
|
}
|
2405
|
-
async
|
2406
|
-
|
2407
|
-
|
2408
|
-
|
2409
|
-
|
2410
|
-
|
2411
|
-
|
2412
|
-
|
2413
|
-
|
2414
|
-
|
2483
|
+
// async _getKey(): Promise<string | undefined> {
|
2484
|
+
// if (this.key) return this.key;
|
2485
|
+
// // generate a random key
|
2486
|
+
// if (!this.ebOpts.public) {
|
2487
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
2488
|
+
// }
|
2489
|
+
// return this.key || undefined;
|
2490
|
+
// }
|
2491
|
+
async commitFiles(t, done) {
|
2492
|
+
await this.ready();
|
2493
|
+
const fstore = await this.fileStore();
|
2494
|
+
const wstore = await this.WALStore();
|
2495
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
2415
2496
|
}
|
2416
|
-
async
|
2417
|
-
return this.
|
2497
|
+
async loadFileCar(cid) {
|
2498
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
2418
2499
|
}
|
2419
|
-
async
|
2420
|
-
await
|
2421
|
-
|
2500
|
+
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
2501
|
+
await this.ready();
|
2502
|
+
const carStore = await this.carStore();
|
2503
|
+
const params = {
|
2504
|
+
encoder: (await carStore.keyedCrypto()).codec(),
|
2505
|
+
carLog: this.carLog,
|
2506
|
+
carStore,
|
2507
|
+
WALStore: await this.WALStore(),
|
2508
|
+
metaStore: await this.metaStore(),
|
2509
|
+
threshold: this.ebOpts.threshold
|
2510
|
+
};
|
2511
|
+
return this.commitQueue.enqueue(async () => {
|
2512
|
+
await this.cacheTransaction(t);
|
2513
|
+
const ret = await commit(params, t, done, opts);
|
2514
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
2515
|
+
return ret.cgrp;
|
2516
|
+
});
|
2422
2517
|
}
|
2423
|
-
async
|
2424
|
-
|
2425
|
-
|
2426
|
-
|
2427
|
-
|
2428
|
-
|
2429
|
-
|
2430
|
-
|
2431
|
-
buffer = buffer || new Uint8Array(frag.len);
|
2432
|
-
buffer.set(frag.data, frag.ofs);
|
2518
|
+
async updateCarLog(cids, fp, compact) {
|
2519
|
+
if (compact) {
|
2520
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
2521
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
2522
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
2523
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
2524
|
+
} else {
|
2525
|
+
this.carLog.unshift(cids);
|
2433
2526
|
}
|
2434
|
-
return import_cement8.Result.Ok(buffer || new Uint8Array(0));
|
2435
2527
|
}
|
2436
|
-
async
|
2437
|
-
|
2438
|
-
|
2439
|
-
|
2440
|
-
|
2528
|
+
async cacheTransaction(t) {
|
2529
|
+
for await (const block of t.entries()) {
|
2530
|
+
const sBlock = block.cid.toString();
|
2531
|
+
if (!this.getBlockCache.has(sBlock)) {
|
2532
|
+
this.getBlockCache.set(sBlock, block);
|
2533
|
+
}
|
2441
2534
|
}
|
2442
2535
|
}
|
2443
|
-
async
|
2444
|
-
|
2445
|
-
|
2446
|
-
|
2447
|
-
|
2536
|
+
async cacheCarReader(carCidStr, reader) {
|
2537
|
+
if (this.processedCars.has(carCidStr)) return;
|
2538
|
+
this.processedCars.add(carCidStr);
|
2539
|
+
for await (const block of reader.blocks()) {
|
2540
|
+
const sBlock = block.cid.toString();
|
2541
|
+
if (!this.getBlockCache.has(sBlock)) {
|
2542
|
+
this.getBlockCache.set(sBlock, block);
|
2448
2543
|
}
|
2449
|
-
const frag = rfrag.Ok();
|
2450
|
-
const fidStr = import_base584.base58btc.encode(frag.fid);
|
2451
|
-
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
2452
|
-
await this.innerGW.delete(fragUrl);
|
2453
2544
|
}
|
2454
|
-
return import_cement8.Result.Ok(void 0);
|
2455
2545
|
}
|
2456
|
-
|
2457
|
-
|
2458
|
-
|
2459
|
-
|
2460
|
-
|
2461
|
-
|
2462
|
-
|
2463
|
-
|
2464
|
-
|
2465
|
-
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
2466
|
-
if (!crdtEntries.length) {
|
2467
|
-
sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
|
2468
|
-
return [];
|
2546
|
+
async removeCidsForCompact(cid) {
|
2547
|
+
const carHeader = await this.loadCarHeaderFromMeta({
|
2548
|
+
cars: [cid]
|
2549
|
+
});
|
2550
|
+
for (const cids of carHeader.compact) {
|
2551
|
+
for (const cid2 of cids) {
|
2552
|
+
await (await this.carStore()).remove(cid2);
|
2553
|
+
}
|
2554
|
+
}
|
2469
2555
|
}
|
2470
|
-
|
2471
|
-
|
2472
|
-
|
2556
|
+
// async flushCars() {
|
2557
|
+
// await this.ready
|
2558
|
+
// // for each cid in car log, make a dbMeta
|
2559
|
+
// for (const cid of this.carLog) {
|
2560
|
+
// const dbMeta = { car: cid, key: this.key || null } as DbMeta
|
2561
|
+
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
2562
|
+
// }
|
2563
|
+
// }
|
2564
|
+
async *entries(cache2 = true) {
|
2565
|
+
await this.ready();
|
2566
|
+
if (cache2) {
|
2567
|
+
for (const [, block] of this.getBlockCache) {
|
2568
|
+
yield block;
|
2569
|
+
}
|
2570
|
+
} else {
|
2571
|
+
for (const [, block] of this.getBlockCache) {
|
2572
|
+
yield block;
|
2573
|
+
}
|
2574
|
+
for (const cids of this.carLog) {
|
2575
|
+
for (const cid of cids) {
|
2576
|
+
const reader = await this.loadCar(cid);
|
2577
|
+
if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
|
2578
|
+
for await (const block of reader.blocks()) {
|
2579
|
+
const sCid = block.cid.toString();
|
2580
|
+
if (!this.getBlockCache.has(sCid)) {
|
2581
|
+
yield block;
|
2582
|
+
}
|
2583
|
+
}
|
2584
|
+
}
|
2585
|
+
}
|
2586
|
+
}
|
2473
2587
|
}
|
2474
|
-
|
2475
|
-
|
2476
|
-
|
2477
|
-
|
2478
|
-
|
2479
|
-
|
2480
|
-
|
2481
|
-
|
2482
|
-
|
2483
|
-
|
2484
|
-
|
2485
|
-
|
2486
|
-
|
2487
|
-
|
2488
|
-
|
2489
|
-
|
2490
|
-
|
2491
|
-
const
|
2492
|
-
if (
|
2493
|
-
|
2494
|
-
|
2495
|
-
|
2496
|
-
|
2497
|
-
|
2498
|
-
|
2588
|
+
async getBlock(cid) {
|
2589
|
+
await this.ready();
|
2590
|
+
const sCid = cid.toString();
|
2591
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
2592
|
+
const getCarCid = async (carCid) => {
|
2593
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
2594
|
+
const reader = await this.loadCar(carCid);
|
2595
|
+
if (!reader) {
|
2596
|
+
throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
|
2597
|
+
}
|
2598
|
+
await this.cacheCarReader(carCid.toString(), reader).catch(() => {
|
2599
|
+
return;
|
2600
|
+
});
|
2601
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
2602
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
|
2603
|
+
};
|
2604
|
+
const getCompactCarCids = async (carCid) => {
|
2605
|
+
const reader = await this.loadCar(carCid);
|
2606
|
+
if (!reader) {
|
2607
|
+
throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
|
2608
|
+
}
|
2609
|
+
const header = await parseCarFile(reader, this.logger);
|
2610
|
+
const compacts = header.compact;
|
2611
|
+
let got2;
|
2612
|
+
const batchSize2 = 5;
|
2613
|
+
for (let i = 0; i < compacts.length; i += batchSize2) {
|
2614
|
+
const promises = [];
|
2615
|
+
for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
|
2616
|
+
for (const cid2 of compacts[j]) {
|
2617
|
+
promises.push(getCarCid(cid2));
|
2618
|
+
}
|
2619
|
+
}
|
2620
|
+
try {
|
2621
|
+
got2 = await Promise.any(promises);
|
2622
|
+
} catch {
|
2499
2623
|
}
|
2624
|
+
if (got2) break;
|
2500
2625
|
}
|
2501
|
-
|
2502
|
-
|
2626
|
+
if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
|
2627
|
+
throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
|
2628
|
+
};
|
2629
|
+
let got;
|
2630
|
+
const batchSize = 5;
|
2631
|
+
for (let i = 0; i < this.carLog.length; i += batchSize) {
|
2632
|
+
const batch = this.carLog.slice(i, i + batchSize);
|
2633
|
+
const promises = batch.flatMap((slice) => slice.map(getCarCid));
|
2634
|
+
try {
|
2635
|
+
got = await Promise.any(promises);
|
2636
|
+
} catch {
|
2637
|
+
}
|
2638
|
+
if (got) break;
|
2503
2639
|
}
|
2504
|
-
|
2505
|
-
|
2506
|
-
|
2507
|
-
|
2508
|
-
|
2640
|
+
if (!got) {
|
2641
|
+
try {
|
2642
|
+
got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
|
2643
|
+
} catch {
|
2644
|
+
}
|
2645
|
+
}
|
2646
|
+
return got;
|
2509
2647
|
}
|
2510
|
-
|
2511
|
-
|
2512
|
-
|
2513
|
-
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
2514
|
-
const keyName = getStoreKeyName(uri);
|
2515
|
-
const kb = await getKeyBag(sthis);
|
2516
|
-
const res = await kb.getNamedExtractableKey(keyName, true);
|
2517
|
-
if (res.isErr()) {
|
2518
|
-
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
2519
|
-
throw res.Err();
|
2648
|
+
async loadCar(cid) {
|
2649
|
+
if (!this.carStore) {
|
2650
|
+
throw this.logger.Error().Msg("car store not initialized").AsError();
|
2520
2651
|
}
|
2521
|
-
const
|
2522
|
-
|
2523
|
-
const { dbMeta, parents } = dbMetas[0];
|
2524
|
-
const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
|
2525
|
-
dbMeta.key = keyData.keyStr;
|
2526
|
-
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
2527
|
-
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
2528
|
-
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
2529
|
-
return import_cement9.Result.Ok(encoded);
|
2530
|
-
} catch (error) {
|
2531
|
-
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
2532
|
-
return import_cement9.Result.Err(error);
|
2652
|
+
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
2653
|
+
return loaded;
|
2533
2654
|
}
|
2534
|
-
|
2535
|
-
|
2536
|
-
|
2537
|
-
|
2538
|
-
|
2539
|
-
|
2655
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
2656
|
+
const cidsString = cid.toString();
|
2657
|
+
let loadedCar = void 0;
|
2658
|
+
let activeStore = local;
|
2659
|
+
try {
|
2660
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
2661
|
+
loadedCar = await local.load(cid);
|
2662
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
2663
|
+
} catch (e) {
|
2664
|
+
if (remote) {
|
2665
|
+
const remoteCar = await remote.load(cid);
|
2666
|
+
if (remoteCar) {
|
2667
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
2668
|
+
await local.save(remoteCar);
|
2669
|
+
loadedCar = remoteCar;
|
2670
|
+
activeStore = remote;
|
2671
|
+
}
|
2672
|
+
} else {
|
2673
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
2674
|
+
}
|
2675
|
+
}
|
2676
|
+
if (!loadedCar) {
|
2677
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
2678
|
+
}
|
2679
|
+
const bytes = await decode3({ bytes: loadedCar.bytes, hasher: import_sha24.sha256, codec: (await activeStore.keyedCrypto()).codec() });
|
2680
|
+
const rawReader = await import_car.CarReader.fromBytes(bytes.value);
|
2681
|
+
const readerP = Promise.resolve(rawReader);
|
2682
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
2683
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
2684
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
2685
|
+
return;
|
2686
|
+
});
|
2687
|
+
return reader;
|
2688
|
+
});
|
2689
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
2690
|
+
return readerP;
|
2540
2691
|
}
|
2541
|
-
|
2542
|
-
|
2543
|
-
|
2544
|
-
|
2545
|
-
|
2546
|
-
|
2547
|
-
|
2548
|
-
}
|
2549
|
-
|
2550
|
-
|
2551
|
-
|
2552
|
-
|
2553
|
-
|
2554
|
-
|
2555
|
-
|
2556
|
-
|
2557
|
-
cid: event.cid.toString(),
|
2558
|
-
data: base64String,
|
2559
|
-
parents: parents.map((p) => p.toString())
|
2560
|
-
};
|
2561
|
-
});
|
2562
|
-
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
2563
|
-
}
|
2692
|
+
//What if instead it returns an Array of CarHeader
|
2693
|
+
async storesLoadCar(cid, local, remote) {
|
2694
|
+
const cidsString = cid.toString();
|
2695
|
+
let dacr = this.carReaders.get(cidsString);
|
2696
|
+
if (!dacr) {
|
2697
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
2698
|
+
this.carReaders.set(cidsString, dacr);
|
2699
|
+
}
|
2700
|
+
return dacr;
|
2701
|
+
}
|
2702
|
+
async getMoreReaders(cids) {
|
2703
|
+
const limit = (0, import_p_limit.default)(5);
|
2704
|
+
const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
|
2705
|
+
await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
|
2706
|
+
}
|
2707
|
+
};
|
2564
2708
|
|
2565
2709
|
// src/blockstore/store.ts
|
2566
|
-
var import_p_retry = __toESM(require("p-retry"), 1);
|
2567
|
-
var import_p_map = __toESM(require("p-map"), 1);
|
2568
2710
|
function guardVersion(url) {
|
2569
2711
|
if (!url.hasParam("version")) {
|
2570
2712
|
return import_cement10.Result.Err(`missing version: ${url.toString()}`);
|
@@ -2572,16 +2714,21 @@ function guardVersion(url) {
|
|
2572
2714
|
return import_cement10.Result.Ok(url);
|
2573
2715
|
}
|
2574
2716
|
var BaseStoreImpl = class {
|
2575
|
-
constructor(
|
2717
|
+
constructor(sthis, url, opts, logger) {
|
2576
2718
|
this._onStarted = [];
|
2577
2719
|
this._onClosed = [];
|
2578
|
-
this.name = name;
|
2579
2720
|
this._url = url;
|
2580
2721
|
this.keybag = opts.keybag;
|
2722
|
+
this.loader = opts.loader;
|
2581
2723
|
this.sthis = sthis;
|
2582
|
-
|
2724
|
+
const name = this._url.getParam("name" /* NAME */);
|
2725
|
+
if (!name) {
|
2726
|
+
throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
|
2727
|
+
}
|
2728
|
+
this.name = name;
|
2729
|
+
this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
|
2730
|
+
this.realGateway = opts.gateway;
|
2583
2731
|
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
2584
|
-
this.loader = opts.loader;
|
2585
2732
|
}
|
2586
2733
|
url() {
|
2587
2734
|
return this._url;
|
@@ -2596,20 +2743,20 @@ var BaseStoreImpl = class {
|
|
2596
2743
|
return;
|
2597
2744
|
}
|
2598
2745
|
async keyedCrypto() {
|
2599
|
-
return keyedCryptoFactory(this._url,
|
2746
|
+
return keyedCryptoFactory(this._url, this.keybag, this.sthis);
|
2600
2747
|
}
|
2601
2748
|
async start() {
|
2602
2749
|
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
2603
|
-
this._url = this._url.build().setParam("store"
|
2750
|
+
this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
|
2604
2751
|
const res = await this.gateway.start(this._url);
|
2605
2752
|
if (res.isErr()) {
|
2606
2753
|
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
2607
2754
|
return res;
|
2608
2755
|
}
|
2609
2756
|
this._url = res.Ok();
|
2610
|
-
const kb = await this.keybag
|
2757
|
+
const kb = await this.keybag;
|
2611
2758
|
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
2612
|
-
const idx = this._url.getParam("index");
|
2759
|
+
const idx = this._url.getParam("index" /* INDEX */);
|
2613
2760
|
const storeKeyName = [this.name];
|
2614
2761
|
if (idx) {
|
2615
2762
|
storeKeyName.push(idx);
|
@@ -2642,8 +2789,8 @@ var BaseStoreImpl = class {
|
|
2642
2789
|
};
|
2643
2790
|
var MetaStoreImpl = class extends BaseStoreImpl {
|
2644
2791
|
// remote: boolean;
|
2645
|
-
constructor(sthis,
|
2646
|
-
super(
|
2792
|
+
constructor(sthis, url, opts) {
|
2793
|
+
super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
|
2647
2794
|
this.storeType = "meta";
|
2648
2795
|
this.subscribers = /* @__PURE__ */ new Map();
|
2649
2796
|
this.parents = [];
|
@@ -2714,13 +2861,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
|
|
2714
2861
|
return import_cement10.Result.Ok(void 0);
|
2715
2862
|
}
|
2716
2863
|
async destroy() {
|
2864
|
+
this.logger.Debug().Msg("destroy");
|
2717
2865
|
return this.gateway.destroy(this.url());
|
2718
2866
|
}
|
2719
2867
|
};
|
2720
2868
|
var DataStoreImpl = class extends BaseStoreImpl {
|
2721
2869
|
// readonly tag: string = "car-base";
|
2722
|
-
constructor(sthis,
|
2723
|
-
super(
|
2870
|
+
constructor(sthis, url, opts) {
|
2871
|
+
super(
|
2872
|
+
sthis,
|
2873
|
+
url,
|
2874
|
+
{
|
2875
|
+
...opts
|
2876
|
+
},
|
2877
|
+
ensureLogger(sthis, "DataStoreImpl")
|
2878
|
+
);
|
2724
2879
|
this.storeType = "data";
|
2725
2880
|
}
|
2726
2881
|
async load(cid) {
|
@@ -2761,23 +2916,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
|
|
2761
2916
|
return import_cement10.Result.Ok(void 0);
|
2762
2917
|
}
|
2763
2918
|
destroy() {
|
2919
|
+
this.logger.Debug().Msg("destroy");
|
2764
2920
|
return this.gateway.destroy(this.url());
|
2765
2921
|
}
|
2766
2922
|
};
|
2767
2923
|
var WALStoreImpl = class extends BaseStoreImpl {
|
2768
|
-
constructor(
|
2769
|
-
super(
|
2924
|
+
constructor(sthis, url, opts) {
|
2925
|
+
super(
|
2926
|
+
sthis,
|
2927
|
+
url,
|
2928
|
+
{
|
2929
|
+
...opts
|
2930
|
+
},
|
2931
|
+
ensureLogger(sthis, "WALStoreImpl")
|
2932
|
+
);
|
2770
2933
|
this.storeType = "wal";
|
2934
|
+
// readonly tag: string = "rwal-base";
|
2935
|
+
// readonly loader: Loadable;
|
2771
2936
|
this._ready = new import_cement10.ResolveOnce();
|
2772
2937
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
2773
2938
|
this.processing = void 0;
|
2774
2939
|
this.processQueue = new CommitQueue();
|
2775
|
-
this.loader = loader;
|
2776
2940
|
}
|
2777
2941
|
async ready() {
|
2778
2942
|
return this._ready.once(async () => {
|
2779
2943
|
const walState = await this.load().catch((e) => {
|
2780
|
-
this.logger.Error().
|
2944
|
+
this.logger.Error().Err(e).Msg("error loading wal");
|
2781
2945
|
return void 0;
|
2782
2946
|
});
|
2783
2947
|
if (!walState) {
|
@@ -2810,7 +2974,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2810
2974
|
}
|
2811
2975
|
async process() {
|
2812
2976
|
await this.ready();
|
2813
|
-
if (!this.loader
|
2977
|
+
if (!this.loader?.remoteCarStore) return;
|
2814
2978
|
await this.processQueue.enqueue(async () => {
|
2815
2979
|
try {
|
2816
2980
|
await this._doProcess();
|
@@ -2823,6 +2987,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2823
2987
|
});
|
2824
2988
|
}
|
2825
2989
|
async _doProcess() {
|
2990
|
+
if (!this.loader) return;
|
2826
2991
|
if (!this.loader.remoteCarStore) return;
|
2827
2992
|
const operations = [...this.walState.operations];
|
2828
2993
|
const noLoaderOps = [...this.walState.noLoaderOps];
|
@@ -2840,6 +3005,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2840
3005
|
noLoaderOps,
|
2841
3006
|
async (dbMeta) => {
|
2842
3007
|
await retryableUpload(async () => {
|
3008
|
+
if (!this.loader) return;
|
2843
3009
|
for (const cid of dbMeta.cars) {
|
2844
3010
|
const car = await (await this.loader.carStore()).load(cid);
|
2845
3011
|
if (!car) {
|
@@ -2859,6 +3025,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2859
3025
|
operations,
|
2860
3026
|
async (dbMeta) => {
|
2861
3027
|
await retryableUpload(async () => {
|
3028
|
+
if (!this.loader) return;
|
2862
3029
|
for (const cid of dbMeta.cars) {
|
2863
3030
|
const car = await (await this.loader.carStore()).load(cid);
|
2864
3031
|
if (!car) {
|
@@ -2878,6 +3045,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2878
3045
|
fileOperations,
|
2879
3046
|
async ({ cid: fileCid, public: publicFile }) => {
|
2880
3047
|
await retryableUpload(async () => {
|
3048
|
+
if (!this.loader) return;
|
2881
3049
|
const fileBlock = await (await this.loader.fileStore()).load(fileCid);
|
2882
3050
|
if (!fileBlock) {
|
2883
3051
|
throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
|
@@ -2891,6 +3059,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2891
3059
|
if (operations.length) {
|
2892
3060
|
const lastOp = operations[operations.length - 1];
|
2893
3061
|
await retryableUpload(async () => {
|
3062
|
+
if (!this.loader) return;
|
2894
3063
|
await this.loader.remoteMetaStore?.save(lastOp);
|
2895
3064
|
}, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
|
2896
3065
|
}
|
@@ -2916,6 +3085,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2916
3085
|
}
|
2917
3086
|
try {
|
2918
3087
|
return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
|
3088
|
+
return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
|
2919
3089
|
} catch (e) {
|
2920
3090
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
2921
3091
|
}
|
@@ -2942,73 +3112,98 @@ var WALStoreImpl = class extends BaseStoreImpl {
|
|
2942
3112
|
return import_cement10.Result.Ok(void 0);
|
2943
3113
|
}
|
2944
3114
|
destroy() {
|
3115
|
+
this.logger.Debug().Msg("destroy");
|
2945
3116
|
return this.gateway.destroy(this.url());
|
2946
3117
|
}
|
2947
3118
|
};
|
2948
3119
|
|
2949
3120
|
// src/blockstore/store-factory.ts
|
3121
|
+
init_types();
|
3122
|
+
|
3123
|
+
// src/blockstore/register-store-protocol.ts
|
3124
|
+
var import_cement14 = require("@adviser/cement");
|
3125
|
+
init_version();
|
3126
|
+
|
3127
|
+
// src/runtime/gateways/memory/gateway.ts
|
3128
|
+
var import_cement11 = require("@adviser/cement");
|
3129
|
+
init_types();
|
3130
|
+
|
3131
|
+
// src/runtime/gateways/memory/version.ts
|
3132
|
+
var MEMORY_VERSION = "v0.19-memory";
|
3133
|
+
|
3134
|
+
// src/runtime/gateways/memory/gateway.ts
|
2950
3135
|
init_utils();
|
2951
|
-
|
2952
|
-
|
2953
|
-
|
3136
|
+
var MemoryGateway = class {
|
3137
|
+
constructor(memorys) {
|
3138
|
+
this.memorys = memorys;
|
2954
3139
|
}
|
2955
|
-
|
2956
|
-
|
2957
|
-
function ensureName(name, url) {
|
2958
|
-
if (!url.hasParam("name")) {
|
2959
|
-
return url.build().setParam("name", name).URI();
|
3140
|
+
buildUrl(baseUrl, key) {
|
3141
|
+
return Promise.resolve(import_cement11.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
|
2960
3142
|
}
|
2961
|
-
|
2962
|
-
|
3143
|
+
start(baseUrl) {
|
3144
|
+
return Promise.resolve(import_cement11.Result.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
|
3145
|
+
}
|
3146
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
3147
|
+
close(baseUrl) {
|
3148
|
+
return Promise.resolve(import_cement11.Result.Ok(void 0));
|
3149
|
+
}
|
3150
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
3151
|
+
destroy(baseUrl) {
|
3152
|
+
this.memorys.clear();
|
3153
|
+
return Promise.resolve(import_cement11.Result.Ok(void 0));
|
3154
|
+
}
|
3155
|
+
put(url, body) {
|
3156
|
+
this.memorys.set(url.toString(), body);
|
3157
|
+
return Promise.resolve(import_cement11.Result.Ok(void 0));
|
3158
|
+
}
|
3159
|
+
// get could return a NotFoundError if the key is not found
|
3160
|
+
get(url) {
|
3161
|
+
const x = this.memorys.get(url.toString());
|
3162
|
+
if (x === void 0) {
|
3163
|
+
return Promise.resolve(import_cement11.Result.Err(new NotFoundError("not found")));
|
3164
|
+
}
|
3165
|
+
return Promise.resolve(import_cement11.Result.Ok(x));
|
3166
|
+
}
|
3167
|
+
delete(url) {
|
3168
|
+
this.memorys.delete(url.toString());
|
3169
|
+
return Promise.resolve(import_cement11.Result.Ok(void 0));
|
3170
|
+
}
|
3171
|
+
};
|
3172
|
+
var MemoryTestGateway = class {
|
3173
|
+
constructor(memorys) {
|
3174
|
+
this.memorys = memorys;
|
3175
|
+
}
|
3176
|
+
async get(url, key) {
|
3177
|
+
return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
|
3178
|
+
}
|
3179
|
+
};
|
3180
|
+
|
3181
|
+
// src/blockstore/register-store-protocol.ts
|
2963
3182
|
var storeFactory = /* @__PURE__ */ new Map();
|
2964
|
-
function
|
2965
|
-
|
2966
|
-
|
2967
|
-
|
2968
|
-
if (obuItem && obuItem.overrideBaseURL) {
|
2969
|
-
obuUrl = import_cement13.URI.from(obuItem.overrideBaseURL);
|
2970
|
-
}
|
2971
|
-
const ret = ensureIsIndex(
|
2972
|
-
import_cement13.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2973
|
-
storeOpts.isIndex
|
2974
|
-
);
|
2975
|
-
return ret;
|
2976
|
-
}
|
2977
|
-
var onceGateway = new import_cement13.KeyedResolvOnce();
|
2978
|
-
async function getGatewayFromURL(url, sthis) {
|
2979
|
-
return onceGateway.get(url.toString()).once(async () => {
|
2980
|
-
const item = storeFactory.get(url.protocol);
|
2981
|
-
if (item) {
|
2982
|
-
const ret = {
|
2983
|
-
gateway: await item.gateway(sthis),
|
2984
|
-
test: await item.test(sthis)
|
2985
|
-
};
|
2986
|
-
const res = await ret.gateway.start(url);
|
2987
|
-
if (res.isErr()) {
|
2988
|
-
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2989
|
-
return void 0;
|
2990
|
-
}
|
2991
|
-
return ret;
|
3183
|
+
function getDefaultURI(sthis, protocol) {
|
3184
|
+
if (protocol) {
|
3185
|
+
if (!protocol.endsWith(":")) {
|
3186
|
+
protocol += ":";
|
2992
3187
|
}
|
2993
|
-
|
2994
|
-
|
2995
|
-
|
3188
|
+
const gfi = storeFactory.get(protocol);
|
3189
|
+
if (gfi) {
|
3190
|
+
return gfi.defaultURI(sthis);
|
3191
|
+
}
|
3192
|
+
}
|
3193
|
+
const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
|
3194
|
+
if (!found) {
|
3195
|
+
throw sthis.logger.Error().Msg("no default found").AsError();
|
3196
|
+
}
|
3197
|
+
return found.defaultURI(sthis);
|
2996
3198
|
}
|
2997
3199
|
function registerStoreProtocol(item) {
|
2998
3200
|
let protocol = item.protocol;
|
2999
3201
|
if (!protocol.endsWith(":")) {
|
3000
3202
|
protocol += ":";
|
3001
3203
|
}
|
3002
|
-
if (
|
3003
|
-
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
3004
|
-
throw new Error(`we need a logger here`);
|
3005
|
-
return () => {
|
3006
|
-
};
|
3007
|
-
}
|
3008
|
-
}
|
3009
|
-
if (item.overrideBaseURL) {
|
3204
|
+
if (item.isDefault) {
|
3010
3205
|
Array.from(storeFactory.values()).forEach((items) => {
|
3011
|
-
items.
|
3206
|
+
items.isDefault = false;
|
3012
3207
|
});
|
3013
3208
|
}
|
3014
3209
|
storeFactory.set(protocol, item);
|
@@ -3016,134 +3211,186 @@ function registerStoreProtocol(item) {
|
|
3016
3211
|
storeFactory.delete(protocol);
|
3017
3212
|
};
|
3018
3213
|
}
|
3019
|
-
|
3020
|
-
|
3021
|
-
|
3022
|
-
|
3023
|
-
return
|
3024
|
-
|
3025
|
-
|
3026
|
-
|
3027
|
-
|
3028
|
-
|
3029
|
-
|
3030
|
-
|
3031
|
-
|
3032
|
-
|
3033
|
-
|
3034
|
-
|
3214
|
+
function getGatewayFactoryItem(protocol) {
|
3215
|
+
return storeFactory.get(protocol);
|
3216
|
+
}
|
3217
|
+
function fileGatewayFactoryItem() {
|
3218
|
+
return {
|
3219
|
+
protocol: "file:",
|
3220
|
+
isDefault: true,
|
3221
|
+
defaultURI: (sthis) => {
|
3222
|
+
return import_cement14.BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
|
3223
|
+
},
|
3224
|
+
gateway: async (sthis) => {
|
3225
|
+
const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
|
3226
|
+
return new FileGateway2(sthis);
|
3227
|
+
},
|
3228
|
+
test: async (sthis) => {
|
3229
|
+
const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
|
3230
|
+
return new FileTestStore2(sthis);
|
3231
|
+
}
|
3232
|
+
};
|
3233
|
+
}
|
3234
|
+
if ((0, import_cement14.runtimeFn)().isBrowser) {
|
3235
|
+
registerStoreProtocol({
|
3236
|
+
protocol: "indexdb:",
|
3237
|
+
isDefault: true,
|
3238
|
+
defaultURI: () => {
|
3239
|
+
return import_cement14.BuildURI.from("indexdb://").pathname("fp").URI();
|
3240
|
+
},
|
3241
|
+
gateway: async (logger) => {
|
3242
|
+
const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
|
3243
|
+
return new IndexDBGateway2(logger);
|
3244
|
+
},
|
3245
|
+
test: async (logger) => {
|
3246
|
+
const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
|
3247
|
+
return new IndexDBTestStore2(logger);
|
3248
|
+
}
|
3035
3249
|
});
|
3250
|
+
} else {
|
3251
|
+
registerStoreProtocol(fileGatewayFactoryItem());
|
3036
3252
|
}
|
3037
|
-
var
|
3038
|
-
|
3039
|
-
|
3040
|
-
|
3041
|
-
|
3042
|
-
|
3043
|
-
|
3044
|
-
|
3045
|
-
|
3046
|
-
|
3047
|
-
|
3048
|
-
|
3049
|
-
|
3050
|
-
|
3051
|
-
|
3052
|
-
|
3053
|
-
|
3253
|
+
var memory = /* @__PURE__ */ new Map();
|
3254
|
+
registerStoreProtocol({
|
3255
|
+
protocol: "memory:",
|
3256
|
+
isDefault: false,
|
3257
|
+
defaultURI: () => {
|
3258
|
+
return import_cement14.BuildURI.from("memory://").pathname("ram").URI();
|
3259
|
+
},
|
3260
|
+
gateway: async () => {
|
3261
|
+
return new MemoryGateway(memory);
|
3262
|
+
},
|
3263
|
+
test: async () => {
|
3264
|
+
return new MemoryTestGateway(memory);
|
3265
|
+
}
|
3266
|
+
});
|
3267
|
+
|
3268
|
+
// src/blockstore/store-factory.ts
|
3269
|
+
var onceGateway = new import_cement15.KeyedResolvOnce();
|
3270
|
+
var gatewayInstances = new import_cement15.KeyedResolvOnce();
|
3271
|
+
async function getStartedGateway(sthis, url) {
|
3272
|
+
return onceGateway.get(url.toString()).once(async () => {
|
3273
|
+
const item = getGatewayFactoryItem(url.protocol);
|
3274
|
+
if (item) {
|
3275
|
+
const ret = {
|
3276
|
+
url,
|
3277
|
+
...await gatewayInstances.get(url.protocol).once(async () => ({
|
3278
|
+
gateway: await item.gateway(sthis),
|
3279
|
+
test: await item.test(sthis)
|
3280
|
+
}))
|
3281
|
+
};
|
3282
|
+
const res = await ret.gateway.start(url);
|
3283
|
+
if (res.isErr()) {
|
3284
|
+
return import_cement15.Result.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
|
3285
|
+
}
|
3286
|
+
ret.url = res.Ok();
|
3287
|
+
return import_cement15.Result.Ok(ret);
|
3288
|
+
}
|
3289
|
+
return import_cement15.Result.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
|
3054
3290
|
});
|
3055
3291
|
}
|
3056
|
-
|
3057
|
-
|
3058
|
-
const
|
3059
|
-
|
3060
|
-
|
3061
|
-
|
3062
|
-
|
3063
|
-
|
3064
|
-
|
3065
|
-
|
3066
|
-
|
3067
|
-
|
3068
|
-
|
3069
|
-
|
3070
|
-
|
3071
|
-
|
3072
|
-
|
3292
|
+
async function dataStoreFactory(sfi) {
|
3293
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
|
3294
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
3295
|
+
if (rgateway.isErr()) {
|
3296
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
3297
|
+
}
|
3298
|
+
const gateway = rgateway.Ok();
|
3299
|
+
const store = new DataStoreImpl(sfi.sthis, gateway.url, {
|
3300
|
+
gateway: gateway.gateway,
|
3301
|
+
keybag: sfi.keybag
|
3302
|
+
});
|
3303
|
+
return store;
|
3304
|
+
}
|
3305
|
+
async function metaStoreFactory(sfi) {
|
3306
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
|
3307
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
3308
|
+
if (rgateway.isErr()) {
|
3309
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
3310
|
+
}
|
3311
|
+
const gateway = rgateway.Ok();
|
3312
|
+
const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
|
3313
|
+
gateway: gateway.gateway,
|
3314
|
+
keybag: sfi.keybag
|
3073
3315
|
});
|
3316
|
+
return store;
|
3317
|
+
}
|
3318
|
+
async function WALStoreFactory(sfi) {
|
3319
|
+
const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
|
3320
|
+
const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
|
3321
|
+
if (rgateway.isErr()) {
|
3322
|
+
throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
|
3323
|
+
}
|
3324
|
+
const gateway = rgateway.Ok();
|
3325
|
+
const store = new WALStoreImpl(sfi.sthis, gateway.url, {
|
3326
|
+
gateway: gateway.gateway,
|
3327
|
+
keybag: sfi.keybag
|
3328
|
+
});
|
3329
|
+
return store;
|
3074
3330
|
}
|
3075
3331
|
async function testStoreFactory(url, sthis) {
|
3076
|
-
|
3077
|
-
|
3078
|
-
if (!gateway) {
|
3332
|
+
const rgateway = await getStartedGateway(sthis, url);
|
3333
|
+
if (!rgateway) {
|
3079
3334
|
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
3080
3335
|
}
|
3081
|
-
return
|
3336
|
+
return rgateway.Ok().test;
|
3082
3337
|
}
|
3083
|
-
async function ensureStart(store
|
3338
|
+
async function ensureStart(store) {
|
3084
3339
|
const ret = await store.start();
|
3085
3340
|
if (ret.isErr()) {
|
3086
|
-
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
3341
|
+
throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
|
3087
3342
|
}
|
3088
|
-
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
3343
|
+
store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
3089
3344
|
return store;
|
3090
3345
|
}
|
3091
|
-
function
|
3092
|
-
|
3346
|
+
function ensureStoreEnDeFile(ende) {
|
3347
|
+
ende = ende || {};
|
3093
3348
|
return {
|
3094
|
-
|
3095
|
-
|
3096
|
-
return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
|
3097
|
-
},
|
3098
|
-
makeDataStore: async (loader) => {
|
3099
|
-
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
|
3100
|
-
return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
|
3101
|
-
},
|
3102
|
-
makeWALStore: async (loader) => {
|
3103
|
-
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
|
3104
|
-
return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
|
3105
|
-
},
|
3106
|
-
encodeFile: opts.encodeFile || encodeFile,
|
3107
|
-
decodeFile: opts.decodeFile || decodeFile
|
3349
|
+
encodeFile: ende.encodeFile || encodeFile,
|
3350
|
+
decodeFile: ende.decodeFile || decodeFile
|
3108
3351
|
};
|
3109
3352
|
}
|
3110
|
-
|
3111
|
-
|
3112
|
-
|
3113
|
-
|
3114
|
-
|
3115
|
-
|
3116
|
-
|
3117
|
-
|
3118
|
-
return
|
3119
|
-
|
3120
|
-
|
3121
|
-
|
3122
|
-
|
3123
|
-
|
3124
|
-
|
3125
|
-
|
3126
|
-
|
3127
|
-
|
3128
|
-
|
3129
|
-
|
3130
|
-
|
3131
|
-
|
3353
|
+
function toStoreRuntime(sthis, endeOpts = {}) {
|
3354
|
+
return {
|
3355
|
+
makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
|
3356
|
+
// async (loader: Loadable) => {
|
3357
|
+
// logger
|
3358
|
+
// .Debug()
|
3359
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
|
3360
|
+
// .Msg("makeMetaStore");
|
3361
|
+
// return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
|
3362
|
+
// },
|
3363
|
+
makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
|
3364
|
+
// async (loader: Loadable) => {
|
3365
|
+
// logger
|
3366
|
+
// .Debug()
|
3367
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
|
3368
|
+
// .Msg("makeDataStore");
|
3369
|
+
// return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
|
3370
|
+
// },
|
3371
|
+
makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
|
3372
|
+
// async (loader: Loadable) => {
|
3373
|
+
// logger
|
3374
|
+
// .Debug()
|
3375
|
+
// .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
|
3376
|
+
// .Msg("makeRemoteWAL");
|
3377
|
+
// return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
|
3378
|
+
// },
|
3379
|
+
...ensureStoreEnDeFile(endeOpts)
|
3380
|
+
};
|
3381
|
+
}
|
3382
|
+
|
3383
|
+
// src/blockstore/connection-base.ts
|
3384
|
+
init_types();
|
3132
3385
|
|
3133
3386
|
// src/blockstore/store-remote.ts
|
3134
3387
|
async function RemoteDataStore(sthis, name, url, opts) {
|
3135
|
-
const ds = new DataStoreImpl(sthis,
|
3388
|
+
const ds = new DataStoreImpl(sthis, url, opts);
|
3136
3389
|
await ds.start();
|
3137
3390
|
return ds;
|
3138
3391
|
}
|
3139
3392
|
async function RemoteMetaStore(sthis, name, url, opts) {
|
3140
|
-
const ms = new MetaStoreImpl(
|
3141
|
-
sthis,
|
3142
|
-
name,
|
3143
|
-
url,
|
3144
|
-
opts
|
3145
|
-
/* , true*/
|
3146
|
-
);
|
3393
|
+
const ms = new MetaStoreImpl(sthis, url, opts);
|
3147
3394
|
await ms.start();
|
3148
3395
|
return ms;
|
3149
3396
|
}
|
@@ -3168,14 +3415,17 @@ var ConnectionBase = class {
|
|
3168
3415
|
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
3169
3416
|
this.loader = loader;
|
3170
3417
|
await this.onConnect();
|
3171
|
-
const metaUrl = this.url.build().defParam("store"
|
3172
|
-
const
|
3173
|
-
if (
|
3418
|
+
const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
|
3419
|
+
const rgateway = await getStartedGateway(loader.sthis, metaUrl);
|
3420
|
+
if (rgateway.isErr())
|
3421
|
+
throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
3422
|
+
const name = metaUrl.toString();
|
3174
3423
|
const dbName = metaUrl.getParam("name");
|
3175
|
-
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X:
|
3176
|
-
const
|
3424
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
|
3425
|
+
const gateway = rgateway.Ok();
|
3426
|
+
const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
|
3177
3427
|
gateway: gateway.gateway,
|
3178
|
-
keybag:
|
3428
|
+
keybag: await loader.keyBag(),
|
3179
3429
|
loader
|
3180
3430
|
});
|
3181
3431
|
this.loader.remoteMetaStore = remote;
|
@@ -3188,14 +3438,15 @@ var ConnectionBase = class {
|
|
3188
3438
|
async connectStorage_X({ loader }) {
|
3189
3439
|
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
3190
3440
|
this.loader = loader;
|
3191
|
-
const dataUrl = this.url.build().defParam("store"
|
3192
|
-
const
|
3193
|
-
if (
|
3441
|
+
const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
|
3442
|
+
const rgateway = await getStartedGateway(loader.sthis, dataUrl);
|
3443
|
+
if (rgateway.isErr())
|
3444
|
+
throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
3194
3445
|
const name = dataUrl.getParam("name");
|
3195
3446
|
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
3196
3447
|
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
3197
|
-
gateway:
|
3198
|
-
keybag:
|
3448
|
+
gateway: rgateway.Ok().gateway,
|
3449
|
+
keybag: await loader.keyBag()
|
3199
3450
|
});
|
3200
3451
|
loader.remoteFileStore = loader.remoteCarStore;
|
3201
3452
|
}
|
@@ -3235,6 +3486,13 @@ var ConnectionBase = class {
|
|
3235
3486
|
};
|
3236
3487
|
|
3237
3488
|
// src/crdt-helpers.ts
|
3489
|
+
var import_link = require("multiformats/link");
|
3490
|
+
var import_sha25 = require("multiformats/hashes/sha2");
|
3491
|
+
var codec = __toESM(require("@ipld/dag-cbor"), 1);
|
3492
|
+
var import_crdt = require("@web3-storage/pail/crdt");
|
3493
|
+
var import_clock2 = require("@web3-storage/pail/clock");
|
3494
|
+
var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
|
3495
|
+
init_types();
|
3238
3496
|
function time(tag) {
|
3239
3497
|
}
|
3240
3498
|
function timeEnd(tag) {
|
@@ -3282,7 +3540,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
3282
3540
|
await processFiles(store, blocks, update.value, logger);
|
3283
3541
|
value = { doc: update.value };
|
3284
3542
|
}
|
3285
|
-
const block = await
|
3543
|
+
const block = await encode3({ value, hasher: import_sha25.sha256, codec });
|
3286
3544
|
blocks.putSync(block.cid, block.bytes);
|
3287
3545
|
return block.cid;
|
3288
3546
|
}
|
@@ -3373,7 +3631,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
3373
3631
|
async function getValueFromLink(blocks, link, logger) {
|
3374
3632
|
const block = await blocks.get(link);
|
3375
3633
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
3376
|
-
const { value } = await
|
3634
|
+
const { value } = await decode3({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
|
3377
3635
|
const cvalue = {
|
3378
3636
|
...value,
|
3379
3637
|
cid: link
|
@@ -3494,16 +3752,19 @@ async function doCompact(blockLog, head, logger) {
|
|
3494
3752
|
async function getBlock(blocks, cidString) {
|
3495
3753
|
const block = await blocks.get((0, import_link.parse)(cidString));
|
3496
3754
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
3497
|
-
const { cid, value } = await
|
3755
|
+
const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
|
3498
3756
|
return new Block({ cid, value, bytes: block.bytes });
|
3499
3757
|
}
|
3500
3758
|
|
3759
|
+
// src/indexer.ts
|
3760
|
+
init_types();
|
3761
|
+
|
3501
3762
|
// src/indexer-helpers.ts
|
3502
3763
|
var import_sha26 = require("multiformats/hashes/sha2");
|
3503
3764
|
var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
3504
3765
|
var import_charwise = __toESM(require("charwise"), 1);
|
3505
3766
|
var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
|
3506
|
-
var
|
3767
|
+
var import_utils15 = require("prolly-trees/utils");
|
3507
3768
|
var import_cache = require("prolly-trees/cache");
|
3508
3769
|
var IndexTree = class {
|
3509
3770
|
};
|
@@ -3511,17 +3772,17 @@ function refCompare(aRef, bRef) {
|
|
3511
3772
|
if (Number.isNaN(aRef)) return -1;
|
3512
3773
|
if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
|
3513
3774
|
if (aRef === Infinity) return 1;
|
3514
|
-
return (0,
|
3775
|
+
return (0, import_utils15.simpleCompare)(aRef, bRef);
|
3515
3776
|
}
|
3516
3777
|
function compare(a, b) {
|
3517
3778
|
const [aKey, aRef] = a;
|
3518
3779
|
const [bKey, bRef] = b;
|
3519
|
-
const comp = (0,
|
3780
|
+
const comp = (0, import_utils15.simpleCompare)(aKey, bKey);
|
3520
3781
|
if (comp !== 0) return comp;
|
3521
3782
|
return refCompare(aRef, bRef);
|
3522
3783
|
}
|
3523
|
-
var byKeyOpts = { cache: import_cache.nocache, chunker: (0,
|
3524
|
-
var byIdOpts = { cache: import_cache.nocache, chunker: (0,
|
3784
|
+
var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
|
3785
|
+
var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils15.simpleCompare };
|
3525
3786
|
function indexEntriesForChanges(changes, mapFn) {
|
3526
3787
|
const indexEntries = [];
|
3527
3788
|
changes.forEach(({ id: key, value, del }) => {
|
@@ -3552,7 +3813,8 @@ function makeProllyGetBlock(blocks) {
|
|
3552
3813
|
return create({ cid, bytes, hasher: import_sha26.sha256, codec: codec2 });
|
3553
3814
|
};
|
3554
3815
|
}
|
3555
|
-
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
3816
|
+
async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
|
3817
|
+
logger.Debug().Msg("enter bulkIndex");
|
3556
3818
|
if (!indexEntries.length) return inIndex;
|
3557
3819
|
if (!inIndex.root) {
|
3558
3820
|
if (!inIndex.cid) {
|
@@ -3569,18 +3831,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
|
3569
3831
|
returnNode = node;
|
3570
3832
|
}
|
3571
3833
|
if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
|
3834
|
+
logger.Debug().Msg("exit !root bulkIndex");
|
3572
3835
|
return { root: returnNode, cid: returnRootBlock.cid };
|
3573
3836
|
} else {
|
3574
3837
|
inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
|
3575
3838
|
}
|
3576
3839
|
}
|
3840
|
+
logger.Debug().Msg("pre bulk bulkIndex");
|
3577
3841
|
const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
|
3578
3842
|
if (root3) {
|
3843
|
+
logger.Debug().Msg("pre root put bulkIndex");
|
3579
3844
|
for await (const block of newBlocks) {
|
3580
3845
|
await tblocks.put(block.cid, block.bytes);
|
3581
3846
|
}
|
3582
3847
|
return { root: root3, cid: (await root3.block).cid };
|
3583
3848
|
} else {
|
3849
|
+
logger.Debug().Msg("pre !root bulkIndex");
|
3584
3850
|
return { root: void 0, cid: void 0 };
|
3585
3851
|
}
|
3586
3852
|
}
|
@@ -3621,17 +3887,17 @@ function encodeKey(key) {
|
|
3621
3887
|
|
3622
3888
|
// src/indexer.ts
|
3623
3889
|
init_utils();
|
3624
|
-
function index(
|
3625
|
-
if (mapFn && meta) throw
|
3626
|
-
if (mapFn && mapFn.constructor.name !== "Function") throw
|
3627
|
-
if (
|
3628
|
-
const idx =
|
3890
|
+
function index(refDb, name, mapFn, meta) {
|
3891
|
+
if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
3892
|
+
if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
3893
|
+
if (refDb.crdt.indexers.has(name)) {
|
3894
|
+
const idx = refDb.crdt.indexers.get(name);
|
3629
3895
|
idx.applyMapFn(name, mapFn, meta);
|
3630
3896
|
} else {
|
3631
|
-
const idx = new Index(sthis,
|
3632
|
-
|
3897
|
+
const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
|
3898
|
+
refDb.crdt.indexers.set(name, idx);
|
3633
3899
|
}
|
3634
|
-
return
|
3900
|
+
return refDb.crdt.indexers.get(name);
|
3635
3901
|
}
|
3636
3902
|
var Index = class {
|
3637
3903
|
constructor(sthis, crdt, name, mapFn, meta) {
|
@@ -3650,18 +3916,9 @@ var Index = class {
|
|
3650
3916
|
return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
|
3651
3917
|
});
|
3652
3918
|
}
|
3653
|
-
close() {
|
3654
|
-
return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
|
3655
|
-
});
|
3656
|
-
}
|
3657
|
-
destroy() {
|
3658
|
-
return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
|
3659
|
-
});
|
3660
|
-
}
|
3661
3919
|
applyMapFn(name, mapFn, meta) {
|
3662
3920
|
if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
3663
3921
|
if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
|
3664
|
-
this.name = name;
|
3665
3922
|
try {
|
3666
3923
|
if (meta) {
|
3667
3924
|
if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
|
@@ -3709,9 +3966,13 @@ var Index = class {
|
|
3709
3966
|
}
|
3710
3967
|
}
|
3711
3968
|
async query(opts = {}) {
|
3969
|
+
this.logger.Debug().Msg("enter query");
|
3712
3970
|
await this.ready();
|
3971
|
+
this.logger.Debug().Msg("post ready query");
|
3713
3972
|
await this._updateIndex();
|
3973
|
+
this.logger.Debug().Msg("post _updateIndex query");
|
3714
3974
|
await this._hydrateIndex();
|
3975
|
+
this.logger.Debug().Msg("post _hydrateIndex query");
|
3715
3976
|
if (!this.byKey.root) {
|
3716
3977
|
return await applyQuery(this.crdt, { result: [] }, opts);
|
3717
3978
|
}
|
@@ -3767,13 +4028,16 @@ var Index = class {
|
|
3767
4028
|
}
|
3768
4029
|
async _updateIndex() {
|
3769
4030
|
await this.ready();
|
4031
|
+
this.logger.Debug().Msg("enter _updateIndex");
|
3770
4032
|
if (this.initError) throw this.initError;
|
3771
4033
|
if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
|
3772
4034
|
let result, head;
|
3773
4035
|
if (!this.indexHead || this.indexHead.length === 0) {
|
3774
4036
|
({ result, head } = await this.crdt.allDocs());
|
4037
|
+
this.logger.Debug().Msg("enter crdt.allDocs");
|
3775
4038
|
} else {
|
3776
4039
|
({ result, head } = await this.crdt.changes(this.indexHead));
|
4040
|
+
this.logger.Debug().Msg("enter crdt.changes");
|
3777
4041
|
}
|
3778
4042
|
if (result.length === 0) {
|
3779
4043
|
this.indexHead = head;
|
@@ -3806,9 +4070,22 @@ var Index = class {
|
|
3806
4070
|
if (result.length === 0) {
|
3807
4071
|
return indexerMeta;
|
3808
4072
|
}
|
4073
|
+
this.logger.Debug().Msg("pre this.blockstore.transaction");
|
3809
4074
|
const { meta } = await this.blockstore.transaction(async (tblocks) => {
|
3810
|
-
this.byId = await bulkIndex(
|
3811
|
-
|
4075
|
+
this.byId = await bulkIndex(
|
4076
|
+
this.logger,
|
4077
|
+
tblocks,
|
4078
|
+
this.byId,
|
4079
|
+
removeIdIndexEntries.concat(byIdIndexEntries),
|
4080
|
+
byIdOpts
|
4081
|
+
);
|
4082
|
+
this.byKey = await bulkIndex(
|
4083
|
+
this.logger,
|
4084
|
+
tblocks,
|
4085
|
+
this.byKey,
|
4086
|
+
staleKeyIndexEntries.concat(indexEntries),
|
4087
|
+
byKeyOpts
|
4088
|
+
);
|
3812
4089
|
this.indexHead = head;
|
3813
4090
|
if (this.byId.cid && this.byKey.cid) {
|
3814
4091
|
const idxMeta = {
|
@@ -3820,8 +4097,10 @@ var Index = class {
|
|
3820
4097
|
};
|
3821
4098
|
indexerMeta.indexes?.set(this.name, idxMeta);
|
3822
4099
|
}
|
4100
|
+
this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
|
3823
4101
|
return indexerMeta;
|
3824
4102
|
});
|
4103
|
+
this.logger.Debug().Msg("post this.blockstore.transaction");
|
3825
4104
|
return meta;
|
3826
4105
|
}
|
3827
4106
|
};
|
@@ -3829,7 +4108,8 @@ var Index = class {
|
|
3829
4108
|
// src/crdt-clock.ts
|
3830
4109
|
var import_clock3 = require("@web3-storage/pail/clock");
|
3831
4110
|
var import_crdt2 = require("@web3-storage/pail/crdt");
|
3832
|
-
var
|
4111
|
+
var import_cement16 = require("@adviser/cement");
|
4112
|
+
init_types();
|
3833
4113
|
|
3834
4114
|
// src/apply-head-queue.ts
|
3835
4115
|
function applyHeadQueue(worker, logger) {
|
@@ -3887,7 +4167,7 @@ var CRDTClock = class {
|
|
3887
4167
|
this.zoomers = /* @__PURE__ */ new Set();
|
3888
4168
|
this.watchers = /* @__PURE__ */ new Set();
|
3889
4169
|
this.emptyWatchers = /* @__PURE__ */ new Set();
|
3890
|
-
this._ready = new
|
4170
|
+
this._ready = new import_cement16.ResolveOnce();
|
3891
4171
|
this.blockstore = blockstore;
|
3892
4172
|
this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
|
3893
4173
|
this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
|
@@ -3950,21 +4230,23 @@ var CRDTClock = class {
|
|
3950
4230
|
throw this.logger.Error().Msg("missing blockstore").AsError();
|
3951
4231
|
}
|
3952
4232
|
await validateBlocks(this.logger, newHead, this.blockstore);
|
3953
|
-
|
3954
|
-
|
3955
|
-
|
3956
|
-
|
3957
|
-
|
3958
|
-
|
3959
|
-
|
3960
|
-
|
3961
|
-
|
3962
|
-
|
3963
|
-
|
3964
|
-
|
3965
|
-
|
3966
|
-
|
3967
|
-
|
4233
|
+
if (!this.transaction) {
|
4234
|
+
this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
|
4235
|
+
}
|
4236
|
+
const tblocks = this.transaction;
|
4237
|
+
const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
|
4238
|
+
const result = await (0, import_crdt2.root)(tblocks, advancedHead);
|
4239
|
+
for (const { cid, bytes } of [
|
4240
|
+
...result.additions
|
4241
|
+
// ...result.removals
|
4242
|
+
]) {
|
4243
|
+
tblocks.putSync(cid, bytes);
|
4244
|
+
}
|
4245
|
+
if (!noLoader) {
|
4246
|
+
await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
|
4247
|
+
this.transaction = void 0;
|
4248
|
+
}
|
4249
|
+
this.setHead(advancedHead);
|
3968
4250
|
}
|
3969
4251
|
};
|
3970
4252
|
function sortClockHead(clockHead) {
|
@@ -3997,15 +4279,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
3997
4279
|
// src/crdt.ts
|
3998
4280
|
init_utils();
|
3999
4281
|
var CRDT = class {
|
4000
|
-
constructor(sthis,
|
4282
|
+
constructor(sthis, opts) {
|
4001
4283
|
this.indexers = /* @__PURE__ */ new Map();
|
4002
|
-
this.onceReady = new
|
4284
|
+
this.onceReady = new import_cement17.ResolveOnce();
|
4003
4285
|
this.sthis = sthis;
|
4004
|
-
this.name = name;
|
4005
4286
|
this.logger = ensureLogger(sthis, "CRDT");
|
4006
4287
|
this.opts = opts;
|
4007
4288
|
this.blockstore = blockstoreFactory(sthis, {
|
4008
|
-
name,
|
4009
4289
|
applyMeta: async (meta) => {
|
4010
4290
|
const crdtMeta = meta;
|
4011
4291
|
if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
|
@@ -4015,23 +4295,27 @@ var CRDT = class {
|
|
4015
4295
|
await doCompact(blocks, this.clock.head, this.logger);
|
4016
4296
|
return { head: this.clock.head };
|
4017
4297
|
},
|
4018
|
-
autoCompact: this.opts.autoCompact || 100,
|
4019
|
-
|
4020
|
-
|
4021
|
-
|
4022
|
-
|
4298
|
+
// autoCompact: this.opts.autoCompact || 100,
|
4299
|
+
storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
|
4300
|
+
storeUrls: this.opts.storeUrls.data,
|
4301
|
+
keyBag: this.opts.keyBag,
|
4302
|
+
// public: this.opts.public,
|
4303
|
+
meta: this.opts.meta
|
4304
|
+
// threshold: this.opts.threshold,
|
4023
4305
|
});
|
4024
4306
|
this.indexBlockstore = blockstoreFactory(sthis, {
|
4025
|
-
name,
|
4307
|
+
// name: opts.name,
|
4026
4308
|
applyMeta: async (meta) => {
|
4027
4309
|
const idxCarMeta = meta;
|
4028
4310
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
4029
|
-
for (const [
|
4030
|
-
index(
|
4311
|
+
for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
|
4312
|
+
index({ crdt: this }, name, void 0, idx);
|
4031
4313
|
}
|
4032
4314
|
},
|
4033
|
-
|
4034
|
-
|
4315
|
+
storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
|
4316
|
+
storeUrls: this.opts.storeUrls.idx,
|
4317
|
+
keyBag: this.opts.keyBag
|
4318
|
+
// public: this.opts.public,
|
4035
4319
|
});
|
4036
4320
|
this.clock = new CRDTClock(this.blockstore);
|
4037
4321
|
this.clock.onZoom(() => {
|
@@ -4113,52 +4397,166 @@ var CRDT = class {
|
|
4113
4397
|
};
|
4114
4398
|
|
4115
4399
|
// src/database.ts
|
4400
|
+
init_types();
|
4116
4401
|
init_utils();
|
4117
|
-
var
|
4118
|
-
|
4119
|
-
|
4402
|
+
var databases = new import_cement18.KeyedResolvOnce();
|
4403
|
+
function toSortedArray(set) {
|
4404
|
+
if (!set) return [];
|
4405
|
+
return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
|
4406
|
+
}
|
4407
|
+
function keyConfigOpts(sthis, name, opts) {
|
4408
|
+
return JSON.stringify(
|
4409
|
+
toSortedArray({
|
4410
|
+
name,
|
4411
|
+
stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
|
4412
|
+
})
|
4413
|
+
);
|
4414
|
+
}
|
4415
|
+
function isDatabase(db) {
|
4416
|
+
return db instanceof DatabaseImpl || db instanceof DatabaseShell;
|
4417
|
+
}
|
4418
|
+
function DatabaseFactory(name, opts) {
|
4419
|
+
const sthis = ensureSuperThis(opts);
|
4420
|
+
return new DatabaseShell(
|
4421
|
+
databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
|
4422
|
+
const db = new DatabaseImpl(sthis, {
|
4423
|
+
name,
|
4424
|
+
meta: opts?.meta,
|
4425
|
+
keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
|
4426
|
+
storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
|
4427
|
+
storeEnDe: {
|
4428
|
+
encodeFile,
|
4429
|
+
decodeFile,
|
4430
|
+
...opts?.storeEnDe
|
4431
|
+
}
|
4432
|
+
});
|
4433
|
+
db.onClosed(() => {
|
4434
|
+
databases.unget(key);
|
4435
|
+
});
|
4436
|
+
return db;
|
4437
|
+
})
|
4438
|
+
);
|
4439
|
+
}
|
4440
|
+
var DatabaseShell = class {
|
4441
|
+
constructor(ref) {
|
4442
|
+
this.ref = ref;
|
4443
|
+
ref.addShell(this);
|
4444
|
+
}
|
4445
|
+
get id() {
|
4446
|
+
return this.ref.id;
|
4447
|
+
}
|
4448
|
+
get logger() {
|
4449
|
+
return this.ref.logger;
|
4450
|
+
}
|
4451
|
+
get sthis() {
|
4452
|
+
return this.ref.sthis;
|
4453
|
+
}
|
4454
|
+
get crdt() {
|
4455
|
+
return this.ref.crdt;
|
4456
|
+
}
|
4457
|
+
name() {
|
4458
|
+
return this.ref.name();
|
4459
|
+
}
|
4460
|
+
onClosed(fn) {
|
4461
|
+
return this.ref.onClosed(fn);
|
4462
|
+
}
|
4463
|
+
close() {
|
4464
|
+
return this.ref.shellClose(this);
|
4465
|
+
}
|
4466
|
+
destroy() {
|
4467
|
+
return this.ref.destroy();
|
4468
|
+
}
|
4469
|
+
ready() {
|
4470
|
+
return this.ref.ready();
|
4471
|
+
}
|
4472
|
+
get(id) {
|
4473
|
+
return this.ref.get(id);
|
4474
|
+
}
|
4475
|
+
put(doc) {
|
4476
|
+
return this.ref.put(doc);
|
4477
|
+
}
|
4478
|
+
del(id) {
|
4479
|
+
return this.ref.del(id);
|
4480
|
+
}
|
4481
|
+
changes(since, opts) {
|
4482
|
+
return this.ref.changes(since, opts);
|
4483
|
+
}
|
4484
|
+
allDocs(opts) {
|
4485
|
+
return this.ref.allDocs(opts);
|
4486
|
+
}
|
4487
|
+
allDocuments() {
|
4488
|
+
return this.ref.allDocuments();
|
4489
|
+
}
|
4490
|
+
subscribe(listener, updates) {
|
4491
|
+
return this.ref.subscribe(listener, updates);
|
4492
|
+
}
|
4493
|
+
query(field, opts) {
|
4494
|
+
return this.ref.query(field, opts);
|
4495
|
+
}
|
4496
|
+
compact() {
|
4497
|
+
return this.ref.compact();
|
4498
|
+
}
|
4499
|
+
};
|
4500
|
+
var DatabaseImpl = class {
|
4501
|
+
constructor(sthis, opts) {
|
4120
4502
|
this._listening = false;
|
4121
4503
|
this._listeners = /* @__PURE__ */ new Set();
|
4122
4504
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
4123
|
-
|
4124
|
-
this.
|
4125
|
-
this.
|
4126
|
-
this.
|
4505
|
+
// readonly blockstore: BaseBlockstore;
|
4506
|
+
this.shells = /* @__PURE__ */ new Set();
|
4507
|
+
this._onClosedFns = /* @__PURE__ */ new Set();
|
4508
|
+
this._ready = new import_cement18.ResolveOnce();
|
4509
|
+
this.opts = opts;
|
4510
|
+
this.sthis = sthis;
|
4511
|
+
this.id = sthis.timeOrderedNextId().str;
|
4127
4512
|
this.logger = ensureLogger(this.sthis, "Database");
|
4128
|
-
this.
|
4129
|
-
this.blockstore = this._crdt.blockstore;
|
4513
|
+
this.crdt = new CRDT(this.sthis, this.opts);
|
4130
4514
|
this._writeQueue = writeQueue(async (updates) => {
|
4131
|
-
return await this.
|
4515
|
+
return await this.crdt.bulk(updates);
|
4132
4516
|
});
|
4133
|
-
this.
|
4517
|
+
this.crdt.clock.onTock(() => {
|
4134
4518
|
this._no_update_notify();
|
4135
4519
|
});
|
4136
4520
|
}
|
4137
|
-
|
4138
|
-
this.
|
4521
|
+
addShell(shell) {
|
4522
|
+
this.shells.add(shell);
|
4523
|
+
}
|
4524
|
+
onClosed(fn) {
|
4525
|
+
this._onClosedFns.add(fn);
|
4139
4526
|
}
|
4140
4527
|
async close() {
|
4141
|
-
|
4142
|
-
|
4143
|
-
|
4528
|
+
throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
|
4529
|
+
}
|
4530
|
+
async shellClose(db) {
|
4531
|
+
if (!this.shells.has(db)) {
|
4532
|
+
throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
|
4533
|
+
}
|
4534
|
+
this.shells.delete(db);
|
4535
|
+
if (this.shells.size === 0) {
|
4536
|
+
await this.ready();
|
4537
|
+
await this.crdt.close();
|
4538
|
+
this._onClosedFns.forEach((fn) => fn());
|
4539
|
+
}
|
4144
4540
|
}
|
4145
4541
|
async destroy() {
|
4146
4542
|
await this.ready();
|
4147
|
-
await this.
|
4148
|
-
await this.blockstore.destroy();
|
4543
|
+
await this.crdt.destroy();
|
4149
4544
|
}
|
4150
4545
|
async ready() {
|
4151
|
-
|
4546
|
+
const ret = await this._ready.once(async () => {
|
4152
4547
|
await this.sthis.start();
|
4153
|
-
await this.
|
4154
|
-
await this.blockstore.ready();
|
4548
|
+
await this.crdt.ready();
|
4155
4549
|
});
|
4550
|
+
return ret;
|
4551
|
+
}
|
4552
|
+
name() {
|
4553
|
+
return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
|
4156
4554
|
}
|
4157
4555
|
async get(id) {
|
4158
|
-
if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
|
4556
|
+
if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
|
4159
4557
|
await this.ready();
|
4160
4558
|
this.logger.Debug().Str("id", id).Msg("get");
|
4161
|
-
const got = await this.
|
4559
|
+
const got = await this.crdt.get(id).catch((e) => {
|
4162
4560
|
throw new NotFoundError(`Not found: ${id} - ${e.message}`);
|
4163
4561
|
});
|
4164
4562
|
if (!got) throw new NotFoundError(`Not found: ${id}`);
|
@@ -4177,34 +4575,34 @@ var Database = class {
|
|
4177
4575
|
_id: docId
|
4178
4576
|
}
|
4179
4577
|
});
|
4180
|
-
return { id: docId, clock: result?.head, name: this.name };
|
4578
|
+
return { id: docId, clock: result?.head, name: this.name() };
|
4181
4579
|
}
|
4182
4580
|
async del(id) {
|
4183
4581
|
await this.ready();
|
4184
4582
|
this.logger.Debug().Str("id", id).Msg("del");
|
4185
4583
|
const result = await this._writeQueue.push({ id, del: true });
|
4186
|
-
return { id, clock: result?.head, name: this.name };
|
4584
|
+
return { id, clock: result?.head, name: this.name() };
|
4187
4585
|
}
|
4188
4586
|
async changes(since = [], opts = {}) {
|
4189
4587
|
await this.ready();
|
4190
4588
|
this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
|
4191
|
-
const { result, head } = await this.
|
4589
|
+
const { result, head } = await this.crdt.changes(since, opts);
|
4192
4590
|
const rows = result.map(({ id: key, value, del, clock }) => ({
|
4193
4591
|
key,
|
4194
4592
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
|
4195
4593
|
clock
|
4196
4594
|
}));
|
4197
|
-
return { rows, clock: head, name: this.name };
|
4595
|
+
return { rows, clock: head, name: this.name() };
|
4198
4596
|
}
|
4199
4597
|
async allDocs(opts = {}) {
|
4200
4598
|
await this.ready();
|
4201
4599
|
this.logger.Debug().Msg("allDocs");
|
4202
|
-
const { result, head } = await this.
|
4600
|
+
const { result, head } = await this.crdt.allDocs();
|
4203
4601
|
const rows = result.map(({ id: key, value, del }) => ({
|
4204
4602
|
key,
|
4205
4603
|
value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
|
4206
4604
|
}));
|
4207
|
-
return { rows, clock: head, name: this.name };
|
4605
|
+
return { rows, clock: head, name: this.name() };
|
4208
4606
|
}
|
4209
4607
|
async allDocuments() {
|
4210
4608
|
return this.allDocs();
|
@@ -4214,7 +4612,7 @@ var Database = class {
|
|
4214
4612
|
if (updates) {
|
4215
4613
|
if (!this._listening) {
|
4216
4614
|
this._listening = true;
|
4217
|
-
this.
|
4615
|
+
this.crdt.clock.onTick((updates2) => {
|
4218
4616
|
void this._notify(updates2);
|
4219
4617
|
});
|
4220
4618
|
}
|
@@ -4233,13 +4631,13 @@ var Database = class {
|
|
4233
4631
|
async query(field, opts = {}) {
|
4234
4632
|
await this.ready();
|
4235
4633
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
4236
|
-
const _crdt = this.
|
4237
|
-
const idx = typeof field === "string" ? index(
|
4634
|
+
const _crdt = this.crdt;
|
4635
|
+
const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
|
4238
4636
|
return await idx.query(opts);
|
4239
4637
|
}
|
4240
4638
|
async compact() {
|
4241
4639
|
await this.ready();
|
4242
|
-
await this.
|
4640
|
+
await this.crdt.compact();
|
4243
4641
|
}
|
4244
4642
|
async _notify(updates) {
|
4245
4643
|
await this.ready();
|
@@ -4263,23 +4661,62 @@ var Database = class {
|
|
4263
4661
|
}
|
4264
4662
|
}
|
4265
4663
|
};
|
4266
|
-
function
|
4267
|
-
|
4268
|
-
|
4664
|
+
function defaultURI(sthis, curi, uri, store, ctx) {
|
4665
|
+
ctx = ctx || {};
|
4666
|
+
const ret = (curi ? import_cement18.URI.from(curi) : uri).build().setParam("store" /* STORE */, store);
|
4667
|
+
if (!ret.hasParam("name" /* NAME */)) {
|
4668
|
+
const name = sthis.pathOps.basename(ret.URI().pathname);
|
4669
|
+
if (!name) {
|
4670
|
+
throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
|
4671
|
+
}
|
4672
|
+
ret.setParam("name" /* NAME */, name);
|
4673
|
+
}
|
4674
|
+
if (ctx.idx) {
|
4675
|
+
ret.defParam("index" /* INDEX */, "idx");
|
4676
|
+
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
|
4677
|
+
} else {
|
4678
|
+
ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
|
4679
|
+
}
|
4680
|
+
if (store === "data") {
|
4681
|
+
if (ctx.file) {
|
4682
|
+
} else {
|
4683
|
+
ret.defParam("suffix" /* SUFFIX */, ".car");
|
4684
|
+
}
|
4685
|
+
}
|
4686
|
+
return ret.URI();
|
4269
4687
|
}
|
4270
|
-
function
|
4271
|
-
|
4272
|
-
|
4273
|
-
|
4274
|
-
|
4275
|
-
|
4276
|
-
|
4277
|
-
|
4278
|
-
|
4279
|
-
|
4280
|
-
|
4688
|
+
function toStoreURIRuntime(sthis, name, sopts) {
|
4689
|
+
sopts = sopts || {};
|
4690
|
+
if (!sopts.base) {
|
4691
|
+
const fp_env = sthis.env.get("FP_STORAGE_URL");
|
4692
|
+
if (fp_env) {
|
4693
|
+
sopts = { ...sopts, base: import_cement18.BuildURI.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
|
4694
|
+
} else {
|
4695
|
+
sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
|
4696
|
+
}
|
4697
|
+
}
|
4698
|
+
const bbase = import_cement18.BuildURI.from(sopts.base);
|
4699
|
+
if (name) {
|
4700
|
+
bbase.setParam("name" /* NAME */, name);
|
4281
4701
|
}
|
4282
|
-
|
4702
|
+
const base = bbase.URI();
|
4703
|
+
return {
|
4704
|
+
idx: {
|
4705
|
+
data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
|
4706
|
+
file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
|
4707
|
+
meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
|
4708
|
+
wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
|
4709
|
+
},
|
4710
|
+
data: {
|
4711
|
+
data: defaultURI(sthis, sopts.data?.data, base, "data"),
|
4712
|
+
file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
|
4713
|
+
meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
|
4714
|
+
wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
|
4715
|
+
}
|
4716
|
+
};
|
4717
|
+
}
|
4718
|
+
function fireproof(name, opts) {
|
4719
|
+
return DatabaseFactory(name, opts);
|
4283
4720
|
}
|
4284
4721
|
function makeName(fnString) {
|
4285
4722
|
const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
|
@@ -4298,6 +4735,9 @@ function makeName(fnString) {
|
|
4298
4735
|
}
|
4299
4736
|
}
|
4300
4737
|
|
4738
|
+
// src/index.ts
|
4739
|
+
init_types();
|
4740
|
+
|
4301
4741
|
// src/runtime/index.ts
|
4302
4742
|
var runtime_exports = {};
|
4303
4743
|
__export(runtime_exports, {
|
@@ -4310,7 +4750,7 @@ __export(runtime_exports, {
|
|
4310
4750
|
kb: () => key_bag_exports,
|
4311
4751
|
kc: () => keyed_crypto_exports,
|
4312
4752
|
mf: () => wait_pr_multiformats_exports,
|
4313
|
-
runtimeFn: () =>
|
4753
|
+
runtimeFn: () => import_cement19.runtimeFn,
|
4314
4754
|
toArrayBuffer: () => toArrayBuffer
|
4315
4755
|
});
|
4316
4756
|
init_utils2();
|
@@ -4326,7 +4766,7 @@ __export(wait_pr_multiformats_exports, {
|
|
4326
4766
|
var codec_interface_exports = {};
|
4327
4767
|
|
4328
4768
|
// src/runtime/index.ts
|
4329
|
-
var
|
4769
|
+
var import_cement19 = require("@adviser/cement");
|
4330
4770
|
init_version();
|
4331
4771
|
init_version2();
|
4332
4772
|
|
@@ -4335,6 +4775,6 @@ init_utils();
|
|
4335
4775
|
|
4336
4776
|
// src/version.ts
|
4337
4777
|
var PACKAGE_VERSION = Object.keys({
|
4338
|
-
"0.19.
|
4778
|
+
"0.19.101": "xxxx"
|
4339
4779
|
})[0];
|
4340
4780
|
//# sourceMappingURL=index.cjs.map
|