@fireproof/core 0.19.100 → 0.19.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
  2. package/chunk-3EB3ENHT.js.map +1 -0
  3. package/chunk-HQ7D3PEU.js +61 -0
  4. package/chunk-HQ7D3PEU.js.map +1 -0
  5. package/chunk-PZ5AY32C.js +10 -0
  6. package/deno-filesystem-Q2IJ7YDR.js +57 -0
  7. package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
  8. package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
  9. package/gateway-GK5QZ6KP.js.map +1 -0
  10. package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
  11. package/gateway-TQTGDRCN.js.map +1 -0
  12. package/index.cjs +2158 -1718
  13. package/index.cjs.map +1 -1
  14. package/index.d.cts +261 -117
  15. package/index.d.ts +261 -117
  16. package/index.global.js +11834 -11354
  17. package/index.global.js.map +1 -1
  18. package/index.js +1865 -1519
  19. package/index.js.map +1 -1
  20. package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
  21. package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
  22. package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
  23. package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
  24. package/key-bag-memory-LWE6ARPX.js +29 -0
  25. package/key-bag-memory-LWE6ARPX.js.map +1 -0
  26. package/metafile-cjs.json +1 -1
  27. package/metafile-esm.json +1 -1
  28. package/metafile-iife.json +1 -1
  29. package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
  30. package/node-filesystem-CFRXFSO7.js.map +1 -0
  31. package/package.json +1 -1
  32. package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
  33. package/tests/blockstore/keyed-crypto.test.ts +63 -227
  34. package/tests/blockstore/loader.test.ts +19 -11
  35. package/tests/blockstore/store.test.ts +23 -19
  36. package/tests/blockstore/transaction.test.ts +12 -12
  37. package/tests/fireproof/all-gateway.test.ts +201 -193
  38. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
  39. package/tests/fireproof/crdt.test.ts +67 -16
  40. package/tests/fireproof/database.test.ts +183 -21
  41. package/tests/fireproof/fireproof.test.ts +83 -74
  42. package/tests/fireproof/hello.test.ts +18 -14
  43. package/tests/fireproof/indexer.test.ts +53 -43
  44. package/tests/fireproof/utils.test.ts +18 -6
  45. package/tests/gateway/file/loader-config.test.ts +303 -0
  46. package/tests/gateway/indexdb/loader-config.test.ts +75 -0
  47. package/tests/helpers.ts +27 -9
  48. package/tests/react/useFireproof.test.tsx +1 -1
  49. package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
  50. package/utils-STA2C35G.js.map +1 -0
  51. package/chunk-OFGPKRCM.js.map +0 -1
  52. package/chunk-WS3YRPIA.js +0 -75
  53. package/chunk-WS3YRPIA.js.map +0 -1
  54. package/gateway-5FCWPX5W.js.map +0 -1
  55. package/gateway-H7UD6TNB.js.map +0 -1
  56. package/mem-filesystem-YPPJV7Q2.js +0 -41
  57. package/mem-filesystem-YPPJV7Q2.js.map +0 -1
  58. package/node-filesystem-INX4ZTHE.js.map +0 -1
  59. package/tests/fireproof/config.test.ts +0 -172
  60. /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
  61. /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.cjs CHANGED
@@ -30,6 +30,47 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
30
30
  ));
31
31
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
32
32
 
33
+ // src/types.ts
34
+ function isFalsy(value) {
35
+ return value === false && value === null && value === void 0;
36
+ }
37
+ function throwFalsy(value) {
38
+ if (isFalsy(value)) {
39
+ throw new Error("value is Falsy");
40
+ }
41
+ return value;
42
+ }
43
+ function falsyToUndef(value) {
44
+ if (isFalsy(value)) {
45
+ return void 0;
46
+ }
47
+ return value;
48
+ }
49
+ var PARAM;
50
+ var init_types = __esm({
51
+ "src/types.ts"() {
52
+ "use strict";
53
+ PARAM = /* @__PURE__ */ ((PARAM2) => {
54
+ PARAM2["SUFFIX"] = "suffix";
55
+ PARAM2["URL_GEN"] = "urlGen";
56
+ PARAM2["STORE_KEY"] = "storekey";
57
+ PARAM2["STORE"] = "store";
58
+ PARAM2["KEY"] = "key";
59
+ PARAM2["INDEX"] = "index";
60
+ PARAM2["NAME"] = "name";
61
+ PARAM2["VERSION"] = "version";
62
+ PARAM2["FRAG_SIZE"] = "fragSize";
63
+ PARAM2["IV_VERIFY"] = "ivVerify";
64
+ PARAM2["IV_HASH"] = "ivHash";
65
+ PARAM2["FRAG_FID"] = "fid";
66
+ PARAM2["FRAG_OFS"] = "ofs";
67
+ PARAM2["FRAG_LEN"] = "len";
68
+ PARAM2["FRAG_HEAD"] = "headerSize";
69
+ return PARAM2;
70
+ })(PARAM || {});
71
+ }
72
+ });
73
+
33
74
  // src/utils.ts
34
75
  function presetEnv() {
35
76
  const penv = new Map([
@@ -87,6 +128,10 @@ function ensureLogger(sthis, componentName, ctx) {
87
128
  exposeStack = true;
88
129
  delete ctx.exposeStack;
89
130
  }
131
+ if ("exposeStack" in ctx) {
132
+ exposeStack = true;
133
+ delete ctx.exposeStack;
134
+ }
90
135
  if ("this" in ctx) {
91
136
  cLogger.Str("this", sthis.nextId(4).str);
92
137
  delete ctx.this;
@@ -153,10 +198,13 @@ function ensureLogger(sthis, componentName, ctx) {
153
198
  logger.SetExposeStack(true);
154
199
  }
155
200
  const out = cLogger.Logger();
201
+ if (sthis.env.get("FP_CONSTRUCTOR_DEBUG")) {
202
+ out.Debug().Msg("constructor");
203
+ }
156
204
  return out;
157
205
  }
158
206
  function getStore(url, sthis, joiner) {
159
- const store = url.getParam("store");
207
+ const store = url.getParam("store" /* STORE */);
160
208
  switch (store) {
161
209
  case "data":
162
210
  case "wal":
@@ -164,20 +212,21 @@ function getStore(url, sthis, joiner) {
164
212
  break;
165
213
  default:
166
214
  throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
215
+ throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
167
216
  }
168
217
  let name = store;
169
218
  if (url.hasParam("index")) {
170
- name = joiner(url.getParam("index") || "idx", name);
219
+ name = joiner(url.getParam("index" /* INDEX */) || "idx", name);
171
220
  }
172
221
  return { store, name };
173
222
  }
174
223
  function getKey(url, logger) {
175
- const result = url.getParam("key");
224
+ const result = url.getParam("key" /* KEY */);
176
225
  if (!result) throw logger.Error().Str("url", url.toString()).Msg(`key not found`).AsError();
177
226
  return result;
178
227
  }
179
228
  function getName(sthis, url) {
180
- let result = url.getParam("name");
229
+ let result = url.getParam("name" /* NAME */);
181
230
  if (!result) {
182
231
  result = sthis.pathOps.dirname(url.pathname);
183
232
  if (result.length === 0) {
@@ -197,17 +246,6 @@ function isNotFoundError(e) {
197
246
  if (e.code === "ENOENT") return true;
198
247
  return false;
199
248
  }
200
- function dataDir(sthis, name, base) {
201
- if (!base) {
202
- if (!(0, import_cement.runtimeFn)().isBrowser) {
203
- const home = sthis.env.get("HOME") || "./";
204
- base = sthis.env.get("FP_STORAGE_URL") || `file://${sthis.pathOps.join(home, ".fireproof")}`;
205
- } else {
206
- base = sthis.env.get("FP_STORAGE_URL") || `indexdb://fp`;
207
- }
208
- }
209
- return import_cement.URI.from(base.toString()).build().setParam("name", name || "").URI();
210
- }
211
249
  function UInt8ArrayEqual(a, b) {
212
250
  if (a.length !== b.length) {
213
251
  return false;
@@ -224,6 +262,7 @@ var init_utils = __esm({
224
262
  "src/utils.ts"() {
225
263
  "use strict";
226
264
  import_cement = require("@adviser/cement");
265
+ init_types();
227
266
  import_base58 = require("multiformats/bases/base58");
228
267
  globalLogger = new import_cement.LoggerImpl();
229
268
  registerFP_DEBUG = new import_cement.ResolveOnce();
@@ -274,6 +313,9 @@ var init_utils = __esm({
274
313
  dirname(path) {
275
314
  return path.split("/").slice(0, -1).join("/");
276
315
  }
316
+ basename(path) {
317
+ return path.split("/").pop() || "";
318
+ }
277
319
  // homedir() {
278
320
  // throw new Error("SysContainer:homedir is not available in seeded state");
279
321
  // }
@@ -292,45 +334,63 @@ var init_utils = __esm({
292
334
  }
293
335
  });
294
336
 
295
- // src/runtime/gateways/file/mem-filesystem.ts
296
- var mem_filesystem_exports = {};
297
- __export(mem_filesystem_exports, {
298
- MemFileSystem: () => MemFileSystem
337
+ // src/runtime/gateways/file/deno-filesystem.ts
338
+ var deno_filesystem_exports = {};
339
+ __export(deno_filesystem_exports, {
340
+ DenoFileSystem: () => DenoFileSystem
299
341
  });
300
- var import_memfs, MemFileSystem;
301
- var init_mem_filesystem = __esm({
302
- "src/runtime/gateways/file/mem-filesystem.ts"() {
342
+ var DenoFileSystem;
343
+ var init_deno_filesystem = __esm({
344
+ "src/runtime/gateways/file/deno-filesystem.ts"() {
303
345
  "use strict";
304
- import_memfs = require("memfs");
305
- init_utils2();
306
- MemFileSystem = class {
346
+ DenoFileSystem = class {
307
347
  async start() {
348
+ this.fs = Deno;
308
349
  return this;
309
350
  }
310
- mkdir(path, options) {
311
- return import_memfs.fs.promises.mkdir(path, options);
351
+ async mkdir(path, options) {
352
+ return this.fs?.mkdir(path, options).then(() => path);
312
353
  }
313
- readdir(path, options) {
314
- return import_memfs.fs.promises.readdir(path, options);
354
+ async readdir(path) {
355
+ const ret = [];
356
+ for await (const dirEntry of this.fs.readdir(path)) {
357
+ ret.push(dirEntry.name);
358
+ }
359
+ return ret;
315
360
  }
316
- rm(path, options) {
317
- return import_memfs.fs.promises.rm(path, options);
361
+ async rm(path, options) {
362
+ return this.fs?.rm(path, options);
318
363
  }
319
- copyFile(source, destination) {
320
- return import_memfs.fs.promises.copyFile(source, destination);
364
+ async copyFile(source, destination) {
365
+ return this.fs?.copyFile(source, destination);
321
366
  }
322
- async readfile(path, options) {
323
- const ret = await import_memfs.fs.promises.readFile(path, options);
324
- return toArrayBuffer(ret);
367
+ async readfile(path) {
368
+ return this.fs.readFile(path);
325
369
  }
326
- stat(path) {
327
- return import_memfs.fs.promises.stat(path);
370
+ async stat(path) {
371
+ const x = await this.fs.stat(path);
372
+ return {
373
+ isFile: () => x.isFile,
374
+ isDirectory: () => x.isDirectory,
375
+ isBlockDevice: () => !!x.isBlockDevice,
376
+ isCharacterDevice: () => !!x.isCharDevice,
377
+ isSymbolicLink: () => !!x.isSymlink,
378
+ isFIFO: () => !!x.isFifo,
379
+ isSocket: () => !!x.isSocket,
380
+ uid: x.uid,
381
+ gid: x.gid,
382
+ size: x.size,
383
+ atime: x.atime,
384
+ mtime: x.mtime,
385
+ ctime: x.birthtime,
386
+ birthtime: x.birthtime
387
+ };
328
388
  }
329
- unlink(path) {
330
- return import_memfs.fs.promises.unlink(path);
389
+ async unlink(path) {
390
+ return this.fs?.unlink(path);
331
391
  }
332
- writefile(path, data) {
333
- return import_memfs.fs.promises.writeFile(path, Buffer.from(data));
392
+ async writefile(path, data) {
393
+ return this.fs?.writeFile(path, Buffer.from(data));
334
394
  }
335
395
  };
336
396
  }
@@ -341,12 +401,11 @@ var node_filesystem_exports = {};
341
401
  __export(node_filesystem_exports, {
342
402
  NodeFileSystem: () => NodeFileSystem
343
403
  });
344
- var import_cement4, NodeFileSystem;
404
+ var NodeFileSystem;
345
405
  var init_node_filesystem = __esm({
346
406
  "src/runtime/gateways/file/node-filesystem.ts"() {
347
407
  "use strict";
348
408
  init_utils2();
349
- import_cement4 = require("@adviser/cement");
350
409
  NodeFileSystem = class {
351
410
  async start() {
352
411
  this.fs = await import("fs/promises");
@@ -369,15 +428,12 @@ var init_node_filesystem = __esm({
369
428
  return toArrayBuffer(ret);
370
429
  }
371
430
  stat(path) {
372
- return this.fs?.stat(path);
431
+ return this.fs.stat(path);
373
432
  }
374
433
  async unlink(path) {
375
434
  return this.fs?.unlink(path);
376
435
  }
377
436
  async writefile(path, data) {
378
- if ((0, import_cement4.runtimeFn)().isDeno) {
379
- return this.fs?.writeFile(path, data);
380
- }
381
437
  return this.fs?.writeFile(path, Buffer.from(data));
382
438
  }
383
439
  };
@@ -393,49 +449,34 @@ __export(utils_exports, {
393
449
  toArrayBuffer: () => toArrayBuffer
394
450
  });
395
451
  async function getFileSystem(url) {
396
- const name = url.getParam("fs");
397
- let fs2;
398
- switch (name) {
399
- case "mem":
400
- {
401
- const { MemFileSystem: MemFileSystem2 } = await Promise.resolve().then(() => (init_mem_filesystem(), mem_filesystem_exports));
402
- fs2 = new MemFileSystem2();
403
- }
404
- break;
405
- // case 'deno': {
406
- // const { DenoFileSystem } = await import("./deno-filesystem.js");
407
- // fs = new DenoFileSystem();
408
- // break;
409
- // }
410
- case "node": {
411
- const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
412
- fs2 = new NodeFileSystem2();
413
- break;
414
- }
415
- case "sys":
416
- default: {
417
- return getFileSystem(url.build().setParam("fs", "node").URI());
418
- }
452
+ let fs;
453
+ if ((0, import_cement4.runtimeFn)().isDeno) {
454
+ const { DenoFileSystem: DenoFileSystem2 } = await Promise.resolve().then(() => (init_deno_filesystem(), deno_filesystem_exports));
455
+ fs = new DenoFileSystem2();
456
+ } else if ((0, import_cement4.runtimeFn)().isNodeIsh) {
457
+ const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
458
+ fs = new NodeFileSystem2();
459
+ } else {
460
+ throw new Error("unsupported runtime");
419
461
  }
420
- return fs2.start();
462
+ return fs.start();
421
463
  }
422
464
  function getPath(url, sthis) {
423
465
  const basePath = url.pathname;
424
- const name = url.getParam("name");
466
+ const name = url.getParam("name" /* NAME */);
425
467
  if (name) {
426
- const version = url.getParam("version");
427
- if (!version) throw sthis.logger.Error().Url(url).Msg(`version not found`).AsError();
428
- return sthis.pathOps.join(basePath, version, name);
468
+ return sthis.pathOps.join(basePath, name);
429
469
  }
430
470
  return sthis.pathOps.join(basePath);
431
471
  }
432
472
  function getFileName(url, sthis) {
433
- const key = url.getParam("key");
473
+ const key = url.getParam("key" /* KEY */);
434
474
  if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
435
475
  const res = getStore(url, sthis, (...a) => a.join("-"));
436
476
  switch (res.store) {
437
- case "data":
438
- return sthis.pathOps.join(res.name, key + ".car");
477
+ case "data": {
478
+ return sthis.pathOps.join(res.name, key + (url.getParam("suffix" /* SUFFIX */) || ""));
479
+ }
439
480
  case "wal":
440
481
  case "meta":
441
482
  return sthis.pathOps.join(res.name, key + ".json");
@@ -454,10 +495,13 @@ function toArrayBuffer(buffer) {
454
495
  }
455
496
  return view;
456
497
  }
498
+ var import_cement4;
457
499
  var init_utils2 = __esm({
458
500
  "src/runtime/gateways/file/utils.ts"() {
459
501
  "use strict";
502
+ import_cement4 = require("@adviser/cement");
460
503
  init_utils();
504
+ init_types();
461
505
  }
462
506
  });
463
507
 
@@ -572,6 +616,40 @@ var init_key_bag_indexdb = __esm({
572
616
  }
573
617
  });
574
618
 
619
+ // src/runtime/key-bag-memory.ts
620
+ var key_bag_memory_exports = {};
621
+ __export(key_bag_memory_exports, {
622
+ KeyBagProviderMemory: () => KeyBagProviderMemory
623
+ });
624
+ var memoryKeyBag, KeyBagProviderMemory;
625
+ var init_key_bag_memory = __esm({
626
+ "src/runtime/key-bag-memory.ts"() {
627
+ "use strict";
628
+ memoryKeyBag = /* @__PURE__ */ new Map();
629
+ KeyBagProviderMemory = class {
630
+ constructor(url, sthis) {
631
+ this.url = url;
632
+ this.sthis = sthis;
633
+ }
634
+ key(id) {
635
+ return `${this.url.pathname}/${id}`;
636
+ }
637
+ async get(id) {
638
+ const binKeyItem = memoryKeyBag.get(this.key(id));
639
+ if (binKeyItem) {
640
+ const ki = JSON.parse(this.sthis.txt.decode(binKeyItem));
641
+ return ki;
642
+ }
643
+ return void 0;
644
+ }
645
+ async set(id, item) {
646
+ const p = this.sthis.txt.encode(JSON.stringify(item, null, 2));
647
+ memoryKeyBag.set(this.key(id), p);
648
+ }
649
+ };
650
+ }
651
+ });
652
+
575
653
  // src/runtime/gateways/file/version.ts
576
654
  var FILESTORE_VERSION;
577
655
  var init_version = __esm({
@@ -587,23 +665,24 @@ __export(gateway_exports, {
587
665
  FileGateway: () => FileGateway,
588
666
  FileTestStore: () => FileTestStore
589
667
  });
590
- var import_cement11, versionFiles, FileGateway, FileTestStore;
668
+ var import_cement12, versionFiles, FileGateway, FileTestStore;
591
669
  var init_gateway = __esm({
592
670
  "src/runtime/gateways/file/gateway.ts"() {
593
671
  "use strict";
594
672
  init_version();
595
- import_cement11 = require("@adviser/cement");
673
+ import_cement12 = require("@adviser/cement");
596
674
  init_utils();
597
675
  init_utils2();
598
- versionFiles = new import_cement11.KeyedResolvOnce();
676
+ init_types();
677
+ versionFiles = new import_cement12.KeyedResolvOnce();
599
678
  FileGateway = class {
600
679
  get fs() {
601
680
  if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
602
681
  return this._fs;
603
682
  }
604
683
  constructor(sthis) {
605
- this.sthis = sthis;
606
- this.logger = sthis.logger;
684
+ this.sthis = ensureSuperLog(sthis, "FileGateway", { this: 1 });
685
+ this.logger = this.sthis.logger;
607
686
  }
608
687
  async getVersionFromFile(path, logger) {
609
688
  return versionFiles.get(path).once(async () => {
@@ -625,34 +704,33 @@ var init_gateway = __esm({
625
704
  });
626
705
  }
627
706
  start(baseURL) {
628
- return (0, import_cement11.exception2Result)(async () => {
707
+ return (0, import_cement12.exception2Result)(async () => {
629
708
  this._fs = await getFileSystem(baseURL);
630
- await this.fs.start();
631
709
  const url = baseURL.build();
632
- url.defParam("version", FILESTORE_VERSION);
710
+ url.defParam("version" /* VERSION */, FILESTORE_VERSION);
633
711
  const dbUrl = await this.buildUrl(url.URI(), "dummy");
634
712
  const dbdirFile = this.getFilePath(dbUrl.Ok());
635
713
  await this.fs.mkdir(this.sthis.pathOps.dirname(dbdirFile), { recursive: true });
636
714
  const dbroot = this.sthis.pathOps.dirname(dbdirFile);
637
715
  this.logger.Debug().Url(url.URI()).Str("dbroot", dbroot).Msg("start");
638
- url.setParam("version", await this.getVersionFromFile(dbroot, this.logger));
716
+ url.setParam("version" /* VERSION */, await this.getVersionFromFile(dbroot, this.logger));
639
717
  return url.URI();
640
718
  });
641
719
  }
642
720
  async buildUrl(baseUrl, key) {
643
- return import_cement11.Result.Ok(baseUrl.build().setParam("key", key).URI());
721
+ return import_cement12.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI());
644
722
  }
645
723
  async close() {
646
- return import_cement11.Result.Ok(void 0);
724
+ return import_cement12.Result.Ok(void 0);
647
725
  }
648
726
  // abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
649
727
  getFilePath(url) {
650
- const key = url.getParam("key");
728
+ const key = url.getParam("key" /* KEY */);
651
729
  if (!key) throw this.logger.Error().Url(url).Msg(`key not found`).AsError();
652
730
  return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
653
731
  }
654
732
  async put(url, body) {
655
- return (0, import_cement11.exception2Result)(async () => {
733
+ return (0, import_cement12.exception2Result)(async () => {
656
734
  const file = await this.getFilePath(url);
657
735
  this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
658
736
  await this.fs.writefile(file, body);
@@ -662,19 +740,19 @@ var init_gateway = __esm({
662
740
  return exceptionWrapper(async () => {
663
741
  const file = this.getFilePath(url);
664
742
  try {
743
+ this.logger.Debug().Url(url).Str("file", file).Msg("get");
665
744
  const res = await this.fs.readfile(file);
666
- this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get");
667
- return import_cement11.Result.Ok(new Uint8Array(res));
745
+ return import_cement12.Result.Ok(new Uint8Array(res));
668
746
  } catch (e) {
669
747
  if (isNotFoundError(e)) {
670
- return import_cement11.Result.Err(new NotFoundError(`file not found: ${file}`));
748
+ return import_cement12.Result.Err(new NotFoundError(`file not found: ${file}`));
671
749
  }
672
- return import_cement11.Result.Err(e);
750
+ return import_cement12.Result.Err(e);
673
751
  }
674
752
  });
675
753
  }
676
754
  async delete(url) {
677
- return (0, import_cement11.exception2Result)(async () => {
755
+ return (0, import_cement12.exception2Result)(async () => {
678
756
  await this.fs.unlink(this.getFilePath(url));
679
757
  });
680
758
  }
@@ -700,7 +778,7 @@ var init_gateway = __esm({
700
778
  }
701
779
  }
702
780
  }
703
- return import_cement11.Result.Ok(void 0);
781
+ return import_cement12.Result.Ok(void 0);
704
782
  }
705
783
  };
706
784
  FileTestStore = class {
@@ -709,7 +787,7 @@ var init_gateway = __esm({
709
787
  this.sthis = sthis;
710
788
  }
711
789
  async get(iurl, key) {
712
- const url = iurl.build().setParam("key", key).URI();
790
+ const url = iurl.build().setParam("key" /* KEY */, key).URI();
713
791
  const dbFile = this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
714
792
  this.logger.Debug().Url(url).Str("dbFile", dbFile).Msg("get");
715
793
  const buffer = await (await getFileSystem(url)).readfile(dbFile);
@@ -737,7 +815,7 @@ __export(gateway_exports2, {
737
815
  getIndexDBName: () => getIndexDBName
738
816
  });
739
817
  function ensureVersion(url) {
740
- return url.build().defParam("version", INDEXDB_VERSION).URI();
818
+ return url.build().defParam("version" /* VERSION */, INDEXDB_VERSION).URI();
741
819
  }
742
820
  function sanitzeKey(key) {
743
821
  if (key.length === 1) {
@@ -758,17 +836,17 @@ async function connectIdb(url, sthis) {
758
836
  }
759
837
  });
760
838
  const found = await db.get("version", "version");
761
- const version = ensureVersion(url).getParam("version");
839
+ const version = ensureVersion(url).getParam("version" /* VERSION */);
762
840
  if (!found) {
763
841
  await db.put("version", { version }, "version");
764
842
  } else if (found.version !== version) {
765
- sthis.logger.Warn().Str("url", url.toString()).Str("version", version).Str("found", found.version).Msg("version mismatch");
843
+ sthis.logger.Warn().Url(url).Str("version", version).Str("found", found.version).Msg("version mismatch");
766
844
  }
767
845
  return { db, dbName, version, url };
768
846
  });
769
847
  return {
770
848
  ...once,
771
- url: url.build().setParam("version", once.version).URI()
849
+ url: url.build().setParam("version" /* VERSION */, once.version).URI()
772
850
  };
773
851
  }
774
852
  function joinDBName(...names) {
@@ -777,7 +855,7 @@ function joinDBName(...names) {
777
855
  function getIndexDBName(iurl, sthis) {
778
856
  const url = ensureVersion(iurl);
779
857
  const fullDb = url.pathname.replace(/^\/+/, "").replace(/\?.*$/, "");
780
- const dbName = url.getParam("name");
858
+ const dbName = url.getParam("name" /* NAME */);
781
859
  if (!dbName) throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
782
860
  const result = joinDBName(fullDb, dbName);
783
861
  const objStore = getStore(url, sthis, joinDBName).name;
@@ -789,15 +867,16 @@ function getIndexDBName(iurl, sthis) {
789
867
  dbName
790
868
  };
791
869
  }
792
- var import_idb2, import_cement12, onceIndexDB, IndexDBGateway, IndexDBTestStore;
870
+ var import_idb2, import_cement13, onceIndexDB, IndexDBGateway, IndexDBTestStore;
793
871
  var init_gateway2 = __esm({
794
872
  "src/runtime/gateways/indexdb/gateway.ts"() {
795
873
  "use strict";
796
874
  import_idb2 = require("idb");
797
- import_cement12 = require("@adviser/cement");
875
+ import_cement13 = require("@adviser/cement");
798
876
  init_version2();
799
877
  init_utils();
800
- onceIndexDB = new import_cement12.KeyedResolvOnce();
878
+ init_types();
879
+ onceIndexDB = new import_cement13.KeyedResolvOnce();
801
880
  IndexDBGateway = class {
802
881
  constructor(sthis) {
803
882
  this._db = {};
@@ -805,7 +884,7 @@ var init_gateway2 = __esm({
805
884
  this.sthis = sthis;
806
885
  }
807
886
  async start(baseURL) {
808
- return (0, import_cement12.exception2Result)(async () => {
887
+ return (0, import_cement13.exception2Result)(async () => {
809
888
  this.logger.Debug().Url(baseURL).Msg("starting");
810
889
  await this.sthis.start();
811
890
  const ic = await connectIdb(baseURL, this.sthis);
@@ -815,10 +894,10 @@ var init_gateway2 = __esm({
815
894
  });
816
895
  }
817
896
  async close() {
818
- return import_cement12.Result.Ok(void 0);
897
+ return import_cement13.Result.Ok(void 0);
819
898
  }
820
899
  async destroy(baseUrl) {
821
- return (0, import_cement12.exception2Result)(async () => {
900
+ return (0, import_cement13.exception2Result)(async () => {
822
901
  const type = getStore(baseUrl, this.sthis, joinDBName).name;
823
902
  const idb = this._db;
824
903
  const trans = idb.transaction(type, "readwrite");
@@ -834,7 +913,7 @@ var init_gateway2 = __esm({
834
913
  });
835
914
  }
836
915
  buildUrl(baseUrl, key) {
837
- return Promise.resolve(import_cement12.Result.Ok(baseUrl.build().setParam("key", key).URI()));
916
+ return Promise.resolve(import_cement13.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
838
917
  }
839
918
  async get(url) {
840
919
  return exceptionWrapper(async () => {
@@ -845,13 +924,13 @@ var init_gateway2 = __esm({
845
924
  const bytes = await tx.objectStore(store).get(sanitzeKey(key));
846
925
  await tx.done;
847
926
  if (!bytes) {
848
- return import_cement12.Result.Err(new NotFoundError(`missing ${key}`));
927
+ return import_cement13.Result.Err(new NotFoundError(`missing ${key}`));
849
928
  }
850
- return import_cement12.Result.Ok(bytes);
929
+ return import_cement13.Result.Ok(bytes);
851
930
  });
852
931
  }
853
932
  async put(url, value) {
854
- return (0, import_cement12.exception2Result)(async () => {
933
+ return (0, import_cement13.exception2Result)(async () => {
855
934
  const key = getKey(url, this.logger);
856
935
  const store = getStore(url, this.sthis, joinDBName).name;
857
936
  this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
@@ -861,14 +940,14 @@ var init_gateway2 = __esm({
861
940
  });
862
941
  }
863
942
  async delete(url) {
864
- return (0, import_cement12.exception2Result)(async () => {
943
+ return (0, import_cement13.exception2Result)(async () => {
865
944
  const key = getKey(url, this.logger);
866
945
  const store = getStore(url, this.sthis, joinDBName).name;
867
946
  this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
868
947
  const tx = this._db.transaction([store], "readwrite");
869
948
  await tx.objectStore(store).delete(sanitzeKey(key));
870
949
  await tx.done;
871
- return import_cement12.Result.Ok(void 0);
950
+ return import_cement13.Result.Ok(void 0);
872
951
  });
873
952
  }
874
953
  };
@@ -896,15 +975,16 @@ var init_gateway2 = __esm({
896
975
  var src_exports = {};
897
976
  __export(src_exports, {
898
977
  CRDT: () => CRDT,
899
- Database: () => Database,
978
+ DatabaseFactory: () => DatabaseFactory,
979
+ DatabaseShell: () => DatabaseShell,
900
980
  Index: () => Index,
901
981
  NotFoundError: () => NotFoundError,
902
982
  PACKAGE_VERSION: () => PACKAGE_VERSION,
983
+ PARAM: () => PARAM,
903
984
  Result: () => import_cement.Result,
904
985
  UInt8ArrayEqual: () => UInt8ArrayEqual,
905
986
  blockstore: () => blockstore_exports,
906
987
  bs: () => blockstore_exports,
907
- dataDir: () => dataDir,
908
988
  ensureLogger: () => ensureLogger,
909
989
  ensureSuperLog: () => ensureSuperLog,
910
990
  ensureSuperThis: () => ensureSuperThis,
@@ -915,16 +995,19 @@ __export(src_exports, {
915
995
  getName: () => getName,
916
996
  getStore: () => getStore,
917
997
  index: () => index,
998
+ isDatabase: () => isDatabase,
918
999
  isFalsy: () => isFalsy,
919
1000
  isNotFoundError: () => isNotFoundError,
1001
+ keyConfigOpts: () => keyConfigOpts,
920
1002
  rt: () => runtime_exports,
921
1003
  runtime: () => runtime_exports,
922
- throwFalsy: () => throwFalsy
1004
+ throwFalsy: () => throwFalsy,
1005
+ toStoreURIRuntime: () => toStoreURIRuntime
923
1006
  });
924
1007
  module.exports = __toCommonJS(src_exports);
925
1008
 
926
1009
  // src/database.ts
927
- var import_cement16 = require("@adviser/cement");
1010
+ var import_cement18 = require("@adviser/cement");
928
1011
 
929
1012
  // src/write-queue.ts
930
1013
  function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -967,86 +1050,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
967
1050
  }
968
1051
 
969
1052
  // src/crdt.ts
970
- var import_cement15 = require("@adviser/cement");
971
-
972
- // src/runtime/wait-pr-multiformats/block.ts
973
- var block_exports = {};
974
- __export(block_exports, {
975
- Block: () => Block,
976
- create: () => create,
977
- createUnsafe: () => createUnsafe,
978
- decode: () => decode,
979
- encode: () => encode
980
- });
981
- var import_multiformats = require("multiformats");
982
- var import_block = require("multiformats/block");
983
- var Block = import_block.Block;
984
- async function decode({
985
- bytes,
986
- codec: codec3,
987
- hasher: hasher7
988
- }) {
989
- if (bytes == null) throw new Error('Missing required argument "bytes"');
990
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
991
- const value = await Promise.resolve(codec3.decode(bytes));
992
- const hash = await hasher7.digest(bytes);
993
- const cid = import_multiformats.CID.create(1, codec3.code, hash);
994
- return new import_block.Block({ value, bytes, cid });
995
- }
996
- async function encode({
997
- value,
998
- codec: codec3,
999
- hasher: hasher7
1000
- }) {
1001
- if (typeof value === "undefined") throw new Error('Missing required argument "value"');
1002
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
1003
- const bytes = await Promise.resolve(codec3.encode(value));
1004
- const hash = await hasher7.digest(bytes);
1005
- const cid = import_multiformats.CID.create(1, codec3.code, hash);
1006
- return new import_block.Block({ value, bytes, cid });
1007
- }
1008
- async function create({
1009
- bytes,
1010
- cid,
1011
- hasher: hasher7,
1012
- codec: codec3
1013
- }) {
1014
- if (bytes == null) throw new Error('Missing required argument "bytes"');
1015
- if (hasher7 == null) throw new Error('Missing required argument "hasher"');
1016
- const value = await Promise.resolve(codec3.decode(bytes));
1017
- const hash = await hasher7.digest(bytes);
1018
- if (!import_multiformats.bytes.equals(cid.multihash.bytes, hash.bytes)) {
1019
- throw new Error("CID hash does not match bytes");
1020
- }
1021
- return createUnsafe({
1022
- bytes,
1023
- cid,
1024
- value,
1025
- codec: codec3
1026
- });
1027
- }
1028
- async function createUnsafe({
1029
- bytes,
1030
- cid,
1031
- value: maybeValue,
1032
- codec: codec3
1033
- }) {
1034
- const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
1035
- if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
1036
- return new Block({
1037
- cid,
1038
- bytes,
1039
- value
1040
- });
1041
- }
1042
-
1043
- // src/crdt-helpers.ts
1044
- var import_link = require("multiformats/link");
1045
- var import_sha25 = require("multiformats/hashes/sha2");
1046
- var codec = __toESM(require("@ipld/dag-cbor"), 1);
1047
- var import_crdt = require("@web3-storage/pail/crdt");
1048
- var import_clock2 = require("@web3-storage/pail/clock");
1049
- var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
1053
+ var import_cement17 = require("@adviser/cement");
1050
1054
 
1051
1055
  // src/blockstore/index.ts
1052
1056
  var blockstore_exports = {};
@@ -1059,8 +1063,11 @@ __export(blockstore_exports, {
1059
1063
  FragmentGateway: () => FragmentGateway,
1060
1064
  Loader: () => Loader,
1061
1065
  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
1062
- ensureStart: () => ensureStart,
1063
- getGatewayFromURL: () => getGatewayFromURL,
1066
+ ensureStoreEnDeFile: () => ensureStoreEnDeFile,
1067
+ fileGatewayFactoryItem: () => fileGatewayFactoryItem,
1068
+ getDefaultURI: () => getDefaultURI,
1069
+ getGatewayFactoryItem: () => getGatewayFactoryItem,
1070
+ getStartedGateway: () => getStartedGateway,
1064
1071
  parseCarFile: () => parseCarFile,
1065
1072
  registerStoreProtocol: () => registerStoreProtocol,
1066
1073
  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
@@ -1075,7 +1082,7 @@ function toCIDBlock(block) {
1075
1082
  }
1076
1083
 
1077
1084
  // src/blockstore/store-factory.ts
1078
- var import_cement13 = require("@adviser/cement");
1085
+ var import_cement15 = require("@adviser/cement");
1079
1086
 
1080
1087
  // src/runtime/files.ts
1081
1088
  var files_exports = {};
@@ -1151,258 +1158,24 @@ var UnixFSFileBuilder = class {
1151
1158
  // src/blockstore/store.ts
1152
1159
  var import_dag_json2 = require("@ipld/dag-json");
1153
1160
  var import_cement10 = require("@adviser/cement");
1154
-
1155
- // src/types.ts
1156
- function isFalsy(value) {
1157
- return value === false && value === null && value === void 0;
1158
- }
1159
- function throwFalsy(value) {
1160
- if (isFalsy(value)) {
1161
- throw new Error("value is Falsy");
1162
- }
1163
- return value;
1164
- }
1165
- function falsyToUndef(value) {
1166
- if (isFalsy(value)) {
1167
- return void 0;
1168
- }
1169
- return value;
1170
- }
1171
-
1172
- // src/blockstore/store.ts
1161
+ init_types();
1173
1162
  init_utils();
1174
1163
 
1175
- // src/blockstore/loader.ts
1176
- var import_p_limit = __toESM(require("p-limit"), 1);
1177
- var import_car = require("@ipld/car");
1178
- var import_cement7 = require("@adviser/cement");
1179
-
1180
- // src/blockstore/loader-helpers.ts
1181
- var import_sha2 = require("multiformats/hashes/sha2");
1182
- var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
1183
- async function parseCarFile(reader, logger) {
1184
- const roots = await reader.getRoots();
1185
- const header = await reader.get(roots[0]);
1186
- if (!header) throw logger.Error().Msg("missing header block").AsError();
1187
- const dec = await decode({ bytes: header.bytes, hasher: import_sha2.sha256, codec: dagCodec });
1188
- const fpvalue = dec.value;
1189
- if (fpvalue && !fpvalue.fp) {
1190
- throw logger.Error().Msg("missing fp").AsError();
1191
- }
1192
- return fpvalue.fp;
1193
- }
1194
-
1195
- // src/blockstore/transaction.ts
1196
- var import_block3 = require("@web3-storage/pail/block");
1164
+ // src/blockstore/commit-queue.ts
1197
1165
  var import_cement2 = require("@adviser/cement");
1198
- init_utils();
1199
- var CarTransaction = class extends import_block3.MemoryBlockstore {
1200
- constructor(parent, opts = { add: true, noLoader: false }) {
1201
- super();
1202
- if (opts.add) {
1203
- parent.transactions.add(this);
1204
- }
1205
- this.parent = parent;
1206
- }
1207
- async get(cid) {
1208
- return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
1209
- }
1210
- async superGet(cid) {
1211
- return super.get(cid);
1166
+ var CommitQueue = class {
1167
+ constructor() {
1168
+ this.queue = [];
1169
+ this.processing = false;
1170
+ this._waitIdleItems = /* @__PURE__ */ new Set();
1212
1171
  }
1213
- };
1214
- function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1215
- const logger = ensureLogger(sthis, component, ctx);
1216
- const store = opts.store || {};
1217
- return {
1218
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1219
- applyMeta: (meta, snap) => {
1172
+ waitIdle() {
1173
+ if (this.queue.length === 0 && !this.processing) {
1220
1174
  return Promise.resolve();
1221
- },
1222
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1223
- compact: async (blocks) => {
1224
- return {};
1225
- },
1226
- autoCompact: 100,
1227
- public: false,
1228
- name: void 0,
1229
- threshold: 1e3 * 1e3,
1230
- ...opts,
1231
- logger,
1232
- keyBag: opts.keyBag || {},
1233
- crypto: (0, import_cement2.toCryptoRuntime)(opts.crypto),
1234
- store,
1235
- storeRuntime: toStoreRuntime(store, sthis)
1236
- };
1237
- }
1238
- function blockstoreFactory(sthis, opts) {
1239
- if (opts.name) {
1240
- return new EncryptedBlockstore(sthis, opts);
1241
- } else {
1242
- return new BaseBlockstore(opts);
1243
- }
1244
- }
1245
- var BaseBlockstore = class {
1246
- constructor(ebOpts = {}) {
1247
- this.transactions = /* @__PURE__ */ new Set();
1248
- this.sthis = ensureSuperThis(ebOpts);
1249
- this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1250
- this.logger = this.ebOpts.logger;
1251
- }
1252
- // ready: Promise<void>;
1253
- ready() {
1254
- return Promise.resolve();
1255
- }
1256
- async close() {
1257
- }
1258
- async destroy() {
1259
- }
1260
- async get(cid) {
1261
- if (!cid) throw this.logger.Error().Msg("required cid").AsError();
1262
- for (const f of this.transactions) {
1263
- const v = await f.superGet(cid);
1264
- if (v) return v;
1265
- }
1266
- }
1267
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1268
- async put(cid, block) {
1269
- throw this.logger.Error().Msg("use a transaction to put").AsError();
1270
- }
1271
- // TransactionMeta
1272
- async transaction(fn, _opts) {
1273
- const t = new CarTransaction(this, _opts);
1274
- const done = await fn(t);
1275
- this.lastTxMeta = done;
1276
- return { t, meta: done };
1277
- }
1278
- async *entries() {
1279
- const seen = /* @__PURE__ */ new Set();
1280
- for (const t of this.transactions) {
1281
- for await (const blk of t.entries()) {
1282
- if (seen.has(blk.cid.toString())) continue;
1283
- seen.add(blk.cid.toString());
1284
- yield blk;
1285
- }
1286
- }
1287
- }
1288
- };
1289
- var EncryptedBlockstore = class extends BaseBlockstore {
1290
- constructor(sthis, ebOpts) {
1291
- super(ebOpts);
1292
- this.compacting = false;
1293
- this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
1294
- const { name } = ebOpts;
1295
- if (!name) {
1296
- throw this.logger.Error().Msg("name required").AsError();
1297
- }
1298
- this.name = name;
1299
- this.loader = new Loader(this.name, ebOpts, sthis);
1300
- }
1301
- ready() {
1302
- return this.loader.ready();
1303
- }
1304
- close() {
1305
- return this.loader.close();
1306
- }
1307
- destroy() {
1308
- return this.loader.destroy();
1309
- }
1310
- async get(cid) {
1311
- const got = await super.get(cid);
1312
- if (got) return got;
1313
- if (!this.loader) {
1314
- return;
1315
- }
1316
- return falsyToUndef(await this.loader.getBlock(cid));
1317
- }
1318
- async transaction(fn, opts = { noLoader: false }) {
1319
- const { t, meta: done } = await super.transaction(fn);
1320
- const cars = await this.loader.commit(t, done, opts);
1321
- if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1322
- setTimeout(() => void this.compact(), 10);
1323
- }
1324
- if (cars) {
1325
- this.transactions.delete(t);
1326
- return { meta: done, cars, t };
1327
- }
1328
- throw this.logger.Error().Msg("failed to commit car files").AsError();
1329
- }
1330
- async getFile(car, cid) {
1331
- await this.ready();
1332
- if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
1333
- const reader = await this.loader.loadFileCar(
1334
- car
1335
- /*, isPublic */
1336
- );
1337
- const block = await reader.get(cid);
1338
- if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
1339
- return block.bytes;
1340
- }
1341
- async compact() {
1342
- await this.ready();
1343
- if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
1344
- if (this.loader.carLog.length < 2) return;
1345
- const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
1346
- if (!compactFn || this.compacting) return;
1347
- const blockLog = new CompactionFetcher(this);
1348
- this.compacting = true;
1349
- const meta = await compactFn(blockLog);
1350
- await this.loader?.commit(blockLog.loggedBlocks, meta, {
1351
- compact: true,
1352
- noLoader: true
1353
- });
1354
- this.compacting = false;
1355
- }
1356
- async defaultCompact(blocks, logger) {
1357
- if (!this.loader) {
1358
- throw logger.Error().Msg("no loader").AsError();
1359
- }
1360
- if (!this.lastTxMeta) {
1361
- throw logger.Error().Msg("no lastTxMeta").AsError();
1362
- }
1363
- for await (const blk of this.loader.entries(false)) {
1364
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1365
- }
1366
- for (const t of this.transactions) {
1367
- for await (const blk of t.entries()) {
1368
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1369
- }
1370
- }
1371
- return this.lastTxMeta;
1372
- }
1373
- async *entries() {
1374
- for await (const blk of this.loader.entries()) {
1375
- yield blk;
1376
- }
1377
- }
1378
- };
1379
- var CompactionFetcher = class {
1380
- constructor(blocks) {
1381
- this.blockstore = blocks;
1382
- this.loggedBlocks = new CarTransaction(blocks);
1383
- }
1384
- async get(cid) {
1385
- const block = await this.blockstore.get(cid);
1386
- if (block) this.loggedBlocks.putSync(cid, block.bytes);
1387
- return falsyToUndef(block);
1388
- }
1389
- };
1390
-
1391
- // src/blockstore/commit-queue.ts
1392
- var import_cement3 = require("@adviser/cement");
1393
- var CommitQueue = class {
1394
- constructor() {
1395
- this.queue = [];
1396
- this.processing = false;
1397
- this._waitIdleItems = /* @__PURE__ */ new Set();
1398
- }
1399
- waitIdle() {
1400
- if (this.queue.length === 0 && !this.processing) {
1401
- return Promise.resolve();
1402
- }
1403
- const fn = new import_cement3.Future();
1404
- this._waitIdleItems.add(fn);
1405
- return fn.asPromise();
1175
+ }
1176
+ const fn = new import_cement2.Future();
1177
+ this._waitIdleItems.add(fn);
1178
+ return fn.asPromise();
1406
1179
  }
1407
1180
  async enqueue(fn) {
1408
1181
  return new Promise((resolve, reject) => {
@@ -1439,1132 +1212,1501 @@ var CommitQueue = class {
1439
1212
  }
1440
1213
  };
1441
1214
 
1442
- // src/runtime/key-bag.ts
1443
- var key_bag_exports = {};
1444
- __export(key_bag_exports, {
1445
- KeyBag: () => KeyBag,
1446
- getKeyBag: () => getKeyBag,
1447
- registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
1215
+ // src/runtime/keyed-crypto.ts
1216
+ var keyed_crypto_exports = {};
1217
+ __export(keyed_crypto_exports, {
1218
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1219
+ keyedCryptoFactory: () => keyedCryptoFactory
1448
1220
  });
1449
- var import_cement6 = require("@adviser/cement");
1450
1221
  init_utils();
1451
1222
  var import_base582 = require("multiformats/bases/base58");
1452
- var KeyBag = class {
1453
- constructor(rt) {
1454
- this.rt = rt;
1455
- this._warnOnce = new import_cement6.ResolveOnce();
1456
- this._seq = new import_cement6.ResolveSeq();
1457
- this.logger = ensureLogger(rt.sthis, "KeyBag");
1458
- this.logger.Debug().Msg("KeyBag created");
1459
- }
1460
- async subtleKey(key) {
1461
- const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
1462
- if (extractable) {
1463
- this._warnOnce.once(
1464
- () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
1465
- );
1466
- }
1467
- return await this.rt.crypto.importKey(
1468
- "raw",
1469
- // raw or jwk
1470
- import_base582.base58btc.decode(key),
1471
- // hexStringToUint8Array(key), // raw data
1472
- "AES-GCM",
1473
- extractable,
1474
- ["encrypt", "decrypt"]
1475
- );
1476
- }
1477
- async ensureKeyFromUrl(url, keyFactory) {
1478
- const storeKey = url.getParam("storekey");
1479
- if (storeKey === "insecure") {
1480
- return import_cement6.Result.Ok(url);
1481
- }
1482
- if (!storeKey) {
1483
- const keyName = `@${keyFactory()}@`;
1484
- const ret = await this.getNamedKey(keyName);
1485
- if (ret.isErr()) {
1486
- return ret;
1487
- }
1488
- const urb = url.build().setParam("storekey", keyName);
1489
- return import_cement6.Result.Ok(urb.URI());
1223
+ var import_sha2 = require("multiformats/hashes/sha2");
1224
+ var CBOR = __toESM(require("cborg"), 1);
1225
+ init_types();
1226
+ var generateIV = {
1227
+ random: {
1228
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1229
+ calc: async (ko, crypto, data) => {
1230
+ return crypto.randomBytes(ko.ivLength);
1231
+ },
1232
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1233
+ verify: async (ko, crypto, iv, data) => {
1234
+ return true;
1490
1235
  }
1491
- if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
1492
- const ret = await this.getNamedKey(storeKey);
1493
- if (ret.isErr()) {
1494
- return ret;
1236
+ },
1237
+ hash: {
1238
+ calc: async (ko, crypto, data) => {
1239
+ const hash = await import_sha2.sha256.digest(data);
1240
+ const hashBytes = new Uint8Array(hash.bytes);
1241
+ const hashArray = new Uint8Array(ko.ivLength);
1242
+ for (let i = 0; i < hashBytes.length; i++) {
1243
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
1495
1244
  }
1245
+ return hashArray;
1246
+ },
1247
+ verify: async function(ko, crypto, iv, data) {
1248
+ return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1496
1249
  }
1497
- return import_cement6.Result.Ok(url);
1498
1250
  }
1499
- async toKeyWithFingerPrint(keyStr) {
1500
- const material = import_base582.base58btc.decode(keyStr);
1501
- const key = await this.subtleKey(keyStr);
1502
- const fpr = await this.rt.crypto.digestSHA256(material);
1503
- return import_cement6.Result.Ok({
1504
- key,
1505
- fingerPrint: import_base582.base58btc.encode(new Uint8Array(fpr))
1251
+ };
1252
+ function getGenerateIVFn(url, opts) {
1253
+ const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
1254
+ return generateIV[ivhash] || generateIV["hash"];
1255
+ }
1256
+ var BlockIvKeyIdCodec = class {
1257
+ constructor(ko, iv, opts) {
1258
+ this.code = 3147065;
1259
+ this.name = "Fireproof@encrypted-block:aes-gcm";
1260
+ this.ko = ko;
1261
+ this.iv = iv;
1262
+ this.opts = opts || {};
1263
+ }
1264
+ async encode(data) {
1265
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1266
+ const { iv } = this.ko.algo(calcIv);
1267
+ const fprt = await this.ko.fingerPrint();
1268
+ const keyId = import_base582.base58btc.decode(fprt);
1269
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1270
+ return CBOR.encode({
1271
+ iv,
1272
+ keyId,
1273
+ data: await this.ko._encrypt({ iv, bytes: data })
1506
1274
  });
1507
1275
  }
1508
- async setNamedKey(name, key) {
1509
- return this._seq.add(() => this._setNamedKey(name, key));
1276
+ async decode(abytes) {
1277
+ let bytes;
1278
+ if (abytes instanceof Uint8Array) {
1279
+ bytes = abytes;
1280
+ } else {
1281
+ bytes = new Uint8Array(abytes);
1282
+ }
1283
+ const { iv, keyId, data } = CBOR.decode(bytes);
1284
+ const fprt = await this.ko.fingerPrint();
1285
+ this.ko.logger.Debug().Str("fp", import_base582.base58btc.encode(keyId)).Msg("decode");
1286
+ if (import_base582.base58btc.encode(keyId) !== fprt) {
1287
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base582.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
1288
+ }
1289
+ const result = await this.ko._decrypt({ iv, bytes: data });
1290
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1291
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1292
+ }
1293
+ return result;
1510
1294
  }
1511
- // avoid deadlock
1512
- async _setNamedKey(name, key) {
1513
- const item = {
1514
- name,
1515
- key
1295
+ };
1296
+ var keyedCrypto = class {
1297
+ constructor(url, key, cyopt, sthis) {
1298
+ this.ivLength = 12;
1299
+ this.isEncrypting = true;
1300
+ this.logger = ensureLogger(sthis, "keyedCrypto");
1301
+ this.crypto = cyopt;
1302
+ this.key = key;
1303
+ this.url = url;
1304
+ }
1305
+ fingerPrint() {
1306
+ return Promise.resolve(this.key.fingerPrint);
1307
+ }
1308
+ codec(iv, opts) {
1309
+ return new BlockIvKeyIdCodec(this, iv, opts);
1310
+ }
1311
+ algo(iv) {
1312
+ return {
1313
+ name: "AES-GCM",
1314
+ iv: iv || this.crypto.randomBytes(this.ivLength),
1315
+ tagLength: 128
1516
1316
  };
1517
- const bag = await this.rt.getBag();
1518
- this.logger.Debug().Str("name", name).Msg("setNamedKey");
1519
- await bag.set(name, item);
1520
- return await this.toKeyWithFingerPrint(item.key);
1521
1317
  }
1522
- async getNamedExtractableKey(name, failIfNotFound = false) {
1523
- const ret = await this.getNamedKey(name, failIfNotFound);
1524
- if (ret.isErr()) {
1525
- return ret;
1526
- }
1527
- const named = ret.Ok();
1528
- return import_cement6.Result.Ok({
1529
- ...named,
1530
- extract: async () => {
1531
- const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
1532
- return {
1533
- key: ext,
1534
- keyStr: import_base582.base58btc.encode(ext)
1535
- };
1536
- }
1537
- });
1318
+ async _decrypt(data) {
1319
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
1320
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
1538
1321
  }
1539
- async getNamedKey(name, failIfNotFound = false) {
1540
- const id = this.rt.sthis.nextId(4).str;
1541
- return this._seq.add(async () => {
1542
- const bag = await this.rt.getBag();
1543
- const named = await bag.get(name);
1544
- if (named) {
1545
- const fpr = await this.toKeyWithFingerPrint(named.key);
1546
- this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
1547
- return fpr;
1548
- }
1549
- if (failIfNotFound) {
1550
- this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
1551
- return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
1552
- }
1553
- const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
1554
- this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
1555
- return ret;
1556
- });
1322
+ async _encrypt(data) {
1323
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
1324
+ const a = this.algo(data.iv);
1325
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
1557
1326
  }
1558
1327
  };
1559
- var keyBagProviderFactories = new Map(
1560
- [
1561
- {
1562
- protocol: "file:",
1563
- factory: async (url, sthis) => {
1564
- const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1565
- return new KeyBagProviderFile2(url, sthis);
1566
- }
1567
- },
1568
- {
1569
- protocol: "indexdb:",
1570
- factory: async (url, sthis) => {
1571
- const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1572
- return new KeyBagProviderIndexDB2(url, sthis);
1573
- }
1574
- }
1575
- ].map((i) => [i.protocol, i])
1576
- );
1577
- function registerKeyBagProviderFactory(item) {
1578
- const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
1579
- keyBagProviderFactories.set(protocol, {
1580
- ...item,
1581
- protocol
1582
- });
1583
- }
1584
- function defaultKeyBagOpts(sthis, kbo) {
1585
- if (kbo.keyRuntime) {
1586
- return kbo.keyRuntime;
1328
+ var nullCodec = class {
1329
+ constructor() {
1330
+ this.code = 0;
1331
+ this.name = "Fireproof@unencrypted-block";
1587
1332
  }
1588
- const logger = ensureLogger(sthis, "KeyBag");
1589
- let url;
1590
- if (kbo.url) {
1591
- url = import_cement6.URI.from(kbo.url);
1592
- logger.Debug().Url(url).Msg("from opts");
1593
- } else {
1594
- let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
1595
- if ((0, import_cement6.runtimeFn)().isBrowser) {
1596
- url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
1597
- } else {
1598
- if (!bagFnameOrUrl) {
1599
- const home = sthis.env.get("HOME");
1600
- bagFnameOrUrl = `${home}/.fireproof/keybag`;
1601
- url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
1602
- } else {
1603
- url = import_cement6.URI.from(bagFnameOrUrl);
1604
- }
1605
- }
1606
- logger.Debug().Url(url).Msg("from env");
1333
+ encode(data) {
1334
+ return data;
1607
1335
  }
1608
- const kitem = keyBagProviderFactories.get(url.protocol);
1609
- if (!kitem) {
1610
- throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
1336
+ decode(data) {
1337
+ return data;
1611
1338
  }
1612
- const getBag = async () => kitem.factory(url, sthis);
1613
- if (url.hasParam("masterkey")) {
1614
- throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
1339
+ };
1340
+ var noCrypto = class {
1341
+ constructor(url, cyrt, sthis) {
1342
+ this.ivLength = 0;
1343
+ this.code = 0;
1344
+ this.name = "Fireproof@unencrypted-block";
1345
+ this.isEncrypting = false;
1346
+ this._fingerPrint = "noCrypto:" + Math.random();
1347
+ this.logger = ensureLogger(sthis, "noCrypto");
1348
+ this.crypto = cyrt;
1349
+ this.url = url;
1615
1350
  }
1616
- return {
1617
- url,
1618
- crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
1619
- sthis,
1620
- logger,
1621
- keyLength: kbo.keyLength || 16,
1622
- getBag,
1623
- id: () => {
1624
- return url.toString();
1625
- }
1626
- };
1627
- }
1628
- var _keyBags = new import_cement6.KeyedResolvOnce();
1629
- async function getKeyBag(sthis, kbo = {}) {
1630
- await sthis.start();
1631
- const rt = defaultKeyBagOpts(sthis, kbo);
1632
- return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
1633
- }
1634
-
1635
- // src/blockstore/commitor.ts
1636
- var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
1637
- var import_sha22 = require("multiformats/hashes/sha2");
1638
- var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
1639
- async function encodeCarFile(roots, t, codec3) {
1640
- let size = 0;
1641
- const headerSize = CBW.headerLength({ roots });
1642
- size += headerSize;
1643
- for (const { cid, bytes } of t.entries()) {
1644
- size += CBW.blockLength({ cid, bytes });
1351
+ fingerPrint() {
1352
+ return Promise.resolve(this._fingerPrint);
1645
1353
  }
1646
- const buffer = new Uint8Array(size);
1647
- const writer = CBW.createWriter(buffer, { headerSize });
1648
- for (const r of roots) {
1649
- writer.addRoot(r);
1354
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1355
+ codec(iv) {
1356
+ return new nullCodec();
1650
1357
  }
1651
- for (const { cid, bytes } of t.entries()) {
1652
- writer.write({ cid, bytes });
1358
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1359
+ algo(iv) {
1360
+ return {
1361
+ name: "noCrypto",
1362
+ iv: new Uint8Array(),
1363
+ tagLength: 0
1364
+ };
1653
1365
  }
1654
- writer.close();
1655
- return await encode({ value: writer.bytes, hasher: import_sha22.sha256, codec: codec3 });
1656
- }
1657
- async function createCarFile(encoder, cid, t) {
1658
- return encodeCarFile([cid], t, encoder);
1659
- }
1660
- async function commitFiles(fileStore, walStore, t, done) {
1661
- const { files: roots } = makeFileCarHeader(done);
1662
- const cids = [];
1663
- const codec3 = (await fileStore.keyedCrypto()).codec();
1664
- const cars = await prepareCarFilesFiles(codec3, roots, t);
1665
- for (const car of cars) {
1666
- const { cid, bytes } = car;
1667
- await fileStore.save({ cid, bytes });
1668
- await walStore.enqueueFile(
1669
- cid
1670
- /*, !!opts.public*/
1671
- );
1672
- cids.push(cid);
1366
+ _decrypt() {
1367
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1673
1368
  }
1674
- return cids;
1675
- }
1676
- function makeFileCarHeader(result) {
1677
- const files = [];
1678
- for (const [, meta] of Object.entries(result.files || {})) {
1679
- if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1680
- files.push(meta.cid);
1369
+ _encrypt() {
1370
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1371
+ }
1372
+ };
1373
+ async function keyedCryptoFactory(url, kb, sthis) {
1374
+ const storekey = url.getParam("storekey" /* STORE_KEY */);
1375
+ if (storekey && storekey !== "insecure") {
1376
+ let rkey = await kb.getNamedKey(storekey, true);
1377
+ if (rkey.isErr()) {
1378
+ try {
1379
+ rkey = await kb.toKeyWithFingerPrint(storekey);
1380
+ } catch (e) {
1381
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
1382
+ }
1681
1383
  }
1384
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
1682
1385
  }
1683
- return { ...result, files };
1684
- }
1685
- async function prepareCarFilesFiles(encoder, roots, t) {
1686
- return [await encodeCarFile(roots, t, encoder)];
1687
- }
1688
- function makeCarHeader(meta, cars, compact = false) {
1689
- const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1690
- return { ...coreHeader, meta };
1691
- }
1692
- async function encodeCarHeader(fp) {
1693
- return await encode({
1694
- value: { fp },
1695
- hasher: import_sha22.sha256,
1696
- codec: dagCodec2
1697
- });
1386
+ return new noCrypto(url, kb.rt.crypto, sthis);
1698
1387
  }
1699
- async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
1700
- const fp = makeCarHeader(done, params.carLog, !!opts.compact);
1701
- const rootBlock = await encodeCarHeader(fp);
1702
- const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
1703
- const cids = [];
1704
- for (const car of cars) {
1705
- const { cid, bytes } = car;
1706
- await params.carStore.save({ cid, bytes });
1707
- cids.push(cid);
1388
+
1389
+ // src/blockstore/fragment-gateway.ts
1390
+ var import_cement3 = require("@adviser/cement");
1391
+ var import_base583 = require("multiformats/bases/base58");
1392
+ var import_cborg = require("cborg");
1393
+ init_types();
1394
+ init_utils();
1395
+ function getFragSize(url) {
1396
+ const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
1397
+ let ret = 0;
1398
+ if (fragSize) {
1399
+ ret = parseInt(fragSize);
1708
1400
  }
1709
- const newDbMeta = { cars: cids };
1710
- await params.WALStore.enqueue(newDbMeta, opts);
1711
- await params.metaStore.save(newDbMeta);
1712
- return { cgrp: cids, header: fp };
1401
+ if (isNaN(ret) || ret <= 0) {
1402
+ ret = 0;
1403
+ }
1404
+ return ret;
1713
1405
  }
1714
- async function prepareCarFiles(encoder, threshold, rootBlock, t) {
1715
- const carFiles = [];
1716
- threshold = threshold || 128e3 * 8;
1717
- let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1718
- clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1719
- let newsize = CBW.blockLength(toCIDBlock(rootBlock));
1720
- let cidRootBlock = rootBlock;
1721
- for (const { cid, bytes } of t.entries()) {
1722
- newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
1723
- if (newsize >= threshold) {
1724
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1725
- clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1726
- clonedt.putSync(cid, bytes);
1727
- cidRootBlock = { cid, bytes };
1728
- newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
1729
- } else {
1730
- clonedt.putSync(cid, bytes);
1406
+ async function getFrags(url, innerGW, headerSize, logger) {
1407
+ const fragSize = getFragSize(url);
1408
+ if (!fragSize) {
1409
+ const res = await innerGW.get(url);
1410
+ if (res.isErr()) {
1411
+ return [res];
1731
1412
  }
1413
+ const data = res.unwrap();
1414
+ return [
1415
+ import_cement3.Result.Ok({
1416
+ fid: new Uint8Array(0),
1417
+ ofs: 0,
1418
+ len: data.length,
1419
+ data
1420
+ })
1421
+ ];
1732
1422
  }
1733
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1734
- return carFiles;
1423
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
1424
+ if (firstRaw.isErr()) {
1425
+ return [firstRaw];
1426
+ }
1427
+ const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
1428
+ const blockSize = firstFragment.data.length;
1429
+ const ops = [Promise.resolve(import_cement3.Result.Ok(firstFragment))];
1430
+ const fidStr = import_base583.base58btc.encode(firstFragment.fid);
1431
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
1432
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
1433
+ ops.push(
1434
+ (async (furl, ofs2) => {
1435
+ const raw2 = await innerGW.get(furl);
1436
+ if (raw2.isErr()) {
1437
+ return raw2;
1438
+ }
1439
+ const fragment = (0, import_cborg.decode)(raw2.unwrap());
1440
+ if (import_base583.base58btc.encode(fragment.fid) !== fidStr) {
1441
+ return import_cement3.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
1442
+ }
1443
+ if (fragment.ofs !== ofs2) {
1444
+ return import_cement3.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
1445
+ }
1446
+ return import_cement3.Result.Ok(fragment);
1447
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
1448
+ );
1449
+ }
1450
+ return Promise.all(ops);
1735
1451
  }
1736
-
1737
- // src/blockstore/loader.ts
1738
- var import_sha23 = require("multiformats/hashes/sha2");
1739
-
1740
- // src/blockstore/task-manager.ts
1741
- init_utils();
1742
- var TaskManager = class {
1743
- constructor(sthis, callback) {
1744
- this.eventsWeHandled = /* @__PURE__ */ new Set();
1745
- this.queue = [];
1746
- this.isProcessing = false;
1747
- this.logger = ensureLogger(sthis, "TaskManager");
1748
- this.callback = callback;
1452
+ var FragmentGateway = class {
1453
+ constructor(sthis, innerGW) {
1454
+ this.fidLength = 4;
1455
+ this.headerSize = 32;
1456
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
1457
+ this.logger = this.sthis.logger;
1458
+ this.innerGW = innerGW;
1749
1459
  }
1750
- async handleEvent(cid, parents, dbMeta) {
1751
- for (const parent of parents) {
1752
- this.eventsWeHandled.add(parent.toString());
1460
+ slicer(url, body) {
1461
+ const fragSize = getFragSize(url);
1462
+ if (!fragSize) {
1463
+ return [this.innerGW.put(url, body)];
1753
1464
  }
1754
- this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
1755
- this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
1756
- void this.processQueue();
1757
- }
1758
- async processQueue() {
1759
- if (this.isProcessing) return;
1760
- this.isProcessing = true;
1761
- const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1762
- const first = filteredQueue[0];
1763
- if (!first) {
1764
- return;
1465
+ const blocksize = fragSize - this.headerSize;
1466
+ if (blocksize <= 0) {
1467
+ throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
1765
1468
  }
1766
- try {
1767
- await this.callback(first.dbMeta);
1768
- this.eventsWeHandled.add(first.cid);
1769
- this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1770
- } catch (err) {
1771
- if (first.retries++ > 3) {
1772
- this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
1773
- this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
1774
- }
1775
- await new Promise((resolve) => setTimeout(resolve, 50));
1776
- throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
1777
- } finally {
1778
- this.isProcessing = false;
1779
- if (this.queue.length > 0) {
1780
- void this.processQueue();
1469
+ const ops = [];
1470
+ const fid = this.sthis.nextId(this.fidLength);
1471
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
1472
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
1473
+ const block = (0, import_cborg.encode)({
1474
+ fid: fid.bin,
1475
+ ofs,
1476
+ len: body.length,
1477
+ data: body.slice(ofs, ofs + blocksize)
1478
+ });
1479
+ if (block.length > fragSize) {
1480
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
1781
1481
  }
1482
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
1782
1483
  }
1484
+ return ops;
1783
1485
  }
1784
- };
1785
-
1786
- // src/blockstore/loader.ts
1787
- function carLogIncludesGroup(list, cids) {
1788
- return list.some((arr) => {
1789
- return arr.toString() === cids.toString();
1790
- });
1791
- }
1792
- function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
1793
- const byString = /* @__PURE__ */ new Map();
1794
- for (const cid of list) {
1795
- if (remove.has(cid.toString())) continue;
1796
- byString.set(cid.toString(), cid);
1797
- }
1798
- return [...byString.values()];
1799
- }
1800
- var Loader = class {
1801
- constructor(name, ebOpts, sthis) {
1802
- this.commitQueue = new CommitQueue();
1803
- this.isCompacting = false;
1804
- this.carReaders = /* @__PURE__ */ new Map();
1805
- this.seenCompacted = /* @__PURE__ */ new Set();
1806
- this.processedCars = /* @__PURE__ */ new Set();
1807
- this.carLog = [];
1808
- this.getBlockCache = /* @__PURE__ */ new Map();
1809
- this.seenMeta = /* @__PURE__ */ new Set();
1810
- this.writeLimit = (0, import_p_limit.default)(1);
1811
- this.onceReady = new import_cement7.ResolveOnce();
1812
- this.name = name;
1813
- this.sthis = sthis;
1814
- this.ebOpts = defaultedBlockstoreRuntime(
1815
- sthis,
1816
- {
1817
- ...ebOpts,
1818
- name
1819
- },
1820
- "Loader"
1821
- );
1822
- this.logger = this.ebOpts.logger;
1823
- this.taskManager = new TaskManager(sthis, async (dbMeta) => {
1824
- await this.handleDbMetasFromStore([dbMeta]);
1825
- });
1826
- }
1827
- // readonly id = uuidv4();
1828
- async keyBag() {
1829
- return getKeyBag(this.sthis, this.ebOpts.keyBag);
1486
+ buildUrl(baseUrl, key) {
1487
+ return this.innerGW.buildUrl(baseUrl, key);
1830
1488
  }
1831
- async carStore() {
1832
- return this.ebOpts.storeRuntime.makeDataStore(this);
1489
+ async destroy(iurl) {
1490
+ return this.innerGW.destroy(iurl);
1833
1491
  }
1834
- async fileStore() {
1835
- return this.ebOpts.storeRuntime.makeDataStore(this);
1492
+ async start(url) {
1493
+ this.headerSize = (0, import_cborg.encode)({
1494
+ fid: this.sthis.nextId(this.fidLength).bin,
1495
+ ofs: 1024 * 1024,
1496
+ // 32bit
1497
+ len: 16 * 1024 * 1024,
1498
+ // 32bit
1499
+ data: new Uint8Array(1024)
1500
+ }).length - 1024;
1501
+ return this.innerGW.start(url);
1836
1502
  }
1837
- async WALStore() {
1838
- return this.ebOpts.storeRuntime.makeWALStore(this);
1503
+ async close(url) {
1504
+ return this.innerGW.close(url);
1839
1505
  }
1840
- async metaStore() {
1841
- return this.ebOpts.storeRuntime.makeMetaStore(this);
1506
+ async put(url, body) {
1507
+ await Promise.all(this.slicer(url, body));
1508
+ return import_cement3.Result.Ok(void 0);
1842
1509
  }
1843
- async ready() {
1844
- return this.onceReady.once(async () => {
1845
- const metas = await (await this.metaStore()).load();
1846
- if (this.ebOpts.meta) {
1847
- await this.handleDbMetasFromStore([this.ebOpts.meta]);
1848
- } else if (metas) {
1849
- await this.handleDbMetasFromStore(metas);
1510
+ async get(url) {
1511
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1512
+ let buffer = void 0;
1513
+ for (const rfrag of rfrags) {
1514
+ if (rfrag.isErr()) {
1515
+ return import_cement3.Result.Err(rfrag.Err());
1850
1516
  }
1851
- });
1852
- }
1853
- async close() {
1854
- const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1855
- await Promise.all(toClose.map((store) => store.close()));
1856
- }
1857
- async destroy() {
1858
- const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1859
- await Promise.all(toDestroy.map((store) => store.destroy()));
1860
- }
1861
- // async snapToCar(carCid: AnyLink | string) {
1862
- // await this.ready
1863
- // if (typeof carCid === 'string') {
1864
- // carCid = CID.parse(carCid)
1865
- // }
1866
- // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
1867
- // this.carLog = [carCid, ...carHeader.cars]
1868
- // await this.getMoreReaders(carHeader.cars)
1869
- // await this._applyCarHeader(carHeader, true)
1870
- // }
1871
- async handleDbMetasFromStore(metas) {
1872
- this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
1873
- for (const meta of metas) {
1874
- await this.writeLimit(async () => {
1875
- await this.mergeDbMetaIntoClock(meta);
1876
- });
1517
+ const frag = rfrag.Ok();
1518
+ buffer = buffer || new Uint8Array(frag.len);
1519
+ buffer.set(frag.data, frag.ofs);
1877
1520
  }
1521
+ return import_cement3.Result.Ok(buffer || new Uint8Array(0));
1878
1522
  }
1879
- async mergeDbMetaIntoClock(meta) {
1880
- if (this.isCompacting) {
1881
- throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1523
+ async subscribe(url, callback) {
1524
+ if (this.innerGW.subscribe) {
1525
+ return this.innerGW.subscribe(url, callback);
1526
+ } else {
1527
+ return import_cement3.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
1882
1528
  }
1883
- if (this.seenMeta.has(meta.cars.toString())) return;
1884
- this.seenMeta.add(meta.cars.toString());
1885
- if (carLogIncludesGroup(this.carLog, meta.cars)) {
1886
- return;
1887
- }
1888
- const carHeader = await this.loadCarHeaderFromMeta(meta);
1889
- carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1890
- await this.getMoreReaders(carHeader.cars.flat());
1891
- this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1892
- await this.ebOpts.applyMeta?.(carHeader.meta);
1893
- }
1894
- // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1895
- // const { key } = meta;
1896
- // if (key) {
1897
- // await this.setKey(key);
1898
- // }
1899
- // }
1900
- async loadCarHeaderFromMeta({ cars: cids }) {
1901
- const reader = await this.loadCar(cids[0]);
1902
- return await parseCarFile(reader, this.logger);
1903
- }
1904
- // async _getKey(): Promise<string | undefined> {
1905
- // if (this.key) return this.key;
1906
- // // generate a random key
1907
- // if (!this.ebOpts.public) {
1908
- // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1909
- // }
1910
- // return this.key || undefined;
1911
- // }
1912
- async commitFiles(t, done) {
1913
- await this.ready();
1914
- const fstore = await this.fileStore();
1915
- const wstore = await this.WALStore();
1916
- return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1917
1529
  }
1918
- async loadFileCar(cid) {
1919
- return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1530
+ async delete(url) {
1531
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1532
+ for (const rfrag of rfrags) {
1533
+ if (rfrag.isErr()) {
1534
+ return import_cement3.Result.Err(rfrag.Err());
1535
+ }
1536
+ const frag = rfrag.Ok();
1537
+ let fragUrl;
1538
+ if (rfrags.length > 1) {
1539
+ const fidStr = import_base583.base58btc.encode(frag.fid);
1540
+ fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
1541
+ } else {
1542
+ fragUrl = url;
1543
+ }
1544
+ await this.innerGW.delete(fragUrl);
1545
+ }
1546
+ return import_cement3.Result.Ok(void 0);
1920
1547
  }
1921
- async commit(t, done, opts = { noLoader: false, compact: false }) {
1922
- await this.ready();
1923
- const fstore = await this.fileStore();
1924
- const params = {
1925
- encoder: (await fstore.keyedCrypto()).codec(),
1926
- carLog: this.carLog,
1927
- carStore: fstore,
1928
- WALStore: await this.WALStore(),
1929
- metaStore: await this.metaStore(),
1930
- threshold: this.ebOpts.threshold
1931
- };
1932
- return this.commitQueue.enqueue(async () => {
1933
- await this.cacheTransaction(t);
1934
- const ret = await commit(params, t, done, opts);
1935
- await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1936
- return ret.cgrp;
1548
+ };
1549
+
1550
+ // src/blockstore/meta-key-helper.ts
1551
+ var import_dag_json = require("@ipld/dag-json");
1552
+ var import_clock = require("@web3-storage/pail/clock");
1553
+ var import_multiformats = require("multiformats");
1554
+ var import_base64 = require("multiformats/bases/base64");
1555
+ var import_cement7 = require("@adviser/cement");
1556
+
1557
+ // src/runtime/key-bag.ts
1558
+ var key_bag_exports = {};
1559
+ __export(key_bag_exports, {
1560
+ KeyBag: () => KeyBag,
1561
+ defaultKeyBagOpts: () => defaultKeyBagOpts,
1562
+ getKeyBag: () => getKeyBag,
1563
+ registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
1564
+ });
1565
+ var import_cement6 = require("@adviser/cement");
1566
+ init_utils();
1567
+ var import_base584 = require("multiformats/bases/base58");
1568
+ init_types();
1569
+ var KeyBag = class {
1570
+ constructor(rt) {
1571
+ this.rt = rt;
1572
+ this._warnOnce = new import_cement6.ResolveOnce();
1573
+ this._seq = new import_cement6.ResolveSeq();
1574
+ this.logger = ensureLogger(rt.sthis, "KeyBag", {
1575
+ id: rt.id()
1937
1576
  });
1577
+ this.logger.Debug().Msg("KeyBag created");
1938
1578
  }
1939
- async updateCarLog(cids, fp, compact) {
1940
- if (compact) {
1941
- const previousCompactCid = fp.compact[fp.compact.length - 1];
1942
- fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1943
- this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1944
- await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1945
- } else {
1946
- this.carLog.unshift(cids);
1579
+ async subtleKey(key) {
1580
+ const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
1581
+ if (extractable) {
1582
+ this._warnOnce.once(
1583
+ () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
1584
+ );
1947
1585
  }
1586
+ return await this.rt.crypto.importKey(
1587
+ "raw",
1588
+ // raw or jwk
1589
+ import_base584.base58btc.decode(key),
1590
+ // hexStringToUint8Array(key), // raw data
1591
+ "AES-GCM",
1592
+ extractable,
1593
+ ["encrypt", "decrypt"]
1594
+ );
1948
1595
  }
1949
- async cacheTransaction(t) {
1950
- for await (const block of t.entries()) {
1951
- const sBlock = block.cid.toString();
1952
- if (!this.getBlockCache.has(sBlock)) {
1953
- this.getBlockCache.set(sBlock, block);
1596
+ async ensureKeyFromUrl(url, keyFactory) {
1597
+ const storeKey = url.getParam("storekey" /* STORE_KEY */);
1598
+ if (storeKey === "insecure") {
1599
+ return import_cement6.Result.Ok(url);
1600
+ }
1601
+ if (!storeKey) {
1602
+ const keyName = `@${keyFactory()}@`;
1603
+ const ret = await this.getNamedKey(keyName);
1604
+ if (ret.isErr()) {
1605
+ return ret;
1954
1606
  }
1607
+ const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
1608
+ return import_cement6.Result.Ok(urb.URI());
1955
1609
  }
1956
- }
1957
- async cacheCarReader(carCidStr, reader) {
1958
- if (this.processedCars.has(carCidStr)) return;
1959
- this.processedCars.add(carCidStr);
1960
- for await (const block of reader.blocks()) {
1961
- const sBlock = block.cid.toString();
1962
- if (!this.getBlockCache.has(sBlock)) {
1963
- this.getBlockCache.set(sBlock, block);
1610
+ if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
1611
+ const ret = await this.getNamedKey(storeKey);
1612
+ if (ret.isErr()) {
1613
+ return ret;
1964
1614
  }
1965
1615
  }
1616
+ return import_cement6.Result.Ok(url);
1966
1617
  }
1967
- async removeCidsForCompact(cid) {
1968
- const carHeader = await this.loadCarHeaderFromMeta({
1969
- cars: [cid]
1618
+ async toKeyWithFingerPrint(keyStr) {
1619
+ const material = import_base584.base58btc.decode(keyStr);
1620
+ const key = await this.subtleKey(keyStr);
1621
+ const fpr = await this.rt.crypto.digestSHA256(material);
1622
+ return import_cement6.Result.Ok({
1623
+ key,
1624
+ fingerPrint: import_base584.base58btc.encode(new Uint8Array(fpr))
1970
1625
  });
1971
- for (const cids of carHeader.compact) {
1972
- for (const cid2 of cids) {
1973
- await (await this.carStore()).remove(cid2);
1974
- }
1975
- }
1976
1626
  }
1977
- // async flushCars() {
1978
- // await this.ready
1979
- // // for each cid in car log, make a dbMeta
1980
- // for (const cid of this.carLog) {
1981
- // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1982
- // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1983
- // }
1984
- // }
1985
- async *entries(cache2 = true) {
1986
- await this.ready();
1987
- if (cache2) {
1988
- for (const [, block] of this.getBlockCache) {
1989
- yield block;
1627
+ async setNamedKey(name, key) {
1628
+ return this._seq.add(() => this._setNamedKey(name, key));
1629
+ }
1630
+ // avoid deadlock
1631
+ async _setNamedKey(name, key) {
1632
+ const item = {
1633
+ name,
1634
+ key
1635
+ };
1636
+ const bag = await this.rt.getBag();
1637
+ this.logger.Debug().Str("name", name).Msg("setNamedKey");
1638
+ await bag.set(name, item);
1639
+ return await this.toKeyWithFingerPrint(item.key);
1640
+ }
1641
+ async getNamedExtractableKey(name, failIfNotFound = false) {
1642
+ const ret = await this.getNamedKey(name, failIfNotFound);
1643
+ if (ret.isErr()) {
1644
+ return ret;
1645
+ }
1646
+ const named = ret.Ok();
1647
+ return import_cement6.Result.Ok({
1648
+ ...named,
1649
+ extract: async () => {
1650
+ const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
1651
+ return {
1652
+ key: ext,
1653
+ keyStr: import_base584.base58btc.encode(ext)
1654
+ };
1990
1655
  }
1991
- } else {
1992
- for (const [, block] of this.getBlockCache) {
1993
- yield block;
1656
+ });
1657
+ }
1658
+ async getNamedKey(name, failIfNotFound = false) {
1659
+ const id = this.rt.sthis.nextId(4).str;
1660
+ return this._seq.add(async () => {
1661
+ const bag = await this.rt.getBag();
1662
+ const named = await bag.get(name);
1663
+ if (named) {
1664
+ const fpr = await this.toKeyWithFingerPrint(named.key);
1665
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
1666
+ return fpr;
1994
1667
  }
1995
- for (const cids of this.carLog) {
1996
- for (const cid of cids) {
1997
- const reader = await this.loadCar(cid);
1998
- if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1999
- for await (const block of reader.blocks()) {
2000
- const sCid = block.cid.toString();
2001
- if (!this.getBlockCache.has(sCid)) {
2002
- yield block;
2003
- }
2004
- }
2005
- }
1668
+ if (failIfNotFound) {
1669
+ this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
1670
+ return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
2006
1671
  }
2007
- }
1672
+ const ret = await this._setNamedKey(name, import_base584.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
1673
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
1674
+ return ret;
1675
+ });
2008
1676
  }
2009
- async getBlock(cid) {
2010
- await this.ready();
2011
- const sCid = cid.toString();
2012
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2013
- const getCarCid = async (carCid) => {
2014
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2015
- const reader = await this.loadCar(carCid);
2016
- if (!reader) {
2017
- throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1677
+ };
1678
+ var keyBagProviderFactories = new Map(
1679
+ [
1680
+ {
1681
+ protocol: "file:",
1682
+ factory: async (url, sthis) => {
1683
+ const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1684
+ return new KeyBagProviderFile2(url, sthis);
2018
1685
  }
2019
- await this.cacheCarReader(carCid.toString(), reader).catch(() => {
2020
- return;
2021
- });
2022
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2023
- throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
2024
- };
2025
- const getCompactCarCids = async (carCid) => {
2026
- const reader = await this.loadCar(carCid);
2027
- if (!reader) {
2028
- throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1686
+ },
1687
+ {
1688
+ protocol: "indexdb:",
1689
+ factory: async (url, sthis) => {
1690
+ const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1691
+ return new KeyBagProviderIndexDB2(url, sthis);
2029
1692
  }
2030
- const header = await parseCarFile(reader, this.logger);
2031
- const compacts = header.compact;
2032
- let got2;
2033
- const batchSize2 = 5;
2034
- for (let i = 0; i < compacts.length; i += batchSize2) {
2035
- const promises = [];
2036
- for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
2037
- for (const cid2 of compacts[j]) {
2038
- promises.push(getCarCid(cid2));
2039
- }
2040
- }
2041
- try {
2042
- got2 = await Promise.any(promises);
2043
- } catch {
2044
- }
2045
- if (got2) break;
2046
- }
2047
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2048
- throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
2049
- };
2050
- let got;
2051
- const batchSize = 5;
2052
- for (let i = 0; i < this.carLog.length; i += batchSize) {
2053
- const batch = this.carLog.slice(i, i + batchSize);
2054
- const promises = batch.flatMap((slice) => slice.map(getCarCid));
2055
- try {
2056
- got = await Promise.any(promises);
2057
- } catch {
2058
- }
2059
- if (got) break;
2060
1693
  }
2061
- if (!got) {
2062
- try {
2063
- got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
2064
- } catch {
1694
+ ].map((i) => [i.protocol, i])
1695
+ );
1696
+ function registerKeyBagProviderFactory(item) {
1697
+ const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
1698
+ keyBagProviderFactories.set(protocol, {
1699
+ ...item,
1700
+ protocol
1701
+ });
1702
+ }
1703
+ function defaultKeyBagOpts(sthis, kbo) {
1704
+ kbo = kbo || {};
1705
+ if (kbo.keyRuntime) {
1706
+ return kbo.keyRuntime;
1707
+ }
1708
+ const logger = ensureLogger(sthis, "KeyBag");
1709
+ let url;
1710
+ if (kbo.url) {
1711
+ url = import_cement6.URI.from(kbo.url);
1712
+ logger.Debug().Url(url).Msg("from opts");
1713
+ } else {
1714
+ let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
1715
+ if ((0, import_cement6.runtimeFn)().isBrowser) {
1716
+ url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
1717
+ } else {
1718
+ if (!bagFnameOrUrl) {
1719
+ const home = sthis.env.get("HOME");
1720
+ bagFnameOrUrl = `${home}/.fireproof/keybag`;
1721
+ url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
1722
+ } else {
1723
+ url = import_cement6.URI.from(bagFnameOrUrl);
2065
1724
  }
2066
1725
  }
2067
- return got;
1726
+ logger.Debug().Url(url).Msg("from env");
2068
1727
  }
2069
- async loadCar(cid) {
2070
- if (!this.carStore) {
2071
- throw this.logger.Error().Msg("car store not initialized").AsError();
1728
+ let keyProviderFactory;
1729
+ switch (url.protocol) {
1730
+ case "file:":
1731
+ keyProviderFactory = async () => {
1732
+ const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1733
+ return new KeyBagProviderFile2(url, sthis);
1734
+ };
1735
+ break;
1736
+ case "indexdb:":
1737
+ keyProviderFactory = async () => {
1738
+ const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1739
+ return new KeyBagProviderIndexDB2(url, sthis);
1740
+ };
1741
+ break;
1742
+ case "memory:":
1743
+ keyProviderFactory = async () => {
1744
+ const { KeyBagProviderMemory: KeyBagProviderMemory2 } = await Promise.resolve().then(() => (init_key_bag_memory(), key_bag_memory_exports));
1745
+ return new KeyBagProviderMemory2(url, sthis);
1746
+ };
1747
+ break;
1748
+ default:
1749
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
1750
+ }
1751
+ if (url.hasParam("masterkey")) {
1752
+ throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
1753
+ }
1754
+ return {
1755
+ url,
1756
+ crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
1757
+ sthis,
1758
+ logger,
1759
+ keyLength: kbo.keyLength || 16,
1760
+ getBag: keyProviderFactory,
1761
+ id: () => {
1762
+ return url.toString();
2072
1763
  }
2073
- const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
2074
- return loaded;
1764
+ };
1765
+ }
1766
+ var _keyBags = new import_cement6.KeyedResolvOnce();
1767
+ async function getKeyBag(sthis, kbo = {}) {
1768
+ await sthis.start();
1769
+ const rt = defaultKeyBagOpts(sthis, kbo);
1770
+ return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
1771
+ }
1772
+
1773
+ // src/blockstore/meta-key-helper.ts
1774
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
1775
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
1776
+ if (!crdtEntries.length) {
1777
+ sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
1778
+ return [];
2075
1779
  }
2076
- async makeDecoderAndCarReader(cid, local, remote) {
2077
- const cidsString = cid.toString();
2078
- let loadedCar = void 0;
2079
- let activeStore = local;
2080
- try {
2081
- this.logger.Debug().Str("cid", cidsString).Msg("loading car");
2082
- loadedCar = await local.load(cid);
2083
- this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
2084
- } catch (e) {
2085
- if (remote) {
2086
- const remoteCar = await remote.load(cid);
2087
- if (remoteCar) {
2088
- this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
2089
- await local.save(remoteCar);
2090
- loadedCar = remoteCar;
2091
- activeStore = remote;
1780
+ if (!crdtEntries.map) {
1781
+ sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
1782
+ return [];
1783
+ }
1784
+ return Promise.all(
1785
+ crdtEntries.map(async (crdtEntry) => {
1786
+ const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
1787
+ const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
1788
+ return {
1789
+ eventCid: eventBlock.cid,
1790
+ parents: crdtEntry.parents,
1791
+ dbMeta
1792
+ };
1793
+ })
1794
+ );
1795
+ }
1796
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
1797
+ try {
1798
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
1799
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
1800
+ if (keyInfo.length) {
1801
+ const dbMeta = keyInfo[0].dbMeta;
1802
+ if (dbMeta.key) {
1803
+ const kb = await getKeyBag(sthis);
1804
+ const keyName = getStoreKeyName(uri);
1805
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
1806
+ if (res.isErr()) {
1807
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
1808
+ throw res.Err();
2092
1809
  }
2093
- } else {
2094
- this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
2095
1810
  }
1811
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
1812
+ return import_cement7.Result.Ok(dbMeta);
2096
1813
  }
2097
- if (!loadedCar) {
2098
- throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
2099
- }
2100
- const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha23.sha256, codec: (await activeStore.keyedCrypto()).codec() });
2101
- const rawReader = await import_car.CarReader.fromBytes(bytes.value);
2102
- const readerP = Promise.resolve(rawReader);
2103
- const cachedReaderP = readerP.then(async (reader) => {
2104
- await this.cacheCarReader(cidsString, reader).catch((e) => {
2105
- this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
2106
- return;
2107
- });
2108
- return reader;
2109
- });
2110
- this.carReaders.set(cidsString, cachedReaderP);
2111
- return readerP;
1814
+ sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
1815
+ return import_cement7.Result.Ok(void 0);
1816
+ } catch (error) {
1817
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
1818
+ return import_cement7.Result.Err(error);
2112
1819
  }
2113
- //What if instead it returns an Array of CarHeader
2114
- async storesLoadCar(cid, local, remote) {
2115
- const cidsString = cid.toString();
2116
- let dacr = this.carReaders.get(cidsString);
2117
- if (!dacr) {
2118
- dacr = this.makeDecoderAndCarReader(cid, local, remote);
2119
- this.carReaders.set(cidsString, dacr);
1820
+ }
1821
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
1822
+ try {
1823
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
1824
+ const keyName = getStoreKeyName(uri);
1825
+ const kb = await getKeyBag(sthis);
1826
+ const res = await kb.getNamedExtractableKey(keyName, true);
1827
+ if (res.isErr()) {
1828
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
1829
+ throw res.Err();
2120
1830
  }
2121
- return dacr;
1831
+ const keyData = await res.Ok().extract();
1832
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
1833
+ const { dbMeta, parents } = dbMetas[0];
1834
+ const parentLinks = parents.map((p) => import_multiformats.CID.parse(p));
1835
+ dbMeta.key = keyData.keyStr;
1836
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
1837
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
1838
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
1839
+ return import_cement7.Result.Ok(encoded);
1840
+ } catch (error) {
1841
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
1842
+ return import_cement7.Result.Err(error);
2122
1843
  }
2123
- async getMoreReaders(cids) {
2124
- const limit = (0, import_p_limit.default)(5);
2125
- const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
2126
- await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1844
+ }
1845
+ function getStoreKeyName(url) {
1846
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
1847
+ const idx = url.getParam("index");
1848
+ if (idx) {
1849
+ storeKeyName.push(idx);
2127
1850
  }
2128
- };
1851
+ storeKeyName.push("data");
1852
+ return `@${storeKeyName.join(":")}@`;
1853
+ }
1854
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
1855
+ const event = await import_clock.EventBlock.create(
1856
+ {
1857
+ dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
1858
+ },
1859
+ parents
1860
+ );
1861
+ return event;
1862
+ }
1863
+ async function encodeEventsWithParents(sthis, events, parents) {
1864
+ const crdtEntries = events.map((event) => {
1865
+ const base64String = import_base64.base64pad.encode(event.bytes);
1866
+ return {
1867
+ cid: event.cid.toString(),
1868
+ data: base64String,
1869
+ parents: parents.map((p) => p.toString())
1870
+ };
1871
+ });
1872
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
1873
+ }
2129
1874
 
2130
- // src/runtime/keyed-crypto.ts
2131
- var keyed_crypto_exports = {};
2132
- __export(keyed_crypto_exports, {
2133
- BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
2134
- keyedCryptoFactory: () => keyedCryptoFactory
1875
+ // src/blockstore/store.ts
1876
+ var import_p_retry = __toESM(require("p-retry"), 1);
1877
+ var import_p_map = __toESM(require("p-map"), 1);
1878
+
1879
+ // src/blockstore/loader.ts
1880
+ var import_p_limit = __toESM(require("p-limit"), 1);
1881
+ var import_car = require("@ipld/car");
1882
+ var import_cement9 = require("@adviser/cement");
1883
+
1884
+ // src/runtime/wait-pr-multiformats/block.ts
1885
+ var block_exports = {};
1886
+ __export(block_exports, {
1887
+ Block: () => Block,
1888
+ create: () => create,
1889
+ createUnsafe: () => createUnsafe,
1890
+ decode: () => decode3,
1891
+ encode: () => encode3
2135
1892
  });
1893
+ var import_multiformats2 = require("multiformats");
1894
+ var import_block = require("multiformats/block");
1895
+ var Block = import_block.Block;
1896
+ async function decode3({
1897
+ bytes,
1898
+ codec: codec3,
1899
+ hasher: hasher7
1900
+ }) {
1901
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
1902
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
1903
+ const value = await Promise.resolve(codec3.decode(bytes));
1904
+ const hash = await hasher7.digest(bytes);
1905
+ const cid = import_multiformats2.CID.create(1, codec3.code, hash);
1906
+ return new import_block.Block({ value, bytes, cid });
1907
+ }
1908
+ async function encode3({
1909
+ value,
1910
+ codec: codec3,
1911
+ hasher: hasher7
1912
+ }) {
1913
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
1914
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
1915
+ const bytes = await Promise.resolve(codec3.encode(value));
1916
+ const hash = await hasher7.digest(bytes);
1917
+ const cid = import_multiformats2.CID.create(1, codec3.code, hash);
1918
+ return new import_block.Block({ value, bytes, cid });
1919
+ }
1920
+ async function create({
1921
+ bytes,
1922
+ cid,
1923
+ hasher: hasher7,
1924
+ codec: codec3
1925
+ }) {
1926
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
1927
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
1928
+ const value = await Promise.resolve(codec3.decode(bytes));
1929
+ const hash = await hasher7.digest(bytes);
1930
+ if (!import_multiformats2.bytes.equals(cid.multihash.bytes, hash.bytes)) {
1931
+ throw new Error("CID hash does not match bytes");
1932
+ }
1933
+ return createUnsafe({
1934
+ bytes,
1935
+ cid,
1936
+ value,
1937
+ codec: codec3
1938
+ });
1939
+ }
1940
+ async function createUnsafe({
1941
+ bytes,
1942
+ cid,
1943
+ value: maybeValue,
1944
+ codec: codec3
1945
+ }) {
1946
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
1947
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
1948
+ return new Block({
1949
+ cid,
1950
+ bytes,
1951
+ value
1952
+ });
1953
+ }
1954
+
1955
+ // src/blockstore/loader-helpers.ts
1956
+ var import_sha22 = require("multiformats/hashes/sha2");
1957
+ var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
1958
+ async function parseCarFile(reader, logger) {
1959
+ const roots = await reader.getRoots();
1960
+ const header = await reader.get(roots[0]);
1961
+ if (!header) throw logger.Error().Msg("missing header block").AsError();
1962
+ const dec = await decode3({ bytes: header.bytes, hasher: import_sha22.sha256, codec: dagCodec });
1963
+ const fpvalue = dec.value;
1964
+ if (fpvalue && !fpvalue.fp) {
1965
+ throw logger.Error().Msg("missing fp").AsError();
1966
+ }
1967
+ return fpvalue.fp;
1968
+ }
1969
+
1970
+ // src/blockstore/transaction.ts
1971
+ var import_block3 = require("@web3-storage/pail/block");
1972
+ init_types();
1973
+ var import_cement8 = require("@adviser/cement");
2136
1974
  init_utils();
2137
- var import_base583 = require("multiformats/bases/base58");
2138
- var import_sha24 = require("multiformats/hashes/sha2");
2139
- var CBOR = __toESM(require("cborg"), 1);
2140
- var generateIV = {
2141
- random: {
1975
+ var CarTransaction = class extends import_block3.MemoryBlockstore {
1976
+ constructor(parent, opts = { add: true, noLoader: false }) {
1977
+ super();
1978
+ if (opts.add) {
1979
+ parent.transactions.add(this);
1980
+ }
1981
+ this.parent = parent;
1982
+ }
1983
+ async get(cid) {
1984
+ return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
1985
+ }
1986
+ async superGet(cid) {
1987
+ return super.get(cid);
1988
+ }
1989
+ };
1990
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1991
+ const logger = ensureLogger(sthis, component, ctx);
1992
+ return {
2142
1993
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
2143
- calc: async (ko, crypto, data) => {
2144
- return crypto.randomBytes(ko.ivLength);
1994
+ applyMeta: (meta, snap) => {
1995
+ return Promise.resolve();
2145
1996
  },
2146
1997
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
2147
- verify: async (ko, crypto, iv, data) => {
2148
- return true;
1998
+ compact: async (blocks) => {
1999
+ return {};
2000
+ },
2001
+ autoCompact: 100,
2002
+ public: false,
2003
+ // name: undefined,
2004
+ threshold: 1e3 * 1e3,
2005
+ ...opts,
2006
+ logger,
2007
+ keyBag: opts.keyBag || {},
2008
+ crypto: (0, import_cement8.toCryptoRuntime)(opts.crypto),
2009
+ storeUrls: opts.storeUrls,
2010
+ // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
2011
+ // store,
2012
+ storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
2013
+ };
2014
+ }
2015
+ function blockstoreFactory(sthis, opts) {
2016
+ return new EncryptedBlockstore(sthis, opts);
2017
+ }
2018
+ var BaseBlockstore = class {
2019
+ constructor(ebOpts) {
2020
+ this.transactions = /* @__PURE__ */ new Set();
2021
+ this.sthis = ensureSuperThis(ebOpts);
2022
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
2023
+ this.sthis = ensureSuperThis(ebOpts);
2024
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
2025
+ this.logger = this.ebOpts.logger;
2026
+ }
2027
+ // readonly name?: string;
2028
+ // ready: Promise<void>;
2029
+ ready() {
2030
+ return Promise.resolve();
2031
+ }
2032
+ async close() {
2033
+ }
2034
+ async destroy() {
2035
+ }
2036
+ async compact() {
2037
+ }
2038
+ async get(cid) {
2039
+ if (!cid) throw this.logger.Error().Msg("required cid").AsError();
2040
+ for (const f of this.transactions) {
2041
+ const v = await f.superGet(cid);
2042
+ if (v) return v;
2149
2043
  }
2150
- },
2151
- hash: {
2152
- calc: async (ko, crypto, data) => {
2153
- const hash = await import_sha24.sha256.digest(data);
2154
- const hashBytes = new Uint8Array(hash.bytes);
2155
- const hashArray = new Uint8Array(ko.ivLength);
2156
- for (let i = 0; i < hashBytes.length; i++) {
2157
- hashArray[i % ko.ivLength] ^= hashBytes[i];
2044
+ }
2045
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2046
+ async put(cid, block) {
2047
+ throw this.logger.Error().Msg("use a transaction to put").AsError();
2048
+ }
2049
+ // TransactionMeta
2050
+ async transaction(fn, _opts) {
2051
+ this.logger.Debug().Msg("enter transaction");
2052
+ const t = new CarTransaction(this, _opts);
2053
+ this.logger.Debug().Msg("post CarTransaction");
2054
+ const done = await fn(t);
2055
+ this.logger.Debug().Msg("post fn");
2056
+ this.lastTxMeta = done;
2057
+ return { t, meta: done };
2058
+ }
2059
+ openTransaction(opts = { add: true, noLoader: false }) {
2060
+ return new CarTransaction(this, opts);
2061
+ }
2062
+ async commitTransaction(t, done, opts) {
2063
+ if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
2064
+ const cars = await this.loader?.commit(t, done, opts);
2065
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
2066
+ setTimeout(() => void this.compact(), 10);
2067
+ }
2068
+ if (cars) {
2069
+ this.transactions.delete(t);
2070
+ return { meta: done, cars, t };
2071
+ }
2072
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
2073
+ }
2074
+ async *entries() {
2075
+ const seen = /* @__PURE__ */ new Set();
2076
+ for (const t of this.transactions) {
2077
+ for await (const blk of t.entries()) {
2078
+ if (seen.has(blk.cid.toString())) continue;
2079
+ seen.add(blk.cid.toString());
2080
+ yield blk;
2158
2081
  }
2159
- return hashArray;
2160
- },
2161
- verify: async function(ko, crypto, iv, data) {
2162
- return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
2163
2082
  }
2164
2083
  }
2165
2084
  };
2166
- function getGenerateIVFn(url, opts) {
2167
- const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
2168
- return generateIV[ivhash] || generateIV["hash"];
2169
- }
2170
- var BlockIvKeyIdCodec = class {
2171
- constructor(ko, iv, opts) {
2172
- this.code = 3147065;
2173
- this.name = "Fireproof@encrypted-block:aes-gcm";
2174
- this.ko = ko;
2175
- this.iv = iv;
2176
- this.opts = opts || {};
2177
- }
2178
- async encode(data) {
2179
- const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
2180
- const { iv } = this.ko.algo(calcIv);
2181
- const fprt = await this.ko.fingerPrint();
2182
- const keyId = import_base583.base58btc.decode(fprt);
2183
- this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
2184
- return CBOR.encode({
2185
- iv,
2186
- keyId,
2187
- data: await this.ko._encrypt({ iv, bytes: data })
2085
+ var EncryptedBlockstore = class extends BaseBlockstore {
2086
+ constructor(sthis, ebOpts) {
2087
+ super(ebOpts);
2088
+ this.compacting = false;
2089
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
2090
+ this: 1
2188
2091
  });
2092
+ this.loader = new Loader(sthis, ebOpts);
2189
2093
  }
2190
- async decode(abytes) {
2191
- let bytes;
2192
- if (abytes instanceof Uint8Array) {
2193
- bytes = abytes;
2194
- } else {
2195
- bytes = new Uint8Array(abytes);
2094
+ ready() {
2095
+ return this.loader.ready();
2096
+ }
2097
+ close() {
2098
+ return this.loader.close();
2099
+ }
2100
+ destroy() {
2101
+ return this.loader.destroy();
2102
+ }
2103
+ async get(cid) {
2104
+ const got = await super.get(cid);
2105
+ if (got) return got;
2106
+ if (!this.loader) {
2107
+ return;
2196
2108
  }
2197
- const { iv, keyId, data } = CBOR.decode(bytes);
2198
- const fprt = await this.ko.fingerPrint();
2199
- this.ko.logger.Debug().Str("fp", import_base583.base58btc.encode(keyId)).Msg("decode");
2200
- if (import_base583.base58btc.encode(keyId) !== fprt) {
2201
- throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base583.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
2109
+ return falsyToUndef(await this.loader.getBlock(cid));
2110
+ }
2111
+ async transaction(fn, opts = { noLoader: false }) {
2112
+ this.logger.Debug().Msg("enter transaction");
2113
+ const { t, meta: done } = await super.transaction(fn);
2114
+ this.logger.Debug().Msg("post super.transaction");
2115
+ const cars = await this.loader.commit(t, done, opts);
2116
+ this.logger.Debug().Msg("post this.loader.commit");
2117
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
2118
+ setTimeout(() => void this.compact(), 10);
2202
2119
  }
2203
- const result = await this.ko._decrypt({ iv, bytes: data });
2204
- if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
2205
- throw this.ko.logger.Error().Msg("iv missmatch").AsError();
2120
+ if (cars) {
2121
+ this.transactions.delete(t);
2122
+ return { meta: done, cars, t };
2206
2123
  }
2207
- return result;
2124
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
2208
2125
  }
2209
- };
2210
- var keyedCrypto = class {
2211
- constructor(url, key, cyopt, sthis) {
2212
- this.ivLength = 12;
2213
- this.isEncrypting = true;
2214
- this.logger = ensureLogger(sthis, "keyedCrypto");
2215
- this.crypto = cyopt;
2216
- this.key = key;
2217
- this.url = url;
2126
+ async getFile(car, cid) {
2127
+ await this.ready();
2128
+ if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
2129
+ const reader = await this.loader.loadFileCar(
2130
+ car
2131
+ /*, isPublic */
2132
+ );
2133
+ const block = await reader.get(cid);
2134
+ if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
2135
+ return block.bytes;
2218
2136
  }
2219
- fingerPrint() {
2220
- return Promise.resolve(this.key.fingerPrint);
2137
+ async compact() {
2138
+ await this.ready();
2139
+ if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
2140
+ if (this.loader.carLog.length < 2) return;
2141
+ const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
2142
+ if (!compactFn || this.compacting) return;
2143
+ const blockLog = new CompactionFetcher(this);
2144
+ this.compacting = true;
2145
+ const meta = await compactFn(blockLog);
2146
+ await this.loader?.commit(blockLog.loggedBlocks, meta, {
2147
+ compact: true,
2148
+ noLoader: true
2149
+ });
2150
+ this.compacting = false;
2221
2151
  }
2222
- codec(iv, opts) {
2223
- return new BlockIvKeyIdCodec(this, iv, opts);
2152
+ async defaultCompact(blocks, logger) {
2153
+ if (!this.loader) {
2154
+ throw logger.Error().Msg("no loader").AsError();
2155
+ }
2156
+ if (!this.lastTxMeta) {
2157
+ throw logger.Error().Msg("no lastTxMeta").AsError();
2158
+ }
2159
+ for await (const blk of this.loader.entries(false)) {
2160
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
2161
+ }
2162
+ for (const t of this.transactions) {
2163
+ for await (const blk of t.entries()) {
2164
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
2165
+ }
2166
+ }
2167
+ return this.lastTxMeta;
2224
2168
  }
2225
- algo(iv) {
2226
- return {
2227
- name: "AES-GCM",
2228
- iv: iv || this.crypto.randomBytes(this.ivLength),
2229
- tagLength: 128
2230
- };
2169
+ async *entries() {
2170
+ for await (const blk of this.loader.entries()) {
2171
+ yield blk;
2172
+ }
2231
2173
  }
2232
- async _decrypt(data) {
2233
- this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
2234
- return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
2174
+ };
2175
+ var CompactionFetcher = class {
2176
+ constructor(blocks) {
2177
+ this.blockstore = blocks;
2178
+ this.loggedBlocks = new CarTransaction(blocks);
2235
2179
  }
2236
- async _encrypt(data) {
2237
- this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
2238
- const a = this.algo(data.iv);
2239
- return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
2180
+ async get(cid) {
2181
+ const block = await this.blockstore.get(cid);
2182
+ if (block) this.loggedBlocks.putSync(cid, block.bytes);
2183
+ return falsyToUndef(block);
2240
2184
  }
2241
2185
  };
2242
- var nullCodec = class {
2243
- constructor() {
2244
- this.code = 0;
2245
- this.name = "Fireproof@unencrypted-block";
2186
+
2187
+ // src/blockstore/commitor.ts
2188
+ var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
2189
+ var import_sha23 = require("multiformats/hashes/sha2");
2190
+ var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
2191
+ async function encodeCarFile(roots, t, codec3) {
2192
+ let size = 0;
2193
+ const headerSize = CBW.headerLength({ roots });
2194
+ size += headerSize;
2195
+ for (const { cid, bytes } of t.entries()) {
2196
+ size += CBW.blockLength({ cid, bytes });
2246
2197
  }
2247
- encode(data) {
2248
- return data;
2198
+ const buffer = new Uint8Array(size);
2199
+ const writer = CBW.createWriter(buffer, { headerSize });
2200
+ for (const r of roots) {
2201
+ writer.addRoot(r);
2249
2202
  }
2250
- decode(data) {
2251
- return data;
2203
+ for (const { cid, bytes } of t.entries()) {
2204
+ writer.write({ cid, bytes });
2252
2205
  }
2253
- };
2254
- var noCrypto = class {
2255
- constructor(url, cyrt, sthis) {
2256
- this.ivLength = 0;
2257
- this.code = 0;
2258
- this.name = "Fireproof@unencrypted-block";
2259
- this.isEncrypting = false;
2260
- this._fingerPrint = "noCrypto:" + Math.random();
2261
- this.logger = ensureLogger(sthis, "noCrypto");
2262
- this.crypto = cyrt;
2263
- this.url = url;
2206
+ writer.close();
2207
+ return await encode3({ value: writer.bytes, hasher: import_sha23.sha256, codec: codec3 });
2208
+ }
2209
+ async function createCarFile(encoder, cid, t) {
2210
+ return encodeCarFile([cid], t, encoder);
2211
+ }
2212
+ async function commitFiles(fileStore, walStore, t, done) {
2213
+ const { files: roots } = makeFileCarHeader(done);
2214
+ const cids = [];
2215
+ const codec3 = (await fileStore.keyedCrypto()).codec();
2216
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
2217
+ for (const car of cars) {
2218
+ const { cid, bytes } = car;
2219
+ await fileStore.save({ cid, bytes });
2220
+ await walStore.enqueueFile(
2221
+ cid
2222
+ /*, !!opts.public*/
2223
+ );
2224
+ cids.push(cid);
2264
2225
  }
2265
- fingerPrint() {
2266
- return Promise.resolve(this._fingerPrint);
2226
+ return cids;
2227
+ }
2228
+ function makeFileCarHeader(result) {
2229
+ const files = [];
2230
+ for (const [, meta] of Object.entries(result.files || {})) {
2231
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
2232
+ files.push(meta.cid);
2233
+ }
2267
2234
  }
2268
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
2269
- codec(iv) {
2270
- return new nullCodec();
2235
+ return { ...result, files };
2236
+ }
2237
+ async function prepareCarFilesFiles(encoder, roots, t) {
2238
+ return [await encodeCarFile(roots, t, encoder)];
2239
+ }
2240
+ function makeCarHeader(meta, cars, compact = false) {
2241
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
2242
+ return { ...coreHeader, meta };
2243
+ }
2244
+ async function encodeCarHeader(fp) {
2245
+ return await encode3({
2246
+ value: { fp },
2247
+ hasher: import_sha23.sha256,
2248
+ codec: dagCodec2
2249
+ });
2250
+ }
2251
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
2252
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
2253
+ const rootBlock = await encodeCarHeader(fp);
2254
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
2255
+ const cids = [];
2256
+ for (const car of cars) {
2257
+ const { cid, bytes } = car;
2258
+ await params.carStore.save({ cid, bytes });
2259
+ cids.push(cid);
2271
2260
  }
2272
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
2273
- algo(iv) {
2274
- return {
2275
- name: "noCrypto",
2276
- iv: new Uint8Array(),
2277
- tagLength: 0
2278
- };
2261
+ const newDbMeta = { cars: cids };
2262
+ await params.WALStore.enqueue(newDbMeta, opts);
2263
+ await params.metaStore.save(newDbMeta);
2264
+ return { cgrp: cids, header: fp };
2265
+ }
2266
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
2267
+ const carFiles = [];
2268
+ threshold = threshold || 128e3 * 8;
2269
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
2270
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
2271
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
2272
+ let cidRootBlock = rootBlock;
2273
+ for (const { cid, bytes } of t.entries()) {
2274
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
2275
+ if (newsize >= threshold) {
2276
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
2277
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
2278
+ clonedt.putSync(cid, bytes);
2279
+ cidRootBlock = { cid, bytes };
2280
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
2281
+ } else {
2282
+ clonedt.putSync(cid, bytes);
2283
+ }
2279
2284
  }
2280
- _decrypt() {
2281
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
2285
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
2286
+ return carFiles;
2287
+ }
2288
+
2289
+ // src/blockstore/loader.ts
2290
+ var import_sha24 = require("multiformats/hashes/sha2");
2291
+
2292
+ // src/blockstore/task-manager.ts
2293
+ init_utils();
2294
+ var TaskManager = class {
2295
+ constructor(sthis, callback) {
2296
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
2297
+ this.queue = [];
2298
+ this.isProcessing = false;
2299
+ this.logger = ensureLogger(sthis, "TaskManager");
2300
+ this.callback = callback;
2282
2301
  }
2283
- _encrypt() {
2284
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
2302
+ async handleEvent(cid, parents, dbMeta) {
2303
+ for (const parent of parents) {
2304
+ this.eventsWeHandled.add(parent.toString());
2305
+ }
2306
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
2307
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
2308
+ void this.processQueue();
2285
2309
  }
2286
- };
2287
- async function keyedCryptoFactory(url, kb, sthis) {
2288
- const storekey = url.getParam("storekey");
2289
- if (storekey && storekey !== "insecure") {
2290
- let rkey = await kb.getNamedKey(storekey, true);
2291
- if (rkey.isErr()) {
2292
- try {
2293
- rkey = await kb.toKeyWithFingerPrint(storekey);
2294
- } catch (e) {
2295
- throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
2310
+ async processQueue() {
2311
+ if (this.isProcessing) return;
2312
+ this.isProcessing = true;
2313
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
2314
+ const first = filteredQueue[0];
2315
+ if (!first) {
2316
+ return;
2317
+ }
2318
+ try {
2319
+ await this.callback(first.dbMeta);
2320
+ this.eventsWeHandled.add(first.cid);
2321
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
2322
+ } catch (err) {
2323
+ if (first.retries++ > 3) {
2324
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
2325
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
2326
+ }
2327
+ await new Promise((resolve) => setTimeout(resolve, 50));
2328
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
2329
+ } finally {
2330
+ this.isProcessing = false;
2331
+ if (this.queue.length > 0) {
2332
+ void this.processQueue();
2296
2333
  }
2297
2334
  }
2298
- return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
2299
2335
  }
2300
- return new noCrypto(url, kb.rt.crypto, sthis);
2301
- }
2336
+ };
2302
2337
 
2303
- // src/blockstore/fragment-gateway.ts
2304
- var import_cement8 = require("@adviser/cement");
2305
- var import_base584 = require("multiformats/bases/base58");
2306
- var import_cborg = require("cborg");
2307
- init_utils();
2308
- function getFragSize(url) {
2309
- const fragSize = url.getParam("fragSize");
2310
- let ret = 0;
2311
- if (fragSize) {
2312
- ret = parseInt(fragSize);
2313
- }
2314
- if (isNaN(ret) || ret <= 0) {
2315
- ret = 0;
2338
+ // src/blockstore/loader.ts
2339
+ function carLogIncludesGroup(list, cids) {
2340
+ return list.some((arr) => {
2341
+ return arr.toString() === cids.toString();
2342
+ });
2343
+ }
2344
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
2345
+ const byString = /* @__PURE__ */ new Map();
2346
+ for (const cid of list) {
2347
+ if (remove.has(cid.toString())) continue;
2348
+ byString.set(cid.toString(), cid);
2316
2349
  }
2317
- return ret;
2350
+ return [...byString.values()];
2318
2351
  }
2319
- async function getFrags(url, innerGW, headerSize, logger) {
2320
- const fragSize = getFragSize(url);
2321
- if (!fragSize) {
2322
- const res = await innerGW.get(url);
2323
- if (res.isErr()) {
2324
- return [res];
2325
- }
2326
- const data = res.unwrap();
2327
- return [
2328
- import_cement8.Result.Ok({
2329
- fid: new Uint8Array(0),
2330
- ofs: 0,
2331
- len: data.length,
2332
- data
2352
+ var Loader = class {
2353
+ constructor(sthis, ebOpts) {
2354
+ this.commitQueue = new CommitQueue();
2355
+ this.isCompacting = false;
2356
+ this.carReaders = /* @__PURE__ */ new Map();
2357
+ this.seenCompacted = /* @__PURE__ */ new Set();
2358
+ this.processedCars = /* @__PURE__ */ new Set();
2359
+ this.carLog = [];
2360
+ this.getBlockCache = /* @__PURE__ */ new Map();
2361
+ this.seenMeta = /* @__PURE__ */ new Set();
2362
+ this.writeLimit = (0, import_p_limit.default)(1);
2363
+ this._carStore = new import_cement9.ResolveOnce();
2364
+ this._fileStore = new import_cement9.ResolveOnce();
2365
+ this._WALStore = new import_cement9.ResolveOnce();
2366
+ this._metaStore = new import_cement9.ResolveOnce();
2367
+ this.onceReady = new import_cement9.ResolveOnce();
2368
+ this.sthis = sthis;
2369
+ this.ebOpts = defaultedBlockstoreRuntime(
2370
+ sthis,
2371
+ {
2372
+ ...ebOpts
2373
+ // name,
2374
+ },
2375
+ "Loader"
2376
+ );
2377
+ this.logger = this.ebOpts.logger;
2378
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
2379
+ await this.handleDbMetasFromStore([dbMeta]);
2380
+ });
2381
+ }
2382
+ async carStore() {
2383
+ return this._carStore.once(
2384
+ async () => this.ebOpts.storeRuntime.makeDataStore({
2385
+ sthis: this.sthis,
2386
+ url: this.ebOpts.storeUrls.data,
2387
+ keybag: await this.keyBag()
2333
2388
  })
2334
- ];
2389
+ );
2335
2390
  }
2336
- const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
2337
- if (firstRaw.isErr()) {
2338
- return [firstRaw];
2391
+ async fileStore() {
2392
+ return this._fileStore.once(
2393
+ async () => this.ebOpts.storeRuntime.makeDataStore({
2394
+ sthis: this.sthis,
2395
+ url: this.ebOpts.storeUrls.file,
2396
+ keybag: await this.keyBag()
2397
+ })
2398
+ );
2339
2399
  }
2340
- const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
2341
- const blockSize = firstFragment.data.length;
2342
- const ops = [Promise.resolve(import_cement8.Result.Ok(firstFragment))];
2343
- const fidStr = import_base584.base58btc.encode(firstFragment.fid);
2344
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
2345
- for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
2346
- ops.push(
2347
- (async (furl, ofs2) => {
2348
- const raw2 = await innerGW.get(furl);
2349
- if (raw2.isErr()) {
2350
- return raw2;
2351
- }
2352
- const fragment = (0, import_cborg.decode)(raw2.unwrap());
2353
- if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
2354
- return import_cement8.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
2355
- }
2356
- if (fragment.ofs !== ofs2) {
2357
- return import_cement8.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
2358
- }
2359
- return import_cement8.Result.Ok(fragment);
2360
- })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
2400
+ async WALStore() {
2401
+ return this._WALStore.once(
2402
+ async () => this.ebOpts.storeRuntime.makeWALStore({
2403
+ sthis: this.sthis,
2404
+ url: this.ebOpts.storeUrls.wal,
2405
+ keybag: await this.keyBag()
2406
+ })
2361
2407
  );
2362
2408
  }
2363
- return Promise.all(ops);
2364
- }
2365
- var FragmentGateway = class {
2366
- constructor(sthis, innerGW) {
2367
- this.fidLength = 4;
2368
- this.headerSize = 32;
2369
- this.sthis = ensureSuperLog(sthis, "FragmentGateway");
2370
- this.logger = this.sthis.logger;
2371
- this.innerGW = innerGW;
2409
+ async metaStore() {
2410
+ return this._metaStore.once(
2411
+ async () => this.ebOpts.storeRuntime.makeMetaStore({
2412
+ sthis: this.sthis,
2413
+ url: this.ebOpts.storeUrls.meta,
2414
+ keybag: await this.keyBag()
2415
+ })
2416
+ );
2372
2417
  }
2373
- slicer(url, body) {
2374
- const fragSize = getFragSize(url);
2375
- if (!fragSize) {
2376
- return [this.innerGW.put(url, body)];
2377
- }
2378
- const blocksize = fragSize - this.headerSize;
2379
- if (blocksize <= 0) {
2380
- throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
2381
- }
2382
- const ops = [];
2383
- const fid = this.sthis.nextId(this.fidLength);
2384
- const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
2385
- for (let ofs = 0; ofs < body.length; ofs += blocksize) {
2386
- const block = (0, import_cborg.encode)({
2387
- fid: fid.bin,
2388
- ofs,
2389
- len: body.length,
2390
- data: body.slice(ofs, ofs + blocksize)
2391
- });
2392
- if (block.length > fragSize) {
2393
- throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
2418
+ keyBag() {
2419
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
2420
+ }
2421
+ async ready() {
2422
+ return this.onceReady.once(async () => {
2423
+ const metas = await (await this.metaStore()).load();
2424
+ if (this.ebOpts.meta) {
2425
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
2426
+ } else if (metas) {
2427
+ await this.handleDbMetasFromStore(metas);
2394
2428
  }
2395
- ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
2429
+ });
2430
+ }
2431
+ async close() {
2432
+ await this.commitQueue.waitIdle();
2433
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
2434
+ await Promise.all(toClose.map((store) => store.close()));
2435
+ }
2436
+ async destroy() {
2437
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
2438
+ await Promise.all(toDestroy.map((store) => store.destroy()));
2439
+ }
2440
+ // async snapToCar(carCid: AnyLink | string) {
2441
+ // await this.ready
2442
+ // if (typeof carCid === 'string') {
2443
+ // carCid = CID.parse(carCid)
2444
+ // }
2445
+ // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
2446
+ // this.carLog = [carCid, ...carHeader.cars]
2447
+ // await this.getMoreReaders(carHeader.cars)
2448
+ // await this._applyCarHeader(carHeader, true)
2449
+ // }
2450
+ async handleDbMetasFromStore(metas) {
2451
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
2452
+ for (const meta of metas) {
2453
+ await this.writeLimit(async () => {
2454
+ await this.mergeDbMetaIntoClock(meta);
2455
+ });
2396
2456
  }
2397
- return ops;
2398
2457
  }
2399
- buildUrl(baseUrl, key) {
2400
- return this.innerGW.buildUrl(baseUrl, key);
2458
+ async mergeDbMetaIntoClock(meta) {
2459
+ if (this.isCompacting) {
2460
+ throw this.logger.Error().Msg("cannot merge while compacting").AsError();
2461
+ }
2462
+ if (this.seenMeta.has(meta.cars.toString())) return;
2463
+ this.seenMeta.add(meta.cars.toString());
2464
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
2465
+ return;
2466
+ }
2467
+ const carHeader = await this.loadCarHeaderFromMeta(meta);
2468
+ carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
2469
+ await this.getMoreReaders(carHeader.cars.flat());
2470
+ this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
2471
+ await this.ebOpts.applyMeta?.(carHeader.meta);
2401
2472
  }
2402
- async destroy(iurl) {
2403
- return this.innerGW.destroy(iurl);
2473
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
2474
+ // const { key } = meta;
2475
+ // if (key) {
2476
+ // await this.setKey(key);
2477
+ // }
2478
+ // }
2479
+ async loadCarHeaderFromMeta({ cars: cids }) {
2480
+ const reader = await this.loadCar(cids[0]);
2481
+ return await parseCarFile(reader, this.logger);
2404
2482
  }
2405
- async start(url) {
2406
- this.headerSize = (0, import_cborg.encode)({
2407
- fid: this.sthis.nextId(this.fidLength).bin,
2408
- ofs: 1024 * 1024,
2409
- // 32bit
2410
- len: 16 * 1024 * 1024,
2411
- // 32bit
2412
- data: new Uint8Array(1024)
2413
- }).length - 1024;
2414
- return this.innerGW.start(url);
2483
+ // async _getKey(): Promise<string | undefined> {
2484
+ // if (this.key) return this.key;
2485
+ // // generate a random key
2486
+ // if (!this.ebOpts.public) {
2487
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
2488
+ // }
2489
+ // return this.key || undefined;
2490
+ // }
2491
+ async commitFiles(t, done) {
2492
+ await this.ready();
2493
+ const fstore = await this.fileStore();
2494
+ const wstore = await this.WALStore();
2495
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
2415
2496
  }
2416
- async close(url) {
2417
- return this.innerGW.close(url);
2497
+ async loadFileCar(cid) {
2498
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
2418
2499
  }
2419
- async put(url, body) {
2420
- await Promise.all(this.slicer(url, body));
2421
- return import_cement8.Result.Ok(void 0);
2500
+ async commit(t, done, opts = { noLoader: false, compact: false }) {
2501
+ await this.ready();
2502
+ const carStore = await this.carStore();
2503
+ const params = {
2504
+ encoder: (await carStore.keyedCrypto()).codec(),
2505
+ carLog: this.carLog,
2506
+ carStore,
2507
+ WALStore: await this.WALStore(),
2508
+ metaStore: await this.metaStore(),
2509
+ threshold: this.ebOpts.threshold
2510
+ };
2511
+ return this.commitQueue.enqueue(async () => {
2512
+ await this.cacheTransaction(t);
2513
+ const ret = await commit(params, t, done, opts);
2514
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
2515
+ return ret.cgrp;
2516
+ });
2422
2517
  }
2423
- async get(url) {
2424
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
2425
- let buffer = void 0;
2426
- for (const rfrag of rfrags) {
2427
- if (rfrag.isErr()) {
2428
- return import_cement8.Result.Err(rfrag.Err());
2429
- }
2430
- const frag = rfrag.Ok();
2431
- buffer = buffer || new Uint8Array(frag.len);
2432
- buffer.set(frag.data, frag.ofs);
2518
+ async updateCarLog(cids, fp, compact) {
2519
+ if (compact) {
2520
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
2521
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
2522
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
2523
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
2524
+ } else {
2525
+ this.carLog.unshift(cids);
2433
2526
  }
2434
- return import_cement8.Result.Ok(buffer || new Uint8Array(0));
2435
2527
  }
2436
- async subscribe(url, callback) {
2437
- if (this.innerGW.subscribe) {
2438
- return this.innerGW.subscribe(url, callback);
2439
- } else {
2440
- return import_cement8.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
2528
+ async cacheTransaction(t) {
2529
+ for await (const block of t.entries()) {
2530
+ const sBlock = block.cid.toString();
2531
+ if (!this.getBlockCache.has(sBlock)) {
2532
+ this.getBlockCache.set(sBlock, block);
2533
+ }
2441
2534
  }
2442
2535
  }
2443
- async delete(url) {
2444
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
2445
- for (const rfrag of rfrags) {
2446
- if (rfrag.isErr()) {
2447
- return import_cement8.Result.Err(rfrag.Err());
2536
+ async cacheCarReader(carCidStr, reader) {
2537
+ if (this.processedCars.has(carCidStr)) return;
2538
+ this.processedCars.add(carCidStr);
2539
+ for await (const block of reader.blocks()) {
2540
+ const sBlock = block.cid.toString();
2541
+ if (!this.getBlockCache.has(sBlock)) {
2542
+ this.getBlockCache.set(sBlock, block);
2448
2543
  }
2449
- const frag = rfrag.Ok();
2450
- const fidStr = import_base584.base58btc.encode(frag.fid);
2451
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
2452
- await this.innerGW.delete(fragUrl);
2453
2544
  }
2454
- return import_cement8.Result.Ok(void 0);
2455
2545
  }
2456
- };
2457
-
2458
- // src/blockstore/meta-key-helper.ts
2459
- var import_dag_json = require("@ipld/dag-json");
2460
- var import_clock = require("@web3-storage/pail/clock");
2461
- var import_multiformats2 = require("multiformats");
2462
- var import_base64 = require("multiformats/bases/base64");
2463
- var import_cement9 = require("@adviser/cement");
2464
- async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
2465
- const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
2466
- if (!crdtEntries.length) {
2467
- sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
2468
- return [];
2546
+ async removeCidsForCompact(cid) {
2547
+ const carHeader = await this.loadCarHeaderFromMeta({
2548
+ cars: [cid]
2549
+ });
2550
+ for (const cids of carHeader.compact) {
2551
+ for (const cid2 of cids) {
2552
+ await (await this.carStore()).remove(cid2);
2553
+ }
2554
+ }
2469
2555
  }
2470
- if (!crdtEntries.map) {
2471
- sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
2472
- return [];
2556
+ // async flushCars() {
2557
+ // await this.ready
2558
+ // // for each cid in car log, make a dbMeta
2559
+ // for (const cid of this.carLog) {
2560
+ // const dbMeta = { car: cid, key: this.key || null } as DbMeta
2561
+ // await this.remoteWAL!.enqueue(dbMeta, { public: false })
2562
+ // }
2563
+ // }
2564
+ async *entries(cache2 = true) {
2565
+ await this.ready();
2566
+ if (cache2) {
2567
+ for (const [, block] of this.getBlockCache) {
2568
+ yield block;
2569
+ }
2570
+ } else {
2571
+ for (const [, block] of this.getBlockCache) {
2572
+ yield block;
2573
+ }
2574
+ for (const cids of this.carLog) {
2575
+ for (const cid of cids) {
2576
+ const reader = await this.loadCar(cid);
2577
+ if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
2578
+ for await (const block of reader.blocks()) {
2579
+ const sCid = block.cid.toString();
2580
+ if (!this.getBlockCache.has(sCid)) {
2581
+ yield block;
2582
+ }
2583
+ }
2584
+ }
2585
+ }
2586
+ }
2473
2587
  }
2474
- return Promise.all(
2475
- crdtEntries.map(async (crdtEntry) => {
2476
- const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
2477
- const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
2478
- return {
2479
- eventCid: eventBlock.cid,
2480
- parents: crdtEntry.parents,
2481
- dbMeta
2482
- };
2483
- })
2484
- );
2485
- }
2486
- async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
2487
- try {
2488
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
2489
- const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
2490
- if (keyInfo.length) {
2491
- const dbMeta = keyInfo[0].dbMeta;
2492
- if (dbMeta.key) {
2493
- const kb = await getKeyBag(sthis);
2494
- const keyName = getStoreKeyName(uri);
2495
- const res = await kb.setNamedKey(keyName, dbMeta.key);
2496
- if (res.isErr()) {
2497
- sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
2498
- throw res.Err();
2588
+ async getBlock(cid) {
2589
+ await this.ready();
2590
+ const sCid = cid.toString();
2591
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2592
+ const getCarCid = async (carCid) => {
2593
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2594
+ const reader = await this.loadCar(carCid);
2595
+ if (!reader) {
2596
+ throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
2597
+ }
2598
+ await this.cacheCarReader(carCid.toString(), reader).catch(() => {
2599
+ return;
2600
+ });
2601
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2602
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
2603
+ };
2604
+ const getCompactCarCids = async (carCid) => {
2605
+ const reader = await this.loadCar(carCid);
2606
+ if (!reader) {
2607
+ throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
2608
+ }
2609
+ const header = await parseCarFile(reader, this.logger);
2610
+ const compacts = header.compact;
2611
+ let got2;
2612
+ const batchSize2 = 5;
2613
+ for (let i = 0; i < compacts.length; i += batchSize2) {
2614
+ const promises = [];
2615
+ for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
2616
+ for (const cid2 of compacts[j]) {
2617
+ promises.push(getCarCid(cid2));
2618
+ }
2619
+ }
2620
+ try {
2621
+ got2 = await Promise.any(promises);
2622
+ } catch {
2499
2623
  }
2624
+ if (got2) break;
2500
2625
  }
2501
- sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
2502
- return import_cement9.Result.Ok(dbMeta);
2626
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2627
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
2628
+ };
2629
+ let got;
2630
+ const batchSize = 5;
2631
+ for (let i = 0; i < this.carLog.length; i += batchSize) {
2632
+ const batch = this.carLog.slice(i, i + batchSize);
2633
+ const promises = batch.flatMap((slice) => slice.map(getCarCid));
2634
+ try {
2635
+ got = await Promise.any(promises);
2636
+ } catch {
2637
+ }
2638
+ if (got) break;
2503
2639
  }
2504
- sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
2505
- return import_cement9.Result.Ok(void 0);
2506
- } catch (error) {
2507
- sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
2508
- return import_cement9.Result.Err(error);
2640
+ if (!got) {
2641
+ try {
2642
+ got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
2643
+ } catch {
2644
+ }
2645
+ }
2646
+ return got;
2509
2647
  }
2510
- }
2511
- async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
2512
- try {
2513
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
2514
- const keyName = getStoreKeyName(uri);
2515
- const kb = await getKeyBag(sthis);
2516
- const res = await kb.getNamedExtractableKey(keyName, true);
2517
- if (res.isErr()) {
2518
- sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
2519
- throw res.Err();
2648
+ async loadCar(cid) {
2649
+ if (!this.carStore) {
2650
+ throw this.logger.Error().Msg("car store not initialized").AsError();
2520
2651
  }
2521
- const keyData = await res.Ok().extract();
2522
- const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
2523
- const { dbMeta, parents } = dbMetas[0];
2524
- const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
2525
- dbMeta.key = keyData.keyStr;
2526
- const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
2527
- const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
2528
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
2529
- return import_cement9.Result.Ok(encoded);
2530
- } catch (error) {
2531
- sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
2532
- return import_cement9.Result.Err(error);
2652
+ const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
2653
+ return loaded;
2533
2654
  }
2534
- }
2535
- function getStoreKeyName(url) {
2536
- const storeKeyName = [url.getParam("localName") || url.getParam("name")];
2537
- const idx = url.getParam("index");
2538
- if (idx) {
2539
- storeKeyName.push(idx);
2655
+ async makeDecoderAndCarReader(cid, local, remote) {
2656
+ const cidsString = cid.toString();
2657
+ let loadedCar = void 0;
2658
+ let activeStore = local;
2659
+ try {
2660
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
2661
+ loadedCar = await local.load(cid);
2662
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
2663
+ } catch (e) {
2664
+ if (remote) {
2665
+ const remoteCar = await remote.load(cid);
2666
+ if (remoteCar) {
2667
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
2668
+ await local.save(remoteCar);
2669
+ loadedCar = remoteCar;
2670
+ activeStore = remote;
2671
+ }
2672
+ } else {
2673
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
2674
+ }
2675
+ }
2676
+ if (!loadedCar) {
2677
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
2678
+ }
2679
+ const bytes = await decode3({ bytes: loadedCar.bytes, hasher: import_sha24.sha256, codec: (await activeStore.keyedCrypto()).codec() });
2680
+ const rawReader = await import_car.CarReader.fromBytes(bytes.value);
2681
+ const readerP = Promise.resolve(rawReader);
2682
+ const cachedReaderP = readerP.then(async (reader) => {
2683
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
2684
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
2685
+ return;
2686
+ });
2687
+ return reader;
2688
+ });
2689
+ this.carReaders.set(cidsString, cachedReaderP);
2690
+ return readerP;
2540
2691
  }
2541
- storeKeyName.push("data");
2542
- return `@${storeKeyName.join(":")}@`;
2543
- }
2544
- async function createDbMetaEventBlock(sthis, dbMeta, parents) {
2545
- const event = await import_clock.EventBlock.create(
2546
- {
2547
- dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
2548
- },
2549
- parents
2550
- );
2551
- return event;
2552
- }
2553
- async function encodeEventsWithParents(sthis, events, parents) {
2554
- const crdtEntries = events.map((event) => {
2555
- const base64String = import_base64.base64pad.encode(event.bytes);
2556
- return {
2557
- cid: event.cid.toString(),
2558
- data: base64String,
2559
- parents: parents.map((p) => p.toString())
2560
- };
2561
- });
2562
- return sthis.txt.encode(JSON.stringify(crdtEntries));
2563
- }
2692
+ //What if instead it returns an Array of CarHeader
2693
+ async storesLoadCar(cid, local, remote) {
2694
+ const cidsString = cid.toString();
2695
+ let dacr = this.carReaders.get(cidsString);
2696
+ if (!dacr) {
2697
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
2698
+ this.carReaders.set(cidsString, dacr);
2699
+ }
2700
+ return dacr;
2701
+ }
2702
+ async getMoreReaders(cids) {
2703
+ const limit = (0, import_p_limit.default)(5);
2704
+ const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
2705
+ await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
2706
+ }
2707
+ };
2564
2708
 
2565
2709
  // src/blockstore/store.ts
2566
- var import_p_retry = __toESM(require("p-retry"), 1);
2567
- var import_p_map = __toESM(require("p-map"), 1);
2568
2710
  function guardVersion(url) {
2569
2711
  if (!url.hasParam("version")) {
2570
2712
  return import_cement10.Result.Err(`missing version: ${url.toString()}`);
@@ -2572,16 +2714,21 @@ function guardVersion(url) {
2572
2714
  return import_cement10.Result.Ok(url);
2573
2715
  }
2574
2716
  var BaseStoreImpl = class {
2575
- constructor(name, url, opts, sthis, logger) {
2717
+ constructor(sthis, url, opts, logger) {
2576
2718
  this._onStarted = [];
2577
2719
  this._onClosed = [];
2578
- this.name = name;
2579
2720
  this._url = url;
2580
2721
  this.keybag = opts.keybag;
2722
+ this.loader = opts.loader;
2581
2723
  this.sthis = sthis;
2582
- this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
2724
+ const name = this._url.getParam("name" /* NAME */);
2725
+ if (!name) {
2726
+ throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
2727
+ }
2728
+ this.name = name;
2729
+ this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
2730
+ this.realGateway = opts.gateway;
2583
2731
  this.gateway = new FragmentGateway(this.sthis, opts.gateway);
2584
- this.loader = opts.loader;
2585
2732
  }
2586
2733
  url() {
2587
2734
  return this._url;
@@ -2596,20 +2743,20 @@ var BaseStoreImpl = class {
2596
2743
  return;
2597
2744
  }
2598
2745
  async keyedCrypto() {
2599
- return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
2746
+ return keyedCryptoFactory(this._url, this.keybag, this.sthis);
2600
2747
  }
2601
2748
  async start() {
2602
2749
  this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
2603
- this._url = this._url.build().setParam("store", this.storeType).URI();
2750
+ this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
2604
2751
  const res = await this.gateway.start(this._url);
2605
2752
  if (res.isErr()) {
2606
2753
  this.logger.Error().Result("gw-start", res).Msg("started-gateway");
2607
2754
  return res;
2608
2755
  }
2609
2756
  this._url = res.Ok();
2610
- const kb = await this.keybag();
2757
+ const kb = await this.keybag;
2611
2758
  const skRes = await kb.ensureKeyFromUrl(this._url, () => {
2612
- const idx = this._url.getParam("index");
2759
+ const idx = this._url.getParam("index" /* INDEX */);
2613
2760
  const storeKeyName = [this.name];
2614
2761
  if (idx) {
2615
2762
  storeKeyName.push(idx);
@@ -2642,8 +2789,8 @@ var BaseStoreImpl = class {
2642
2789
  };
2643
2790
  var MetaStoreImpl = class extends BaseStoreImpl {
2644
2791
  // remote: boolean;
2645
- constructor(sthis, name, url, opts) {
2646
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
2792
+ constructor(sthis, url, opts) {
2793
+ super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
2647
2794
  this.storeType = "meta";
2648
2795
  this.subscribers = /* @__PURE__ */ new Map();
2649
2796
  this.parents = [];
@@ -2714,13 +2861,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
2714
2861
  return import_cement10.Result.Ok(void 0);
2715
2862
  }
2716
2863
  async destroy() {
2864
+ this.logger.Debug().Msg("destroy");
2717
2865
  return this.gateway.destroy(this.url());
2718
2866
  }
2719
2867
  };
2720
2868
  var DataStoreImpl = class extends BaseStoreImpl {
2721
2869
  // readonly tag: string = "car-base";
2722
- constructor(sthis, name, url, opts) {
2723
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
2870
+ constructor(sthis, url, opts) {
2871
+ super(
2872
+ sthis,
2873
+ url,
2874
+ {
2875
+ ...opts
2876
+ },
2877
+ ensureLogger(sthis, "DataStoreImpl")
2878
+ );
2724
2879
  this.storeType = "data";
2725
2880
  }
2726
2881
  async load(cid) {
@@ -2761,23 +2916,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
2761
2916
  return import_cement10.Result.Ok(void 0);
2762
2917
  }
2763
2918
  destroy() {
2919
+ this.logger.Debug().Msg("destroy");
2764
2920
  return this.gateway.destroy(this.url());
2765
2921
  }
2766
2922
  };
2767
2923
  var WALStoreImpl = class extends BaseStoreImpl {
2768
- constructor(loader, url, opts) {
2769
- super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
2924
+ constructor(sthis, url, opts) {
2925
+ super(
2926
+ sthis,
2927
+ url,
2928
+ {
2929
+ ...opts
2930
+ },
2931
+ ensureLogger(sthis, "WALStoreImpl")
2932
+ );
2770
2933
  this.storeType = "wal";
2934
+ // readonly tag: string = "rwal-base";
2935
+ // readonly loader: Loadable;
2771
2936
  this._ready = new import_cement10.ResolveOnce();
2772
2937
  this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
2773
2938
  this.processing = void 0;
2774
2939
  this.processQueue = new CommitQueue();
2775
- this.loader = loader;
2776
2940
  }
2777
2941
  async ready() {
2778
2942
  return this._ready.once(async () => {
2779
2943
  const walState = await this.load().catch((e) => {
2780
- this.logger.Error().Any("error", e).Msg("error loading wal");
2944
+ this.logger.Error().Err(e).Msg("error loading wal");
2781
2945
  return void 0;
2782
2946
  });
2783
2947
  if (!walState) {
@@ -2810,7 +2974,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2810
2974
  }
2811
2975
  async process() {
2812
2976
  await this.ready();
2813
- if (!this.loader.remoteCarStore) return;
2977
+ if (!this.loader?.remoteCarStore) return;
2814
2978
  await this.processQueue.enqueue(async () => {
2815
2979
  try {
2816
2980
  await this._doProcess();
@@ -2823,6 +2987,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2823
2987
  });
2824
2988
  }
2825
2989
  async _doProcess() {
2990
+ if (!this.loader) return;
2826
2991
  if (!this.loader.remoteCarStore) return;
2827
2992
  const operations = [...this.walState.operations];
2828
2993
  const noLoaderOps = [...this.walState.noLoaderOps];
@@ -2840,6 +3005,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2840
3005
  noLoaderOps,
2841
3006
  async (dbMeta) => {
2842
3007
  await retryableUpload(async () => {
3008
+ if (!this.loader) return;
2843
3009
  for (const cid of dbMeta.cars) {
2844
3010
  const car = await (await this.loader.carStore()).load(cid);
2845
3011
  if (!car) {
@@ -2859,6 +3025,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2859
3025
  operations,
2860
3026
  async (dbMeta) => {
2861
3027
  await retryableUpload(async () => {
3028
+ if (!this.loader) return;
2862
3029
  for (const cid of dbMeta.cars) {
2863
3030
  const car = await (await this.loader.carStore()).load(cid);
2864
3031
  if (!car) {
@@ -2878,6 +3045,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2878
3045
  fileOperations,
2879
3046
  async ({ cid: fileCid, public: publicFile }) => {
2880
3047
  await retryableUpload(async () => {
3048
+ if (!this.loader) return;
2881
3049
  const fileBlock = await (await this.loader.fileStore()).load(fileCid);
2882
3050
  if (!fileBlock) {
2883
3051
  throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
@@ -2891,6 +3059,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2891
3059
  if (operations.length) {
2892
3060
  const lastOp = operations[operations.length - 1];
2893
3061
  await retryableUpload(async () => {
3062
+ if (!this.loader) return;
2894
3063
  await this.loader.remoteMetaStore?.save(lastOp);
2895
3064
  }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
2896
3065
  }
@@ -2916,6 +3085,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2916
3085
  }
2917
3086
  try {
2918
3087
  return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
3088
+ return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
2919
3089
  } catch (e) {
2920
3090
  throw this.logger.Error().Err(e).Msg("error parse").AsError();
2921
3091
  }
@@ -2942,73 +3112,98 @@ var WALStoreImpl = class extends BaseStoreImpl {
2942
3112
  return import_cement10.Result.Ok(void 0);
2943
3113
  }
2944
3114
  destroy() {
3115
+ this.logger.Debug().Msg("destroy");
2945
3116
  return this.gateway.destroy(this.url());
2946
3117
  }
2947
3118
  };
2948
3119
 
2949
3120
  // src/blockstore/store-factory.ts
3121
+ init_types();
3122
+
3123
+ // src/blockstore/register-store-protocol.ts
3124
+ var import_cement14 = require("@adviser/cement");
3125
+ init_version();
3126
+
3127
+ // src/runtime/gateways/memory/gateway.ts
3128
+ var import_cement11 = require("@adviser/cement");
3129
+ init_types();
3130
+
3131
+ // src/runtime/gateways/memory/version.ts
3132
+ var MEMORY_VERSION = "v0.19-memory";
3133
+
3134
+ // src/runtime/gateways/memory/gateway.ts
2950
3135
  init_utils();
2951
- function ensureIsIndex(url, isIndex) {
2952
- if (isIndex) {
2953
- return url.build().setParam("index", isIndex).URI();
3136
+ var MemoryGateway = class {
3137
+ constructor(memorys) {
3138
+ this.memorys = memorys;
2954
3139
  }
2955
- return url.build().delParam("index").URI();
2956
- }
2957
- function ensureName(name, url) {
2958
- if (!url.hasParam("name")) {
2959
- return url.build().setParam("name", name).URI();
3140
+ buildUrl(baseUrl, key) {
3141
+ return Promise.resolve(import_cement11.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
2960
3142
  }
2961
- return url;
2962
- }
3143
+ start(baseUrl) {
3144
+ return Promise.resolve(import_cement11.Result.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
3145
+ }
3146
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
3147
+ close(baseUrl) {
3148
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3149
+ }
3150
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
3151
+ destroy(baseUrl) {
3152
+ this.memorys.clear();
3153
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3154
+ }
3155
+ put(url, body) {
3156
+ this.memorys.set(url.toString(), body);
3157
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3158
+ }
3159
+ // get could return a NotFoundError if the key is not found
3160
+ get(url) {
3161
+ const x = this.memorys.get(url.toString());
3162
+ if (x === void 0) {
3163
+ return Promise.resolve(import_cement11.Result.Err(new NotFoundError("not found")));
3164
+ }
3165
+ return Promise.resolve(import_cement11.Result.Ok(x));
3166
+ }
3167
+ delete(url) {
3168
+ this.memorys.delete(url.toString());
3169
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3170
+ }
3171
+ };
3172
+ var MemoryTestGateway = class {
3173
+ constructor(memorys) {
3174
+ this.memorys = memorys;
3175
+ }
3176
+ async get(url, key) {
3177
+ return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
3178
+ }
3179
+ };
3180
+
3181
+ // src/blockstore/register-store-protocol.ts
2963
3182
  var storeFactory = /* @__PURE__ */ new Map();
2964
- function buildURL(optURL, loader) {
2965
- const storeOpts = loader.ebOpts.store;
2966
- const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
2967
- let obuUrl;
2968
- if (obuItem && obuItem.overrideBaseURL) {
2969
- obuUrl = import_cement13.URI.from(obuItem.overrideBaseURL);
2970
- }
2971
- const ret = ensureIsIndex(
2972
- import_cement13.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2973
- storeOpts.isIndex
2974
- );
2975
- return ret;
2976
- }
2977
- var onceGateway = new import_cement13.KeyedResolvOnce();
2978
- async function getGatewayFromURL(url, sthis) {
2979
- return onceGateway.get(url.toString()).once(async () => {
2980
- const item = storeFactory.get(url.protocol);
2981
- if (item) {
2982
- const ret = {
2983
- gateway: await item.gateway(sthis),
2984
- test: await item.test(sthis)
2985
- };
2986
- const res = await ret.gateway.start(url);
2987
- if (res.isErr()) {
2988
- sthis.logger.Error().Result("start", res).Msg("start failed");
2989
- return void 0;
2990
- }
2991
- return ret;
3183
+ function getDefaultURI(sthis, protocol) {
3184
+ if (protocol) {
3185
+ if (!protocol.endsWith(":")) {
3186
+ protocol += ":";
2992
3187
  }
2993
- sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2994
- return void 0;
2995
- });
3188
+ const gfi = storeFactory.get(protocol);
3189
+ if (gfi) {
3190
+ return gfi.defaultURI(sthis);
3191
+ }
3192
+ }
3193
+ const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
3194
+ if (!found) {
3195
+ throw sthis.logger.Error().Msg("no default found").AsError();
3196
+ }
3197
+ return found.defaultURI(sthis);
2996
3198
  }
2997
3199
  function registerStoreProtocol(item) {
2998
3200
  let protocol = item.protocol;
2999
3201
  if (!protocol.endsWith(":")) {
3000
3202
  protocol += ":";
3001
3203
  }
3002
- if (storeFactory.has(protocol)) {
3003
- if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
3004
- throw new Error(`we need a logger here`);
3005
- return () => {
3006
- };
3007
- }
3008
- }
3009
- if (item.overrideBaseURL) {
3204
+ if (item.isDefault) {
3010
3205
  Array.from(storeFactory.values()).forEach((items) => {
3011
- items.overrideBaseURL = void 0;
3206
+ items.isDefault = false;
3012
3207
  });
3013
3208
  }
3014
3209
  storeFactory.set(protocol, item);
@@ -3016,134 +3211,186 @@ function registerStoreProtocol(item) {
3016
3211
  storeFactory.delete(protocol);
3017
3212
  };
3018
3213
  }
3019
- var onceDataStoreFactory = new import_cement13.KeyedResolvOnce();
3020
- async function dataStoreFactory(loader) {
3021
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
3022
- const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
3023
- return onceDataStoreFactory.get(url.toString()).once(async () => {
3024
- const gateway = await getGatewayFromURL(url, sthis);
3025
- if (!gateway) {
3026
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3027
- }
3028
- const store = new DataStoreImpl(sthis, loader.name, url, {
3029
- gateway: gateway.gateway,
3030
- keybag: () => getKeyBag(loader.sthis, {
3031
- ...loader.ebOpts.keyBag
3032
- })
3033
- });
3034
- return store;
3214
+ function getGatewayFactoryItem(protocol) {
3215
+ return storeFactory.get(protocol);
3216
+ }
3217
+ function fileGatewayFactoryItem() {
3218
+ return {
3219
+ protocol: "file:",
3220
+ isDefault: true,
3221
+ defaultURI: (sthis) => {
3222
+ return import_cement14.BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
3223
+ },
3224
+ gateway: async (sthis) => {
3225
+ const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3226
+ return new FileGateway2(sthis);
3227
+ },
3228
+ test: async (sthis) => {
3229
+ const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3230
+ return new FileTestStore2(sthis);
3231
+ }
3232
+ };
3233
+ }
3234
+ if ((0, import_cement14.runtimeFn)().isBrowser) {
3235
+ registerStoreProtocol({
3236
+ protocol: "indexdb:",
3237
+ isDefault: true,
3238
+ defaultURI: () => {
3239
+ return import_cement14.BuildURI.from("indexdb://").pathname("fp").URI();
3240
+ },
3241
+ gateway: async (logger) => {
3242
+ const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3243
+ return new IndexDBGateway2(logger);
3244
+ },
3245
+ test: async (logger) => {
3246
+ const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3247
+ return new IndexDBTestStore2(logger);
3248
+ }
3035
3249
  });
3250
+ } else {
3251
+ registerStoreProtocol(fileGatewayFactoryItem());
3036
3252
  }
3037
- var onceMetaStoreFactory = new import_cement13.KeyedResolvOnce();
3038
- async function metaStoreFactory(loader) {
3039
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
3040
- const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
3041
- return onceMetaStoreFactory.get(url.toString()).once(async () => {
3042
- sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
3043
- const gateway = await getGatewayFromURL(url, sthis);
3044
- if (!gateway) {
3045
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3046
- }
3047
- const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
3048
- gateway: gateway.gateway,
3049
- keybag: () => getKeyBag(loader.sthis, {
3050
- ...loader.ebOpts.keyBag
3051
- })
3052
- });
3053
- return store;
3253
+ var memory = /* @__PURE__ */ new Map();
3254
+ registerStoreProtocol({
3255
+ protocol: "memory:",
3256
+ isDefault: false,
3257
+ defaultURI: () => {
3258
+ return import_cement14.BuildURI.from("memory://").pathname("ram").URI();
3259
+ },
3260
+ gateway: async () => {
3261
+ return new MemoryGateway(memory);
3262
+ },
3263
+ test: async () => {
3264
+ return new MemoryTestGateway(memory);
3265
+ }
3266
+ });
3267
+
3268
+ // src/blockstore/store-factory.ts
3269
+ var onceGateway = new import_cement15.KeyedResolvOnce();
3270
+ var gatewayInstances = new import_cement15.KeyedResolvOnce();
3271
+ async function getStartedGateway(sthis, url) {
3272
+ return onceGateway.get(url.toString()).once(async () => {
3273
+ const item = getGatewayFactoryItem(url.protocol);
3274
+ if (item) {
3275
+ const ret = {
3276
+ url,
3277
+ ...await gatewayInstances.get(url.protocol).once(async () => ({
3278
+ gateway: await item.gateway(sthis),
3279
+ test: await item.test(sthis)
3280
+ }))
3281
+ };
3282
+ const res = await ret.gateway.start(url);
3283
+ if (res.isErr()) {
3284
+ return import_cement15.Result.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
3285
+ }
3286
+ ret.url = res.Ok();
3287
+ return import_cement15.Result.Ok(ret);
3288
+ }
3289
+ return import_cement15.Result.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
3054
3290
  });
3055
3291
  }
3056
- var onceRemoteWalFactory = new import_cement13.KeyedResolvOnce();
3057
- async function remoteWalFactory(loader) {
3058
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
3059
- const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
3060
- return onceRemoteWalFactory.get(url.toString()).once(async () => {
3061
- const gateway = await getGatewayFromURL(url, sthis);
3062
- if (!gateway) {
3063
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3064
- }
3065
- sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
3066
- const store = new WALStoreImpl(loader, url, {
3067
- gateway: gateway.gateway,
3068
- keybag: () => getKeyBag(loader.sthis, {
3069
- ...loader.ebOpts.keyBag
3070
- })
3071
- });
3072
- return store;
3292
+ async function dataStoreFactory(sfi) {
3293
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
3294
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
3295
+ if (rgateway.isErr()) {
3296
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
3297
+ }
3298
+ const gateway = rgateway.Ok();
3299
+ const store = new DataStoreImpl(sfi.sthis, gateway.url, {
3300
+ gateway: gateway.gateway,
3301
+ keybag: sfi.keybag
3302
+ });
3303
+ return store;
3304
+ }
3305
+ async function metaStoreFactory(sfi) {
3306
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
3307
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
3308
+ if (rgateway.isErr()) {
3309
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
3310
+ }
3311
+ const gateway = rgateway.Ok();
3312
+ const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
3313
+ gateway: gateway.gateway,
3314
+ keybag: sfi.keybag
3073
3315
  });
3316
+ return store;
3317
+ }
3318
+ async function WALStoreFactory(sfi) {
3319
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
3320
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
3321
+ if (rgateway.isErr()) {
3322
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
3323
+ }
3324
+ const gateway = rgateway.Ok();
3325
+ const store = new WALStoreImpl(sfi.sthis, gateway.url, {
3326
+ gateway: gateway.gateway,
3327
+ keybag: sfi.keybag
3328
+ });
3329
+ return store;
3074
3330
  }
3075
3331
  async function testStoreFactory(url, sthis) {
3076
- sthis = ensureSuperLog(sthis, "testStoreFactory");
3077
- const gateway = await getGatewayFromURL(url, sthis);
3078
- if (!gateway) {
3332
+ const rgateway = await getStartedGateway(sthis, url);
3333
+ if (!rgateway) {
3079
3334
  throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3080
3335
  }
3081
- return gateway.test;
3336
+ return rgateway.Ok().test;
3082
3337
  }
3083
- async function ensureStart(store, logger) {
3338
+ async function ensureStart(store) {
3084
3339
  const ret = await store.start();
3085
3340
  if (ret.isErr()) {
3086
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
3341
+ throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
3087
3342
  }
3088
- logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
3343
+ store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
3089
3344
  return store;
3090
3345
  }
3091
- function toStoreRuntime(opts, sthis) {
3092
- const logger = ensureLogger(sthis, "toStoreRuntime", {});
3346
+ function ensureStoreEnDeFile(ende) {
3347
+ ende = ende || {};
3093
3348
  return {
3094
- makeMetaStore: async (loader) => {
3095
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
3096
- return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
3097
- },
3098
- makeDataStore: async (loader) => {
3099
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
3100
- return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
3101
- },
3102
- makeWALStore: async (loader) => {
3103
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
3104
- return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
3105
- },
3106
- encodeFile: opts.encodeFile || encodeFile,
3107
- decodeFile: opts.decodeFile || decodeFile
3349
+ encodeFile: ende.encodeFile || encodeFile,
3350
+ decodeFile: ende.decodeFile || decodeFile
3108
3351
  };
3109
3352
  }
3110
- registerStoreProtocol({
3111
- protocol: "file:",
3112
- gateway: async (sthis) => {
3113
- const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3114
- return new FileGateway2(sthis);
3115
- },
3116
- test: async (sthis) => {
3117
- const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3118
- return new FileTestStore2(sthis);
3119
- }
3120
- });
3121
- registerStoreProtocol({
3122
- protocol: "indexdb:",
3123
- gateway: async (sthis) => {
3124
- const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3125
- return new IndexDBGateway2(sthis);
3126
- },
3127
- test: async (sthis) => {
3128
- const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3129
- return new IndexDBTestStore2(sthis);
3130
- }
3131
- });
3353
+ function toStoreRuntime(sthis, endeOpts = {}) {
3354
+ return {
3355
+ makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
3356
+ // async (loader: Loadable) => {
3357
+ // logger
3358
+ // .Debug()
3359
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
3360
+ // .Msg("makeMetaStore");
3361
+ // return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
3362
+ // },
3363
+ makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
3364
+ // async (loader: Loadable) => {
3365
+ // logger
3366
+ // .Debug()
3367
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
3368
+ // .Msg("makeDataStore");
3369
+ // return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
3370
+ // },
3371
+ makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
3372
+ // async (loader: Loadable) => {
3373
+ // logger
3374
+ // .Debug()
3375
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
3376
+ // .Msg("makeRemoteWAL");
3377
+ // return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
3378
+ // },
3379
+ ...ensureStoreEnDeFile(endeOpts)
3380
+ };
3381
+ }
3382
+
3383
+ // src/blockstore/connection-base.ts
3384
+ init_types();
3132
3385
 
3133
3386
  // src/blockstore/store-remote.ts
3134
3387
  async function RemoteDataStore(sthis, name, url, opts) {
3135
- const ds = new DataStoreImpl(sthis, name, url, opts);
3388
+ const ds = new DataStoreImpl(sthis, url, opts);
3136
3389
  await ds.start();
3137
3390
  return ds;
3138
3391
  }
3139
3392
  async function RemoteMetaStore(sthis, name, url, opts) {
3140
- const ms = new MetaStoreImpl(
3141
- sthis,
3142
- name,
3143
- url,
3144
- opts
3145
- /* , true*/
3146
- );
3393
+ const ms = new MetaStoreImpl(sthis, url, opts);
3147
3394
  await ms.start();
3148
3395
  return ms;
3149
3396
  }
@@ -3168,14 +3415,17 @@ var ConnectionBase = class {
3168
3415
  if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
3169
3416
  this.loader = loader;
3170
3417
  await this.onConnect();
3171
- const metaUrl = this.url.build().defParam("store", "meta").URI();
3172
- const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
3173
- if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
3418
+ const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
3419
+ const rgateway = await getStartedGateway(loader.sthis, metaUrl);
3420
+ if (rgateway.isErr())
3421
+ throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
3422
+ const name = metaUrl.toString();
3174
3423
  const dbName = metaUrl.getParam("name");
3175
- if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
3176
- const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
3424
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
3425
+ const gateway = rgateway.Ok();
3426
+ const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
3177
3427
  gateway: gateway.gateway,
3178
- keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
3428
+ keybag: await loader.keyBag(),
3179
3429
  loader
3180
3430
  });
3181
3431
  this.loader.remoteMetaStore = remote;
@@ -3188,14 +3438,15 @@ var ConnectionBase = class {
3188
3438
  async connectStorage_X({ loader }) {
3189
3439
  if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
3190
3440
  this.loader = loader;
3191
- const dataUrl = this.url.build().defParam("store", "data").URI();
3192
- const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
3193
- if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
3441
+ const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
3442
+ const rgateway = await getStartedGateway(loader.sthis, dataUrl);
3443
+ if (rgateway.isErr())
3444
+ throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
3194
3445
  const name = dataUrl.getParam("name");
3195
3446
  if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
3196
3447
  loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
3197
- gateway: gateway.gateway,
3198
- keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
3448
+ gateway: rgateway.Ok().gateway,
3449
+ keybag: await loader.keyBag()
3199
3450
  });
3200
3451
  loader.remoteFileStore = loader.remoteCarStore;
3201
3452
  }
@@ -3235,6 +3486,13 @@ var ConnectionBase = class {
3235
3486
  };
3236
3487
 
3237
3488
  // src/crdt-helpers.ts
3489
+ var import_link = require("multiformats/link");
3490
+ var import_sha25 = require("multiformats/hashes/sha2");
3491
+ var codec = __toESM(require("@ipld/dag-cbor"), 1);
3492
+ var import_crdt = require("@web3-storage/pail/crdt");
3493
+ var import_clock2 = require("@web3-storage/pail/clock");
3494
+ var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
3495
+ init_types();
3238
3496
  function time(tag) {
3239
3497
  }
3240
3498
  function timeEnd(tag) {
@@ -3282,7 +3540,7 @@ async function writeDocContent(store, blocks, update, logger) {
3282
3540
  await processFiles(store, blocks, update.value, logger);
3283
3541
  value = { doc: update.value };
3284
3542
  }
3285
- const block = await encode({ value, hasher: import_sha25.sha256, codec });
3543
+ const block = await encode3({ value, hasher: import_sha25.sha256, codec });
3286
3544
  blocks.putSync(block.cid, block.bytes);
3287
3545
  return block.cid;
3288
3546
  }
@@ -3373,7 +3631,7 @@ function readFileset(blocks, files, isPublic = false) {
3373
3631
  async function getValueFromLink(blocks, link, logger) {
3374
3632
  const block = await blocks.get(link);
3375
3633
  if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
3376
- const { value } = await decode({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
3634
+ const { value } = await decode3({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
3377
3635
  const cvalue = {
3378
3636
  ...value,
3379
3637
  cid: link
@@ -3494,16 +3752,19 @@ async function doCompact(blockLog, head, logger) {
3494
3752
  async function getBlock(blocks, cidString) {
3495
3753
  const block = await blocks.get((0, import_link.parse)(cidString));
3496
3754
  if (!block) throw new Error(`Missing block ${cidString}`);
3497
- const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
3755
+ const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
3498
3756
  return new Block({ cid, value, bytes: block.bytes });
3499
3757
  }
3500
3758
 
3759
+ // src/indexer.ts
3760
+ init_types();
3761
+
3501
3762
  // src/indexer-helpers.ts
3502
3763
  var import_sha26 = require("multiformats/hashes/sha2");
3503
3764
  var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
3504
3765
  var import_charwise = __toESM(require("charwise"), 1);
3505
3766
  var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
3506
- var import_utils16 = require("prolly-trees/utils");
3767
+ var import_utils15 = require("prolly-trees/utils");
3507
3768
  var import_cache = require("prolly-trees/cache");
3508
3769
  var IndexTree = class {
3509
3770
  };
@@ -3511,17 +3772,17 @@ function refCompare(aRef, bRef) {
3511
3772
  if (Number.isNaN(aRef)) return -1;
3512
3773
  if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
3513
3774
  if (aRef === Infinity) return 1;
3514
- return (0, import_utils16.simpleCompare)(aRef, bRef);
3775
+ return (0, import_utils15.simpleCompare)(aRef, bRef);
3515
3776
  }
3516
3777
  function compare(a, b) {
3517
3778
  const [aKey, aRef] = a;
3518
3779
  const [bKey, bRef] = b;
3519
- const comp = (0, import_utils16.simpleCompare)(aKey, bKey);
3780
+ const comp = (0, import_utils15.simpleCompare)(aKey, bKey);
3520
3781
  if (comp !== 0) return comp;
3521
3782
  return refCompare(aRef, bRef);
3522
3783
  }
3523
- var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
3524
- var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils16.simpleCompare };
3784
+ var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
3785
+ var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils15.simpleCompare };
3525
3786
  function indexEntriesForChanges(changes, mapFn) {
3526
3787
  const indexEntries = [];
3527
3788
  changes.forEach(({ id: key, value, del }) => {
@@ -3552,7 +3813,8 @@ function makeProllyGetBlock(blocks) {
3552
3813
  return create({ cid, bytes, hasher: import_sha26.sha256, codec: codec2 });
3553
3814
  };
3554
3815
  }
3555
- async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
3816
+ async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
3817
+ logger.Debug().Msg("enter bulkIndex");
3556
3818
  if (!indexEntries.length) return inIndex;
3557
3819
  if (!inIndex.root) {
3558
3820
  if (!inIndex.cid) {
@@ -3569,18 +3831,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
3569
3831
  returnNode = node;
3570
3832
  }
3571
3833
  if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
3834
+ logger.Debug().Msg("exit !root bulkIndex");
3572
3835
  return { root: returnNode, cid: returnRootBlock.cid };
3573
3836
  } else {
3574
3837
  inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
3575
3838
  }
3576
3839
  }
3840
+ logger.Debug().Msg("pre bulk bulkIndex");
3577
3841
  const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
3578
3842
  if (root3) {
3843
+ logger.Debug().Msg("pre root put bulkIndex");
3579
3844
  for await (const block of newBlocks) {
3580
3845
  await tblocks.put(block.cid, block.bytes);
3581
3846
  }
3582
3847
  return { root: root3, cid: (await root3.block).cid };
3583
3848
  } else {
3849
+ logger.Debug().Msg("pre !root bulkIndex");
3584
3850
  return { root: void 0, cid: void 0 };
3585
3851
  }
3586
3852
  }
@@ -3621,17 +3887,17 @@ function encodeKey(key) {
3621
3887
 
3622
3888
  // src/indexer.ts
3623
3889
  init_utils();
3624
- function index(sthis, { _crdt }, name, mapFn, meta) {
3625
- if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3626
- if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
3627
- if (_crdt.indexers.has(name)) {
3628
- const idx = _crdt.indexers.get(name);
3890
+ function index(refDb, name, mapFn, meta) {
3891
+ if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3892
+ if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
3893
+ if (refDb.crdt.indexers.has(name)) {
3894
+ const idx = refDb.crdt.indexers.get(name);
3629
3895
  idx.applyMapFn(name, mapFn, meta);
3630
3896
  } else {
3631
- const idx = new Index(sthis, _crdt, name, mapFn, meta);
3632
- _crdt.indexers.set(name, idx);
3897
+ const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
3898
+ refDb.crdt.indexers.set(name, idx);
3633
3899
  }
3634
- return _crdt.indexers.get(name);
3900
+ return refDb.crdt.indexers.get(name);
3635
3901
  }
3636
3902
  var Index = class {
3637
3903
  constructor(sthis, crdt, name, mapFn, meta) {
@@ -3650,18 +3916,9 @@ var Index = class {
3650
3916
  return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
3651
3917
  });
3652
3918
  }
3653
- close() {
3654
- return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
3655
- });
3656
- }
3657
- destroy() {
3658
- return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
3659
- });
3660
- }
3661
3919
  applyMapFn(name, mapFn, meta) {
3662
3920
  if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3663
3921
  if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
3664
- this.name = name;
3665
3922
  try {
3666
3923
  if (meta) {
3667
3924
  if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
@@ -3709,9 +3966,13 @@ var Index = class {
3709
3966
  }
3710
3967
  }
3711
3968
  async query(opts = {}) {
3969
+ this.logger.Debug().Msg("enter query");
3712
3970
  await this.ready();
3971
+ this.logger.Debug().Msg("post ready query");
3713
3972
  await this._updateIndex();
3973
+ this.logger.Debug().Msg("post _updateIndex query");
3714
3974
  await this._hydrateIndex();
3975
+ this.logger.Debug().Msg("post _hydrateIndex query");
3715
3976
  if (!this.byKey.root) {
3716
3977
  return await applyQuery(this.crdt, { result: [] }, opts);
3717
3978
  }
@@ -3767,13 +4028,16 @@ var Index = class {
3767
4028
  }
3768
4029
  async _updateIndex() {
3769
4030
  await this.ready();
4031
+ this.logger.Debug().Msg("enter _updateIndex");
3770
4032
  if (this.initError) throw this.initError;
3771
4033
  if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
3772
4034
  let result, head;
3773
4035
  if (!this.indexHead || this.indexHead.length === 0) {
3774
4036
  ({ result, head } = await this.crdt.allDocs());
4037
+ this.logger.Debug().Msg("enter crdt.allDocs");
3775
4038
  } else {
3776
4039
  ({ result, head } = await this.crdt.changes(this.indexHead));
4040
+ this.logger.Debug().Msg("enter crdt.changes");
3777
4041
  }
3778
4042
  if (result.length === 0) {
3779
4043
  this.indexHead = head;
@@ -3806,9 +4070,22 @@ var Index = class {
3806
4070
  if (result.length === 0) {
3807
4071
  return indexerMeta;
3808
4072
  }
4073
+ this.logger.Debug().Msg("pre this.blockstore.transaction");
3809
4074
  const { meta } = await this.blockstore.transaction(async (tblocks) => {
3810
- this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
3811
- this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
4075
+ this.byId = await bulkIndex(
4076
+ this.logger,
4077
+ tblocks,
4078
+ this.byId,
4079
+ removeIdIndexEntries.concat(byIdIndexEntries),
4080
+ byIdOpts
4081
+ );
4082
+ this.byKey = await bulkIndex(
4083
+ this.logger,
4084
+ tblocks,
4085
+ this.byKey,
4086
+ staleKeyIndexEntries.concat(indexEntries),
4087
+ byKeyOpts
4088
+ );
3812
4089
  this.indexHead = head;
3813
4090
  if (this.byId.cid && this.byKey.cid) {
3814
4091
  const idxMeta = {
@@ -3820,8 +4097,10 @@ var Index = class {
3820
4097
  };
3821
4098
  indexerMeta.indexes?.set(this.name, idxMeta);
3822
4099
  }
4100
+ this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
3823
4101
  return indexerMeta;
3824
4102
  });
4103
+ this.logger.Debug().Msg("post this.blockstore.transaction");
3825
4104
  return meta;
3826
4105
  }
3827
4106
  };
@@ -3829,7 +4108,8 @@ var Index = class {
3829
4108
  // src/crdt-clock.ts
3830
4109
  var import_clock3 = require("@web3-storage/pail/clock");
3831
4110
  var import_crdt2 = require("@web3-storage/pail/crdt");
3832
- var import_cement14 = require("@adviser/cement");
4111
+ var import_cement16 = require("@adviser/cement");
4112
+ init_types();
3833
4113
 
3834
4114
  // src/apply-head-queue.ts
3835
4115
  function applyHeadQueue(worker, logger) {
@@ -3887,7 +4167,7 @@ var CRDTClock = class {
3887
4167
  this.zoomers = /* @__PURE__ */ new Set();
3888
4168
  this.watchers = /* @__PURE__ */ new Set();
3889
4169
  this.emptyWatchers = /* @__PURE__ */ new Set();
3890
- this._ready = new import_cement14.ResolveOnce();
4170
+ this._ready = new import_cement16.ResolveOnce();
3891
4171
  this.blockstore = blockstore;
3892
4172
  this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
3893
4173
  this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
@@ -3950,21 +4230,23 @@ var CRDTClock = class {
3950
4230
  throw this.logger.Error().Msg("missing blockstore").AsError();
3951
4231
  }
3952
4232
  await validateBlocks(this.logger, newHead, this.blockstore);
3953
- const { meta } = await this.blockstore.transaction(
3954
- async (tblocks) => {
3955
- const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3956
- const result = await (0, import_crdt2.root)(tblocks, advancedHead);
3957
- for (const { cid, bytes } of [
3958
- ...result.additions
3959
- // ...result.removals
3960
- ]) {
3961
- tblocks.putSync(cid, bytes);
3962
- }
3963
- return { head: advancedHead };
3964
- },
3965
- { noLoader, add: false }
3966
- );
3967
- this.setHead(meta.head);
4233
+ if (!this.transaction) {
4234
+ this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
4235
+ }
4236
+ const tblocks = this.transaction;
4237
+ const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
4238
+ const result = await (0, import_crdt2.root)(tblocks, advancedHead);
4239
+ for (const { cid, bytes } of [
4240
+ ...result.additions
4241
+ // ...result.removals
4242
+ ]) {
4243
+ tblocks.putSync(cid, bytes);
4244
+ }
4245
+ if (!noLoader) {
4246
+ await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
4247
+ this.transaction = void 0;
4248
+ }
4249
+ this.setHead(advancedHead);
3968
4250
  }
3969
4251
  };
3970
4252
  function sortClockHead(clockHead) {
@@ -3997,15 +4279,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
3997
4279
  // src/crdt.ts
3998
4280
  init_utils();
3999
4281
  var CRDT = class {
4000
- constructor(sthis, name, opts = {}) {
4282
+ constructor(sthis, opts) {
4001
4283
  this.indexers = /* @__PURE__ */ new Map();
4002
- this.onceReady = new import_cement15.ResolveOnce();
4284
+ this.onceReady = new import_cement17.ResolveOnce();
4003
4285
  this.sthis = sthis;
4004
- this.name = name;
4005
4286
  this.logger = ensureLogger(sthis, "CRDT");
4006
4287
  this.opts = opts;
4007
4288
  this.blockstore = blockstoreFactory(sthis, {
4008
- name,
4009
4289
  applyMeta: async (meta) => {
4010
4290
  const crdtMeta = meta;
4011
4291
  if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
@@ -4015,23 +4295,27 @@ var CRDT = class {
4015
4295
  await doCompact(blocks, this.clock.head, this.logger);
4016
4296
  return { head: this.clock.head };
4017
4297
  },
4018
- autoCompact: this.opts.autoCompact || 100,
4019
- store: { ...this.opts.store, isIndex: void 0 },
4020
- public: this.opts.public,
4021
- meta: this.opts.meta,
4022
- threshold: this.opts.threshold
4298
+ // autoCompact: this.opts.autoCompact || 100,
4299
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
4300
+ storeUrls: this.opts.storeUrls.data,
4301
+ keyBag: this.opts.keyBag,
4302
+ // public: this.opts.public,
4303
+ meta: this.opts.meta
4304
+ // threshold: this.opts.threshold,
4023
4305
  });
4024
4306
  this.indexBlockstore = blockstoreFactory(sthis, {
4025
- name,
4307
+ // name: opts.name,
4026
4308
  applyMeta: async (meta) => {
4027
4309
  const idxCarMeta = meta;
4028
4310
  if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
4029
- for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
4030
- index(this.sthis, { _crdt: this }, name2, void 0, idx);
4311
+ for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
4312
+ index({ crdt: this }, name, void 0, idx);
4031
4313
  }
4032
4314
  },
4033
- store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
4034
- public: this.opts.public
4315
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
4316
+ storeUrls: this.opts.storeUrls.idx,
4317
+ keyBag: this.opts.keyBag
4318
+ // public: this.opts.public,
4035
4319
  });
4036
4320
  this.clock = new CRDTClock(this.blockstore);
4037
4321
  this.clock.onZoom(() => {
@@ -4113,52 +4397,166 @@ var CRDT = class {
4113
4397
  };
4114
4398
 
4115
4399
  // src/database.ts
4400
+ init_types();
4116
4401
  init_utils();
4117
- var Database = class {
4118
- constructor(name, opts) {
4119
- this.opts = {};
4402
+ var databases = new import_cement18.KeyedResolvOnce();
4403
+ function toSortedArray(set) {
4404
+ if (!set) return [];
4405
+ return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
4406
+ }
4407
+ function keyConfigOpts(sthis, name, opts) {
4408
+ return JSON.stringify(
4409
+ toSortedArray({
4410
+ name,
4411
+ stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
4412
+ })
4413
+ );
4414
+ }
4415
+ function isDatabase(db) {
4416
+ return db instanceof DatabaseImpl || db instanceof DatabaseShell;
4417
+ }
4418
+ function DatabaseFactory(name, opts) {
4419
+ const sthis = ensureSuperThis(opts);
4420
+ return new DatabaseShell(
4421
+ databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
4422
+ const db = new DatabaseImpl(sthis, {
4423
+ name,
4424
+ meta: opts?.meta,
4425
+ keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
4426
+ storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
4427
+ storeEnDe: {
4428
+ encodeFile,
4429
+ decodeFile,
4430
+ ...opts?.storeEnDe
4431
+ }
4432
+ });
4433
+ db.onClosed(() => {
4434
+ databases.unget(key);
4435
+ });
4436
+ return db;
4437
+ })
4438
+ );
4439
+ }
4440
+ var DatabaseShell = class {
4441
+ constructor(ref) {
4442
+ this.ref = ref;
4443
+ ref.addShell(this);
4444
+ }
4445
+ get id() {
4446
+ return this.ref.id;
4447
+ }
4448
+ get logger() {
4449
+ return this.ref.logger;
4450
+ }
4451
+ get sthis() {
4452
+ return this.ref.sthis;
4453
+ }
4454
+ get crdt() {
4455
+ return this.ref.crdt;
4456
+ }
4457
+ name() {
4458
+ return this.ref.name();
4459
+ }
4460
+ onClosed(fn) {
4461
+ return this.ref.onClosed(fn);
4462
+ }
4463
+ close() {
4464
+ return this.ref.shellClose(this);
4465
+ }
4466
+ destroy() {
4467
+ return this.ref.destroy();
4468
+ }
4469
+ ready() {
4470
+ return this.ref.ready();
4471
+ }
4472
+ get(id) {
4473
+ return this.ref.get(id);
4474
+ }
4475
+ put(doc) {
4476
+ return this.ref.put(doc);
4477
+ }
4478
+ del(id) {
4479
+ return this.ref.del(id);
4480
+ }
4481
+ changes(since, opts) {
4482
+ return this.ref.changes(since, opts);
4483
+ }
4484
+ allDocs(opts) {
4485
+ return this.ref.allDocs(opts);
4486
+ }
4487
+ allDocuments() {
4488
+ return this.ref.allDocuments();
4489
+ }
4490
+ subscribe(listener, updates) {
4491
+ return this.ref.subscribe(listener, updates);
4492
+ }
4493
+ query(field, opts) {
4494
+ return this.ref.query(field, opts);
4495
+ }
4496
+ compact() {
4497
+ return this.ref.compact();
4498
+ }
4499
+ };
4500
+ var DatabaseImpl = class {
4501
+ constructor(sthis, opts) {
4120
4502
  this._listening = false;
4121
4503
  this._listeners = /* @__PURE__ */ new Set();
4122
4504
  this._noupdate_listeners = /* @__PURE__ */ new Set();
4123
- this._ready = new import_cement16.ResolveOnce();
4124
- this.name = name;
4125
- this.opts = opts || this.opts;
4126
- this.sthis = ensureSuperThis(this.opts);
4505
+ // readonly blockstore: BaseBlockstore;
4506
+ this.shells = /* @__PURE__ */ new Set();
4507
+ this._onClosedFns = /* @__PURE__ */ new Set();
4508
+ this._ready = new import_cement18.ResolveOnce();
4509
+ this.opts = opts;
4510
+ this.sthis = sthis;
4511
+ this.id = sthis.timeOrderedNextId().str;
4127
4512
  this.logger = ensureLogger(this.sthis, "Database");
4128
- this._crdt = new CRDT(this.sthis, name, this.opts);
4129
- this.blockstore = this._crdt.blockstore;
4513
+ this.crdt = new CRDT(this.sthis, this.opts);
4130
4514
  this._writeQueue = writeQueue(async (updates) => {
4131
- return await this._crdt.bulk(updates);
4515
+ return await this.crdt.bulk(updates);
4132
4516
  });
4133
- this._crdt.clock.onTock(() => {
4517
+ this.crdt.clock.onTock(() => {
4134
4518
  this._no_update_notify();
4135
4519
  });
4136
4520
  }
4137
- static {
4138
- this.databases = /* @__PURE__ */ new Map();
4521
+ addShell(shell) {
4522
+ this.shells.add(shell);
4523
+ }
4524
+ onClosed(fn) {
4525
+ this._onClosedFns.add(fn);
4139
4526
  }
4140
4527
  async close() {
4141
- await this.ready();
4142
- await this._crdt.close();
4143
- await this.blockstore.close();
4528
+ throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
4529
+ }
4530
+ async shellClose(db) {
4531
+ if (!this.shells.has(db)) {
4532
+ throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
4533
+ }
4534
+ this.shells.delete(db);
4535
+ if (this.shells.size === 0) {
4536
+ await this.ready();
4537
+ await this.crdt.close();
4538
+ this._onClosedFns.forEach((fn) => fn());
4539
+ }
4144
4540
  }
4145
4541
  async destroy() {
4146
4542
  await this.ready();
4147
- await this._crdt.destroy();
4148
- await this.blockstore.destroy();
4543
+ await this.crdt.destroy();
4149
4544
  }
4150
4545
  async ready() {
4151
- return this._ready.once(async () => {
4546
+ const ret = await this._ready.once(async () => {
4152
4547
  await this.sthis.start();
4153
- await this._crdt.ready();
4154
- await this.blockstore.ready();
4548
+ await this.crdt.ready();
4155
4549
  });
4550
+ return ret;
4551
+ }
4552
+ name() {
4553
+ return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
4156
4554
  }
4157
4555
  async get(id) {
4158
- if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
4556
+ if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
4159
4557
  await this.ready();
4160
4558
  this.logger.Debug().Str("id", id).Msg("get");
4161
- const got = await this._crdt.get(id).catch((e) => {
4559
+ const got = await this.crdt.get(id).catch((e) => {
4162
4560
  throw new NotFoundError(`Not found: ${id} - ${e.message}`);
4163
4561
  });
4164
4562
  if (!got) throw new NotFoundError(`Not found: ${id}`);
@@ -4177,34 +4575,34 @@ var Database = class {
4177
4575
  _id: docId
4178
4576
  }
4179
4577
  });
4180
- return { id: docId, clock: result?.head, name: this.name };
4578
+ return { id: docId, clock: result?.head, name: this.name() };
4181
4579
  }
4182
4580
  async del(id) {
4183
4581
  await this.ready();
4184
4582
  this.logger.Debug().Str("id", id).Msg("del");
4185
4583
  const result = await this._writeQueue.push({ id, del: true });
4186
- return { id, clock: result?.head, name: this.name };
4584
+ return { id, clock: result?.head, name: this.name() };
4187
4585
  }
4188
4586
  async changes(since = [], opts = {}) {
4189
4587
  await this.ready();
4190
4588
  this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
4191
- const { result, head } = await this._crdt.changes(since, opts);
4589
+ const { result, head } = await this.crdt.changes(since, opts);
4192
4590
  const rows = result.map(({ id: key, value, del, clock }) => ({
4193
4591
  key,
4194
4592
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
4195
4593
  clock
4196
4594
  }));
4197
- return { rows, clock: head, name: this.name };
4595
+ return { rows, clock: head, name: this.name() };
4198
4596
  }
4199
4597
  async allDocs(opts = {}) {
4200
4598
  await this.ready();
4201
4599
  this.logger.Debug().Msg("allDocs");
4202
- const { result, head } = await this._crdt.allDocs();
4600
+ const { result, head } = await this.crdt.allDocs();
4203
4601
  const rows = result.map(({ id: key, value, del }) => ({
4204
4602
  key,
4205
4603
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
4206
4604
  }));
4207
- return { rows, clock: head, name: this.name };
4605
+ return { rows, clock: head, name: this.name() };
4208
4606
  }
4209
4607
  async allDocuments() {
4210
4608
  return this.allDocs();
@@ -4214,7 +4612,7 @@ var Database = class {
4214
4612
  if (updates) {
4215
4613
  if (!this._listening) {
4216
4614
  this._listening = true;
4217
- this._crdt.clock.onTick((updates2) => {
4615
+ this.crdt.clock.onTick((updates2) => {
4218
4616
  void this._notify(updates2);
4219
4617
  });
4220
4618
  }
@@ -4233,13 +4631,13 @@ var Database = class {
4233
4631
  async query(field, opts = {}) {
4234
4632
  await this.ready();
4235
4633
  this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
4236
- const _crdt = this._crdt;
4237
- const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
4634
+ const _crdt = this.crdt;
4635
+ const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
4238
4636
  return await idx.query(opts);
4239
4637
  }
4240
4638
  async compact() {
4241
4639
  await this.ready();
4242
- await this._crdt.compact();
4640
+ await this.crdt.compact();
4243
4641
  }
4244
4642
  async _notify(updates) {
4245
4643
  await this.ready();
@@ -4263,23 +4661,62 @@ var Database = class {
4263
4661
  }
4264
4662
  }
4265
4663
  };
4266
- function toSortedArray(set) {
4267
- if (!set) return [];
4268
- return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
4664
+ function defaultURI(sthis, curi, uri, store, ctx) {
4665
+ ctx = ctx || {};
4666
+ const ret = (curi ? import_cement18.URI.from(curi) : uri).build().setParam("store" /* STORE */, store);
4667
+ if (!ret.hasParam("name" /* NAME */)) {
4668
+ const name = sthis.pathOps.basename(ret.URI().pathname);
4669
+ if (!name) {
4670
+ throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
4671
+ }
4672
+ ret.setParam("name" /* NAME */, name);
4673
+ }
4674
+ if (ctx.idx) {
4675
+ ret.defParam("index" /* INDEX */, "idx");
4676
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
4677
+ } else {
4678
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
4679
+ }
4680
+ if (store === "data") {
4681
+ if (ctx.file) {
4682
+ } else {
4683
+ ret.defParam("suffix" /* SUFFIX */, ".car");
4684
+ }
4685
+ }
4686
+ return ret.URI();
4269
4687
  }
4270
- function fireproof(name, opts) {
4271
- const key = JSON.stringify(
4272
- toSortedArray({
4273
- name,
4274
- stores: toSortedArray(opts?.store?.stores)
4275
- })
4276
- );
4277
- let db = Database.databases.get(key);
4278
- if (!db) {
4279
- db = new Database(name, opts);
4280
- Database.databases.set(key, db);
4688
+ function toStoreURIRuntime(sthis, name, sopts) {
4689
+ sopts = sopts || {};
4690
+ if (!sopts.base) {
4691
+ const fp_env = sthis.env.get("FP_STORAGE_URL");
4692
+ if (fp_env) {
4693
+ sopts = { ...sopts, base: import_cement18.BuildURI.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
4694
+ } else {
4695
+ sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
4696
+ }
4697
+ }
4698
+ const bbase = import_cement18.BuildURI.from(sopts.base);
4699
+ if (name) {
4700
+ bbase.setParam("name" /* NAME */, name);
4281
4701
  }
4282
- return db;
4702
+ const base = bbase.URI();
4703
+ return {
4704
+ idx: {
4705
+ data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
4706
+ file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
4707
+ meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
4708
+ wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
4709
+ },
4710
+ data: {
4711
+ data: defaultURI(sthis, sopts.data?.data, base, "data"),
4712
+ file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
4713
+ meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
4714
+ wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
4715
+ }
4716
+ };
4717
+ }
4718
+ function fireproof(name, opts) {
4719
+ return DatabaseFactory(name, opts);
4283
4720
  }
4284
4721
  function makeName(fnString) {
4285
4722
  const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
@@ -4298,6 +4735,9 @@ function makeName(fnString) {
4298
4735
  }
4299
4736
  }
4300
4737
 
4738
+ // src/index.ts
4739
+ init_types();
4740
+
4301
4741
  // src/runtime/index.ts
4302
4742
  var runtime_exports = {};
4303
4743
  __export(runtime_exports, {
@@ -4310,7 +4750,7 @@ __export(runtime_exports, {
4310
4750
  kb: () => key_bag_exports,
4311
4751
  kc: () => keyed_crypto_exports,
4312
4752
  mf: () => wait_pr_multiformats_exports,
4313
- runtimeFn: () => import_cement17.runtimeFn,
4753
+ runtimeFn: () => import_cement19.runtimeFn,
4314
4754
  toArrayBuffer: () => toArrayBuffer
4315
4755
  });
4316
4756
  init_utils2();
@@ -4326,7 +4766,7 @@ __export(wait_pr_multiformats_exports, {
4326
4766
  var codec_interface_exports = {};
4327
4767
 
4328
4768
  // src/runtime/index.ts
4329
- var import_cement17 = require("@adviser/cement");
4769
+ var import_cement19 = require("@adviser/cement");
4330
4770
  init_version();
4331
4771
  init_version2();
4332
4772
 
@@ -4335,6 +4775,6 @@ init_utils();
4335
4775
 
4336
4776
  // src/version.ts
4337
4777
  var PACKAGE_VERSION = Object.keys({
4338
- "0.19.100": "xxxx"
4778
+ "0.19.101": "xxxx"
4339
4779
  })[0];
4340
4780
  //# sourceMappingURL=index.cjs.map