@fireproof/core 0.19.100 → 0.19.101

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
  2. package/chunk-3EB3ENHT.js.map +1 -0
  3. package/chunk-HQ7D3PEU.js +61 -0
  4. package/chunk-HQ7D3PEU.js.map +1 -0
  5. package/chunk-PZ5AY32C.js +10 -0
  6. package/deno-filesystem-Q2IJ7YDR.js +57 -0
  7. package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
  8. package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
  9. package/gateway-GK5QZ6KP.js.map +1 -0
  10. package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
  11. package/gateway-TQTGDRCN.js.map +1 -0
  12. package/index.cjs +2158 -1718
  13. package/index.cjs.map +1 -1
  14. package/index.d.cts +261 -117
  15. package/index.d.ts +261 -117
  16. package/index.global.js +11834 -11354
  17. package/index.global.js.map +1 -1
  18. package/index.js +1865 -1519
  19. package/index.js.map +1 -1
  20. package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
  21. package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
  22. package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
  23. package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
  24. package/key-bag-memory-LWE6ARPX.js +29 -0
  25. package/key-bag-memory-LWE6ARPX.js.map +1 -0
  26. package/metafile-cjs.json +1 -1
  27. package/metafile-esm.json +1 -1
  28. package/metafile-iife.json +1 -1
  29. package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
  30. package/node-filesystem-CFRXFSO7.js.map +1 -0
  31. package/package.json +1 -1
  32. package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
  33. package/tests/blockstore/keyed-crypto.test.ts +63 -227
  34. package/tests/blockstore/loader.test.ts +19 -11
  35. package/tests/blockstore/store.test.ts +23 -19
  36. package/tests/blockstore/transaction.test.ts +12 -12
  37. package/tests/fireproof/all-gateway.test.ts +201 -193
  38. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
  39. package/tests/fireproof/crdt.test.ts +67 -16
  40. package/tests/fireproof/database.test.ts +183 -21
  41. package/tests/fireproof/fireproof.test.ts +83 -74
  42. package/tests/fireproof/hello.test.ts +18 -14
  43. package/tests/fireproof/indexer.test.ts +53 -43
  44. package/tests/fireproof/utils.test.ts +18 -6
  45. package/tests/gateway/file/loader-config.test.ts +303 -0
  46. package/tests/gateway/indexdb/loader-config.test.ts +75 -0
  47. package/tests/helpers.ts +27 -9
  48. package/tests/react/useFireproof.test.tsx +1 -1
  49. package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
  50. package/utils-STA2C35G.js.map +1 -0
  51. package/chunk-OFGPKRCM.js.map +0 -1
  52. package/chunk-WS3YRPIA.js +0 -75
  53. package/chunk-WS3YRPIA.js.map +0 -1
  54. package/gateway-5FCWPX5W.js.map +0 -1
  55. package/gateway-H7UD6TNB.js.map +0 -1
  56. package/mem-filesystem-YPPJV7Q2.js +0 -41
  57. package/mem-filesystem-YPPJV7Q2.js.map +0 -1
  58. package/node-filesystem-INX4ZTHE.js.map +0 -1
  59. package/tests/fireproof/config.test.ts +0 -172
  60. /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
  61. /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.cjs CHANGED
@@ -30,6 +30,47 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
30
30
  ));
31
31
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
32
32
 
33
+ // src/types.ts
34
+ function isFalsy(value) {
35
+ return value === false && value === null && value === void 0;
36
+ }
37
+ function throwFalsy(value) {
38
+ if (isFalsy(value)) {
39
+ throw new Error("value is Falsy");
40
+ }
41
+ return value;
42
+ }
43
+ function falsyToUndef(value) {
44
+ if (isFalsy(value)) {
45
+ return void 0;
46
+ }
47
+ return value;
48
+ }
49
+ var PARAM;
50
+ var init_types = __esm({
51
+ "src/types.ts"() {
52
+ "use strict";
53
+ PARAM = /* @__PURE__ */ ((PARAM2) => {
54
+ PARAM2["SUFFIX"] = "suffix";
55
+ PARAM2["URL_GEN"] = "urlGen";
56
+ PARAM2["STORE_KEY"] = "storekey";
57
+ PARAM2["STORE"] = "store";
58
+ PARAM2["KEY"] = "key";
59
+ PARAM2["INDEX"] = "index";
60
+ PARAM2["NAME"] = "name";
61
+ PARAM2["VERSION"] = "version";
62
+ PARAM2["FRAG_SIZE"] = "fragSize";
63
+ PARAM2["IV_VERIFY"] = "ivVerify";
64
+ PARAM2["IV_HASH"] = "ivHash";
65
+ PARAM2["FRAG_FID"] = "fid";
66
+ PARAM2["FRAG_OFS"] = "ofs";
67
+ PARAM2["FRAG_LEN"] = "len";
68
+ PARAM2["FRAG_HEAD"] = "headerSize";
69
+ return PARAM2;
70
+ })(PARAM || {});
71
+ }
72
+ });
73
+
33
74
  // src/utils.ts
34
75
  function presetEnv() {
35
76
  const penv = new Map([
@@ -87,6 +128,10 @@ function ensureLogger(sthis, componentName, ctx) {
87
128
  exposeStack = true;
88
129
  delete ctx.exposeStack;
89
130
  }
131
+ if ("exposeStack" in ctx) {
132
+ exposeStack = true;
133
+ delete ctx.exposeStack;
134
+ }
90
135
  if ("this" in ctx) {
91
136
  cLogger.Str("this", sthis.nextId(4).str);
92
137
  delete ctx.this;
@@ -153,10 +198,13 @@ function ensureLogger(sthis, componentName, ctx) {
153
198
  logger.SetExposeStack(true);
154
199
  }
155
200
  const out = cLogger.Logger();
201
+ if (sthis.env.get("FP_CONSTRUCTOR_DEBUG")) {
202
+ out.Debug().Msg("constructor");
203
+ }
156
204
  return out;
157
205
  }
158
206
  function getStore(url, sthis, joiner) {
159
- const store = url.getParam("store");
207
+ const store = url.getParam("store" /* STORE */);
160
208
  switch (store) {
161
209
  case "data":
162
210
  case "wal":
@@ -164,20 +212,21 @@ function getStore(url, sthis, joiner) {
164
212
  break;
165
213
  default:
166
214
  throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
215
+ throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
167
216
  }
168
217
  let name = store;
169
218
  if (url.hasParam("index")) {
170
- name = joiner(url.getParam("index") || "idx", name);
219
+ name = joiner(url.getParam("index" /* INDEX */) || "idx", name);
171
220
  }
172
221
  return { store, name };
173
222
  }
174
223
  function getKey(url, logger) {
175
- const result = url.getParam("key");
224
+ const result = url.getParam("key" /* KEY */);
176
225
  if (!result) throw logger.Error().Str("url", url.toString()).Msg(`key not found`).AsError();
177
226
  return result;
178
227
  }
179
228
  function getName(sthis, url) {
180
- let result = url.getParam("name");
229
+ let result = url.getParam("name" /* NAME */);
181
230
  if (!result) {
182
231
  result = sthis.pathOps.dirname(url.pathname);
183
232
  if (result.length === 0) {
@@ -197,17 +246,6 @@ function isNotFoundError(e) {
197
246
  if (e.code === "ENOENT") return true;
198
247
  return false;
199
248
  }
200
- function dataDir(sthis, name, base) {
201
- if (!base) {
202
- if (!(0, import_cement.runtimeFn)().isBrowser) {
203
- const home = sthis.env.get("HOME") || "./";
204
- base = sthis.env.get("FP_STORAGE_URL") || `file://${sthis.pathOps.join(home, ".fireproof")}`;
205
- } else {
206
- base = sthis.env.get("FP_STORAGE_URL") || `indexdb://fp`;
207
- }
208
- }
209
- return import_cement.URI.from(base.toString()).build().setParam("name", name || "").URI();
210
- }
211
249
  function UInt8ArrayEqual(a, b) {
212
250
  if (a.length !== b.length) {
213
251
  return false;
@@ -224,6 +262,7 @@ var init_utils = __esm({
224
262
  "src/utils.ts"() {
225
263
  "use strict";
226
264
  import_cement = require("@adviser/cement");
265
+ init_types();
227
266
  import_base58 = require("multiformats/bases/base58");
228
267
  globalLogger = new import_cement.LoggerImpl();
229
268
  registerFP_DEBUG = new import_cement.ResolveOnce();
@@ -274,6 +313,9 @@ var init_utils = __esm({
274
313
  dirname(path) {
275
314
  return path.split("/").slice(0, -1).join("/");
276
315
  }
316
+ basename(path) {
317
+ return path.split("/").pop() || "";
318
+ }
277
319
  // homedir() {
278
320
  // throw new Error("SysContainer:homedir is not available in seeded state");
279
321
  // }
@@ -292,45 +334,63 @@ var init_utils = __esm({
292
334
  }
293
335
  });
294
336
 
295
- // src/runtime/gateways/file/mem-filesystem.ts
296
- var mem_filesystem_exports = {};
297
- __export(mem_filesystem_exports, {
298
- MemFileSystem: () => MemFileSystem
337
+ // src/runtime/gateways/file/deno-filesystem.ts
338
+ var deno_filesystem_exports = {};
339
+ __export(deno_filesystem_exports, {
340
+ DenoFileSystem: () => DenoFileSystem
299
341
  });
300
- var import_memfs, MemFileSystem;
301
- var init_mem_filesystem = __esm({
302
- "src/runtime/gateways/file/mem-filesystem.ts"() {
342
+ var DenoFileSystem;
343
+ var init_deno_filesystem = __esm({
344
+ "src/runtime/gateways/file/deno-filesystem.ts"() {
303
345
  "use strict";
304
- import_memfs = require("memfs");
305
- init_utils2();
306
- MemFileSystem = class {
346
+ DenoFileSystem = class {
307
347
  async start() {
348
+ this.fs = Deno;
308
349
  return this;
309
350
  }
310
- mkdir(path, options) {
311
- return import_memfs.fs.promises.mkdir(path, options);
351
+ async mkdir(path, options) {
352
+ return this.fs?.mkdir(path, options).then(() => path);
312
353
  }
313
- readdir(path, options) {
314
- return import_memfs.fs.promises.readdir(path, options);
354
+ async readdir(path) {
355
+ const ret = [];
356
+ for await (const dirEntry of this.fs.readdir(path)) {
357
+ ret.push(dirEntry.name);
358
+ }
359
+ return ret;
315
360
  }
316
- rm(path, options) {
317
- return import_memfs.fs.promises.rm(path, options);
361
+ async rm(path, options) {
362
+ return this.fs?.rm(path, options);
318
363
  }
319
- copyFile(source, destination) {
320
- return import_memfs.fs.promises.copyFile(source, destination);
364
+ async copyFile(source, destination) {
365
+ return this.fs?.copyFile(source, destination);
321
366
  }
322
- async readfile(path, options) {
323
- const ret = await import_memfs.fs.promises.readFile(path, options);
324
- return toArrayBuffer(ret);
367
+ async readfile(path) {
368
+ return this.fs.readFile(path);
325
369
  }
326
- stat(path) {
327
- return import_memfs.fs.promises.stat(path);
370
+ async stat(path) {
371
+ const x = await this.fs.stat(path);
372
+ return {
373
+ isFile: () => x.isFile,
374
+ isDirectory: () => x.isDirectory,
375
+ isBlockDevice: () => !!x.isBlockDevice,
376
+ isCharacterDevice: () => !!x.isCharDevice,
377
+ isSymbolicLink: () => !!x.isSymlink,
378
+ isFIFO: () => !!x.isFifo,
379
+ isSocket: () => !!x.isSocket,
380
+ uid: x.uid,
381
+ gid: x.gid,
382
+ size: x.size,
383
+ atime: x.atime,
384
+ mtime: x.mtime,
385
+ ctime: x.birthtime,
386
+ birthtime: x.birthtime
387
+ };
328
388
  }
329
- unlink(path) {
330
- return import_memfs.fs.promises.unlink(path);
389
+ async unlink(path) {
390
+ return this.fs?.unlink(path);
331
391
  }
332
- writefile(path, data) {
333
- return import_memfs.fs.promises.writeFile(path, Buffer.from(data));
392
+ async writefile(path, data) {
393
+ return this.fs?.writeFile(path, Buffer.from(data));
334
394
  }
335
395
  };
336
396
  }
@@ -341,12 +401,11 @@ var node_filesystem_exports = {};
341
401
  __export(node_filesystem_exports, {
342
402
  NodeFileSystem: () => NodeFileSystem
343
403
  });
344
- var import_cement4, NodeFileSystem;
404
+ var NodeFileSystem;
345
405
  var init_node_filesystem = __esm({
346
406
  "src/runtime/gateways/file/node-filesystem.ts"() {
347
407
  "use strict";
348
408
  init_utils2();
349
- import_cement4 = require("@adviser/cement");
350
409
  NodeFileSystem = class {
351
410
  async start() {
352
411
  this.fs = await import("fs/promises");
@@ -369,15 +428,12 @@ var init_node_filesystem = __esm({
369
428
  return toArrayBuffer(ret);
370
429
  }
371
430
  stat(path) {
372
- return this.fs?.stat(path);
431
+ return this.fs.stat(path);
373
432
  }
374
433
  async unlink(path) {
375
434
  return this.fs?.unlink(path);
376
435
  }
377
436
  async writefile(path, data) {
378
- if ((0, import_cement4.runtimeFn)().isDeno) {
379
- return this.fs?.writeFile(path, data);
380
- }
381
437
  return this.fs?.writeFile(path, Buffer.from(data));
382
438
  }
383
439
  };
@@ -393,49 +449,34 @@ __export(utils_exports, {
393
449
  toArrayBuffer: () => toArrayBuffer
394
450
  });
395
451
  async function getFileSystem(url) {
396
- const name = url.getParam("fs");
397
- let fs2;
398
- switch (name) {
399
- case "mem":
400
- {
401
- const { MemFileSystem: MemFileSystem2 } = await Promise.resolve().then(() => (init_mem_filesystem(), mem_filesystem_exports));
402
- fs2 = new MemFileSystem2();
403
- }
404
- break;
405
- // case 'deno': {
406
- // const { DenoFileSystem } = await import("./deno-filesystem.js");
407
- // fs = new DenoFileSystem();
408
- // break;
409
- // }
410
- case "node": {
411
- const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
412
- fs2 = new NodeFileSystem2();
413
- break;
414
- }
415
- case "sys":
416
- default: {
417
- return getFileSystem(url.build().setParam("fs", "node").URI());
418
- }
452
+ let fs;
453
+ if ((0, import_cement4.runtimeFn)().isDeno) {
454
+ const { DenoFileSystem: DenoFileSystem2 } = await Promise.resolve().then(() => (init_deno_filesystem(), deno_filesystem_exports));
455
+ fs = new DenoFileSystem2();
456
+ } else if ((0, import_cement4.runtimeFn)().isNodeIsh) {
457
+ const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
458
+ fs = new NodeFileSystem2();
459
+ } else {
460
+ throw new Error("unsupported runtime");
419
461
  }
420
- return fs2.start();
462
+ return fs.start();
421
463
  }
422
464
  function getPath(url, sthis) {
423
465
  const basePath = url.pathname;
424
- const name = url.getParam("name");
466
+ const name = url.getParam("name" /* NAME */);
425
467
  if (name) {
426
- const version = url.getParam("version");
427
- if (!version) throw sthis.logger.Error().Url(url).Msg(`version not found`).AsError();
428
- return sthis.pathOps.join(basePath, version, name);
468
+ return sthis.pathOps.join(basePath, name);
429
469
  }
430
470
  return sthis.pathOps.join(basePath);
431
471
  }
432
472
  function getFileName(url, sthis) {
433
- const key = url.getParam("key");
473
+ const key = url.getParam("key" /* KEY */);
434
474
  if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
435
475
  const res = getStore(url, sthis, (...a) => a.join("-"));
436
476
  switch (res.store) {
437
- case "data":
438
- return sthis.pathOps.join(res.name, key + ".car");
477
+ case "data": {
478
+ return sthis.pathOps.join(res.name, key + (url.getParam("suffix" /* SUFFIX */) || ""));
479
+ }
439
480
  case "wal":
440
481
  case "meta":
441
482
  return sthis.pathOps.join(res.name, key + ".json");
@@ -454,10 +495,13 @@ function toArrayBuffer(buffer) {
454
495
  }
455
496
  return view;
456
497
  }
498
+ var import_cement4;
457
499
  var init_utils2 = __esm({
458
500
  "src/runtime/gateways/file/utils.ts"() {
459
501
  "use strict";
502
+ import_cement4 = require("@adviser/cement");
460
503
  init_utils();
504
+ init_types();
461
505
  }
462
506
  });
463
507
 
@@ -572,6 +616,40 @@ var init_key_bag_indexdb = __esm({
572
616
  }
573
617
  });
574
618
 
619
+ // src/runtime/key-bag-memory.ts
620
+ var key_bag_memory_exports = {};
621
+ __export(key_bag_memory_exports, {
622
+ KeyBagProviderMemory: () => KeyBagProviderMemory
623
+ });
624
+ var memoryKeyBag, KeyBagProviderMemory;
625
+ var init_key_bag_memory = __esm({
626
+ "src/runtime/key-bag-memory.ts"() {
627
+ "use strict";
628
+ memoryKeyBag = /* @__PURE__ */ new Map();
629
+ KeyBagProviderMemory = class {
630
+ constructor(url, sthis) {
631
+ this.url = url;
632
+ this.sthis = sthis;
633
+ }
634
+ key(id) {
635
+ return `${this.url.pathname}/${id}`;
636
+ }
637
+ async get(id) {
638
+ const binKeyItem = memoryKeyBag.get(this.key(id));
639
+ if (binKeyItem) {
640
+ const ki = JSON.parse(this.sthis.txt.decode(binKeyItem));
641
+ return ki;
642
+ }
643
+ return void 0;
644
+ }
645
+ async set(id, item) {
646
+ const p = this.sthis.txt.encode(JSON.stringify(item, null, 2));
647
+ memoryKeyBag.set(this.key(id), p);
648
+ }
649
+ };
650
+ }
651
+ });
652
+
575
653
  // src/runtime/gateways/file/version.ts
576
654
  var FILESTORE_VERSION;
577
655
  var init_version = __esm({
@@ -587,23 +665,24 @@ __export(gateway_exports, {
587
665
  FileGateway: () => FileGateway,
588
666
  FileTestStore: () => FileTestStore
589
667
  });
590
- var import_cement11, versionFiles, FileGateway, FileTestStore;
668
+ var import_cement12, versionFiles, FileGateway, FileTestStore;
591
669
  var init_gateway = __esm({
592
670
  "src/runtime/gateways/file/gateway.ts"() {
593
671
  "use strict";
594
672
  init_version();
595
- import_cement11 = require("@adviser/cement");
673
+ import_cement12 = require("@adviser/cement");
596
674
  init_utils();
597
675
  init_utils2();
598
- versionFiles = new import_cement11.KeyedResolvOnce();
676
+ init_types();
677
+ versionFiles = new import_cement12.KeyedResolvOnce();
599
678
  FileGateway = class {
600
679
  get fs() {
601
680
  if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
602
681
  return this._fs;
603
682
  }
604
683
  constructor(sthis) {
605
- this.sthis = sthis;
606
- this.logger = sthis.logger;
684
+ this.sthis = ensureSuperLog(sthis, "FileGateway", { this: 1 });
685
+ this.logger = this.sthis.logger;
607
686
  }
608
687
  async getVersionFromFile(path, logger) {
609
688
  return versionFiles.get(path).once(async () => {
@@ -625,34 +704,33 @@ var init_gateway = __esm({
625
704
  });
626
705
  }
627
706
  start(baseURL) {
628
- return (0, import_cement11.exception2Result)(async () => {
707
+ return (0, import_cement12.exception2Result)(async () => {
629
708
  this._fs = await getFileSystem(baseURL);
630
- await this.fs.start();
631
709
  const url = baseURL.build();
632
- url.defParam("version", FILESTORE_VERSION);
710
+ url.defParam("version" /* VERSION */, FILESTORE_VERSION);
633
711
  const dbUrl = await this.buildUrl(url.URI(), "dummy");
634
712
  const dbdirFile = this.getFilePath(dbUrl.Ok());
635
713
  await this.fs.mkdir(this.sthis.pathOps.dirname(dbdirFile), { recursive: true });
636
714
  const dbroot = this.sthis.pathOps.dirname(dbdirFile);
637
715
  this.logger.Debug().Url(url.URI()).Str("dbroot", dbroot).Msg("start");
638
- url.setParam("version", await this.getVersionFromFile(dbroot, this.logger));
716
+ url.setParam("version" /* VERSION */, await this.getVersionFromFile(dbroot, this.logger));
639
717
  return url.URI();
640
718
  });
641
719
  }
642
720
  async buildUrl(baseUrl, key) {
643
- return import_cement11.Result.Ok(baseUrl.build().setParam("key", key).URI());
721
+ return import_cement12.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI());
644
722
  }
645
723
  async close() {
646
- return import_cement11.Result.Ok(void 0);
724
+ return import_cement12.Result.Ok(void 0);
647
725
  }
648
726
  // abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
649
727
  getFilePath(url) {
650
- const key = url.getParam("key");
728
+ const key = url.getParam("key" /* KEY */);
651
729
  if (!key) throw this.logger.Error().Url(url).Msg(`key not found`).AsError();
652
730
  return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
653
731
  }
654
732
  async put(url, body) {
655
- return (0, import_cement11.exception2Result)(async () => {
733
+ return (0, import_cement12.exception2Result)(async () => {
656
734
  const file = await this.getFilePath(url);
657
735
  this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
658
736
  await this.fs.writefile(file, body);
@@ -662,19 +740,19 @@ var init_gateway = __esm({
662
740
  return exceptionWrapper(async () => {
663
741
  const file = this.getFilePath(url);
664
742
  try {
743
+ this.logger.Debug().Url(url).Str("file", file).Msg("get");
665
744
  const res = await this.fs.readfile(file);
666
- this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get");
667
- return import_cement11.Result.Ok(new Uint8Array(res));
745
+ return import_cement12.Result.Ok(new Uint8Array(res));
668
746
  } catch (e) {
669
747
  if (isNotFoundError(e)) {
670
- return import_cement11.Result.Err(new NotFoundError(`file not found: ${file}`));
748
+ return import_cement12.Result.Err(new NotFoundError(`file not found: ${file}`));
671
749
  }
672
- return import_cement11.Result.Err(e);
750
+ return import_cement12.Result.Err(e);
673
751
  }
674
752
  });
675
753
  }
676
754
  async delete(url) {
677
- return (0, import_cement11.exception2Result)(async () => {
755
+ return (0, import_cement12.exception2Result)(async () => {
678
756
  await this.fs.unlink(this.getFilePath(url));
679
757
  });
680
758
  }
@@ -700,7 +778,7 @@ var init_gateway = __esm({
700
778
  }
701
779
  }
702
780
  }
703
- return import_cement11.Result.Ok(void 0);
781
+ return import_cement12.Result.Ok(void 0);
704
782
  }
705
783
  };
706
784
  FileTestStore = class {
@@ -709,7 +787,7 @@ var init_gateway = __esm({
709
787
  this.sthis = sthis;
710
788
  }
711
789
  async get(iurl, key) {
712
- const url = iurl.build().setParam("key", key).URI();
790
+ const url = iurl.build().setParam("key" /* KEY */, key).URI();
713
791
  const dbFile = this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
714
792
  this.logger.Debug().Url(url).Str("dbFile", dbFile).Msg("get");
715
793
  const buffer = await (await getFileSystem(url)).readfile(dbFile);
@@ -737,7 +815,7 @@ __export(gateway_exports2, {
737
815
  getIndexDBName: () => getIndexDBName
738
816
  });
739
817
  function ensureVersion(url) {
740
- return url.build().defParam("version", INDEXDB_VERSION).URI();
818
+ return url.build().defParam("version" /* VERSION */, INDEXDB_VERSION).URI();
741
819
  }
742
820
  function sanitzeKey(key) {
743
821
  if (key.length === 1) {
@@ -758,17 +836,17 @@ async function connectIdb(url, sthis) {
758
836
  }
759
837
  });
760
838
  const found = await db.get("version", "version");
761
- const version = ensureVersion(url).getParam("version");
839
+ const version = ensureVersion(url).getParam("version" /* VERSION */);
762
840
  if (!found) {
763
841
  await db.put("version", { version }, "version");
764
842
  } else if (found.version !== version) {
765
- sthis.logger.Warn().Str("url", url.toString()).Str("version", version).Str("found", found.version).Msg("version mismatch");
843
+ sthis.logger.Warn().Url(url).Str("version", version).Str("found", found.version).Msg("version mismatch");
766
844
  }
767
845
  return { db, dbName, version, url };
768
846
  });
769
847
  return {
770
848
  ...once,
771
- url: url.build().setParam("version", once.version).URI()
849
+ url: url.build().setParam("version" /* VERSION */, once.version).URI()
772
850
  };
773
851
  }
774
852
  function joinDBName(...names) {
@@ -777,7 +855,7 @@ function joinDBName(...names) {
777
855
  function getIndexDBName(iurl, sthis) {
778
856
  const url = ensureVersion(iurl);
779
857
  const fullDb = url.pathname.replace(/^\/+/, "").replace(/\?.*$/, "");
780
- const dbName = url.getParam("name");
858
+ const dbName = url.getParam("name" /* NAME */);
781
859
  if (!dbName) throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
782
860
  const result = joinDBName(fullDb, dbName);
783
861
  const objStore = getStore(url, sthis, joinDBName).name;
@@ -789,15 +867,16 @@ function getIndexDBName(iurl, sthis) {
789
867
  dbName
790
868
  };
791
869
  }
792
- var import_idb2, import_cement12, onceIndexDB, IndexDBGateway, IndexDBTestStore;
870
+ var import_idb2, import_cement13, onceIndexDB, IndexDBGateway, IndexDBTestStore;
793
871
  var init_gateway2 = __esm({
794
872
  "src/runtime/gateways/indexdb/gateway.ts"() {
795
873
  "use strict";
796
874
  import_idb2 = require("idb");
797
- import_cement12 = require("@adviser/cement");
875
+ import_cement13 = require("@adviser/cement");
798
876
  init_version2();
799
877
  init_utils();
800
- onceIndexDB = new import_cement12.KeyedResolvOnce();
878
+ init_types();
879
+ onceIndexDB = new import_cement13.KeyedResolvOnce();
801
880
  IndexDBGateway = class {
802
881
  constructor(sthis) {
803
882
  this._db = {};
@@ -805,7 +884,7 @@ var init_gateway2 = __esm({
805
884
  this.sthis = sthis;
806
885
  }
807
886
  async start(baseURL) {
808
- return (0, import_cement12.exception2Result)(async () => {
887
+ return (0, import_cement13.exception2Result)(async () => {
809
888
  this.logger.Debug().Url(baseURL).Msg("starting");
810
889
  await this.sthis.start();
811
890
  const ic = await connectIdb(baseURL, this.sthis);
@@ -815,10 +894,10 @@ var init_gateway2 = __esm({
815
894
  });
816
895
  }
817
896
  async close() {
818
- return import_cement12.Result.Ok(void 0);
897
+ return import_cement13.Result.Ok(void 0);
819
898
  }
820
899
  async destroy(baseUrl) {
821
- return (0, import_cement12.exception2Result)(async () => {
900
+ return (0, import_cement13.exception2Result)(async () => {
822
901
  const type = getStore(baseUrl, this.sthis, joinDBName).name;
823
902
  const idb = this._db;
824
903
  const trans = idb.transaction(type, "readwrite");
@@ -834,7 +913,7 @@ var init_gateway2 = __esm({
834
913
  });
835
914
  }
836
915
  buildUrl(baseUrl, key) {
837
- return Promise.resolve(import_cement12.Result.Ok(baseUrl.build().setParam("key", key).URI()));
916
+ return Promise.resolve(import_cement13.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
838
917
  }
839
918
  async get(url) {
840
919
  return exceptionWrapper(async () => {
@@ -845,13 +924,13 @@ var init_gateway2 = __esm({
845
924
  const bytes = await tx.objectStore(store).get(sanitzeKey(key));
846
925
  await tx.done;
847
926
  if (!bytes) {
848
- return import_cement12.Result.Err(new NotFoundError(`missing ${key}`));
927
+ return import_cement13.Result.Err(new NotFoundError(`missing ${key}`));
849
928
  }
850
- return import_cement12.Result.Ok(bytes);
929
+ return import_cement13.Result.Ok(bytes);
851
930
  });
852
931
  }
853
932
  async put(url, value) {
854
- return (0, import_cement12.exception2Result)(async () => {
933
+ return (0, import_cement13.exception2Result)(async () => {
855
934
  const key = getKey(url, this.logger);
856
935
  const store = getStore(url, this.sthis, joinDBName).name;
857
936
  this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
@@ -861,14 +940,14 @@ var init_gateway2 = __esm({
861
940
  });
862
941
  }
863
942
  async delete(url) {
864
- return (0, import_cement12.exception2Result)(async () => {
943
+ return (0, import_cement13.exception2Result)(async () => {
865
944
  const key = getKey(url, this.logger);
866
945
  const store = getStore(url, this.sthis, joinDBName).name;
867
946
  this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
868
947
  const tx = this._db.transaction([store], "readwrite");
869
948
  await tx.objectStore(store).delete(sanitzeKey(key));
870
949
  await tx.done;
871
- return import_cement12.Result.Ok(void 0);
950
+ return import_cement13.Result.Ok(void 0);
872
951
  });
873
952
  }
874
953
  };
@@ -896,15 +975,16 @@ var init_gateway2 = __esm({
896
975
  var src_exports = {};
897
976
  __export(src_exports, {
898
977
  CRDT: () => CRDT,
899
- Database: () => Database,
978
+ DatabaseFactory: () => DatabaseFactory,
979
+ DatabaseShell: () => DatabaseShell,
900
980
  Index: () => Index,
901
981
  NotFoundError: () => NotFoundError,
902
982
  PACKAGE_VERSION: () => PACKAGE_VERSION,
983
+ PARAM: () => PARAM,
903
984
  Result: () => import_cement.Result,
904
985
  UInt8ArrayEqual: () => UInt8ArrayEqual,
905
986
  blockstore: () => blockstore_exports,
906
987
  bs: () => blockstore_exports,
907
- dataDir: () => dataDir,
908
988
  ensureLogger: () => ensureLogger,
909
989
  ensureSuperLog: () => ensureSuperLog,
910
990
  ensureSuperThis: () => ensureSuperThis,
@@ -915,16 +995,19 @@ __export(src_exports, {
915
995
  getName: () => getName,
916
996
  getStore: () => getStore,
917
997
  index: () => index,
998
+ isDatabase: () => isDatabase,
918
999
  isFalsy: () => isFalsy,
919
1000
  isNotFoundError: () => isNotFoundError,
1001
+ keyConfigOpts: () => keyConfigOpts,
920
1002
  rt: () => runtime_exports,
921
1003
  runtime: () => runtime_exports,
922
- throwFalsy: () => throwFalsy
1004
+ throwFalsy: () => throwFalsy,
1005
+ toStoreURIRuntime: () => toStoreURIRuntime
923
1006
  });
924
1007
  module.exports = __toCommonJS(src_exports);
925
1008
 
926
1009
  // src/database.ts
927
- var import_cement16 = require("@adviser/cement");
1010
+ var import_cement18 = require("@adviser/cement");
928
1011
 
929
1012
  // src/write-queue.ts
930
1013
  function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -967,86 +1050,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
967
1050
  }
968
1051
 
969
1052
  // src/crdt.ts
970
- var import_cement15 = require("@adviser/cement");
971
-
972
- // src/runtime/wait-pr-multiformats/block.ts
973
- var block_exports = {};
974
- __export(block_exports, {
975
- Block: () => Block,
976
- create: () => create,
977
- createUnsafe: () => createUnsafe,
978
- decode: () => decode,
979
- encode: () => encode
980
- });
981
- var import_multiformats = require("multiformats");
982
- var import_block = require("multiformats/block");
983
- var Block = import_block.Block;
984
- async function decode({
985
- bytes,
986
- codec: codec3,
987
- hasher: hasher7
988
- }) {
989
- if (bytes == null) throw new Error('Missing required argument "bytes"');
990
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
991
- const value = await Promise.resolve(codec3.decode(bytes));
992
- const hash = await hasher7.digest(bytes);
993
- const cid = import_multiformats.CID.create(1, codec3.code, hash);
994
- return new import_block.Block({ value, bytes, cid });
995
- }
996
- async function encode({
997
- value,
998
- codec: codec3,
999
- hasher: hasher7
1000
- }) {
1001
- if (typeof value === "undefined") throw new Error('Missing required argument "value"');
1002
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
1003
- const bytes = await Promise.resolve(codec3.encode(value));
1004
- const hash = await hasher7.digest(bytes);
1005
- const cid = import_multiformats.CID.create(1, codec3.code, hash);
1006
- return new import_block.Block({ value, bytes, cid });
1007
- }
1008
- async function create({
1009
- bytes,
1010
- cid,
1011
- hasher: hasher7,
1012
- codec: codec3
1013
- }) {
1014
- if (bytes == null) throw new Error('Missing required argument "bytes"');
1015
- if (hasher7 == null) throw new Error('Missing required argument "hasher"');
1016
- const value = await Promise.resolve(codec3.decode(bytes));
1017
- const hash = await hasher7.digest(bytes);
1018
- if (!import_multiformats.bytes.equals(cid.multihash.bytes, hash.bytes)) {
1019
- throw new Error("CID hash does not match bytes");
1020
- }
1021
- return createUnsafe({
1022
- bytes,
1023
- cid,
1024
- value,
1025
- codec: codec3
1026
- });
1027
- }
1028
- async function createUnsafe({
1029
- bytes,
1030
- cid,
1031
- value: maybeValue,
1032
- codec: codec3
1033
- }) {
1034
- const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
1035
- if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
1036
- return new Block({
1037
- cid,
1038
- bytes,
1039
- value
1040
- });
1041
- }
1042
-
1043
- // src/crdt-helpers.ts
1044
- var import_link = require("multiformats/link");
1045
- var import_sha25 = require("multiformats/hashes/sha2");
1046
- var codec = __toESM(require("@ipld/dag-cbor"), 1);
1047
- var import_crdt = require("@web3-storage/pail/crdt");
1048
- var import_clock2 = require("@web3-storage/pail/clock");
1049
- var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
1053
+ var import_cement17 = require("@adviser/cement");
1050
1054
 
1051
1055
  // src/blockstore/index.ts
1052
1056
  var blockstore_exports = {};
@@ -1059,8 +1063,11 @@ __export(blockstore_exports, {
1059
1063
  FragmentGateway: () => FragmentGateway,
1060
1064
  Loader: () => Loader,
1061
1065
  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
1062
- ensureStart: () => ensureStart,
1063
- getGatewayFromURL: () => getGatewayFromURL,
1066
+ ensureStoreEnDeFile: () => ensureStoreEnDeFile,
1067
+ fileGatewayFactoryItem: () => fileGatewayFactoryItem,
1068
+ getDefaultURI: () => getDefaultURI,
1069
+ getGatewayFactoryItem: () => getGatewayFactoryItem,
1070
+ getStartedGateway: () => getStartedGateway,
1064
1071
  parseCarFile: () => parseCarFile,
1065
1072
  registerStoreProtocol: () => registerStoreProtocol,
1066
1073
  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
@@ -1075,7 +1082,7 @@ function toCIDBlock(block) {
1075
1082
  }
1076
1083
 
1077
1084
  // src/blockstore/store-factory.ts
1078
- var import_cement13 = require("@adviser/cement");
1085
+ var import_cement15 = require("@adviser/cement");
1079
1086
 
1080
1087
  // src/runtime/files.ts
1081
1088
  var files_exports = {};
@@ -1151,258 +1158,24 @@ var UnixFSFileBuilder = class {
1151
1158
  // src/blockstore/store.ts
1152
1159
  var import_dag_json2 = require("@ipld/dag-json");
1153
1160
  var import_cement10 = require("@adviser/cement");
1154
-
1155
- // src/types.ts
1156
- function isFalsy(value) {
1157
- return value === false && value === null && value === void 0;
1158
- }
1159
- function throwFalsy(value) {
1160
- if (isFalsy(value)) {
1161
- throw new Error("value is Falsy");
1162
- }
1163
- return value;
1164
- }
1165
- function falsyToUndef(value) {
1166
- if (isFalsy(value)) {
1167
- return void 0;
1168
- }
1169
- return value;
1170
- }
1171
-
1172
- // src/blockstore/store.ts
1161
+ init_types();
1173
1162
  init_utils();
1174
1163
 
1175
- // src/blockstore/loader.ts
1176
- var import_p_limit = __toESM(require("p-limit"), 1);
1177
- var import_car = require("@ipld/car");
1178
- var import_cement7 = require("@adviser/cement");
1179
-
1180
- // src/blockstore/loader-helpers.ts
1181
- var import_sha2 = require("multiformats/hashes/sha2");
1182
- var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
1183
- async function parseCarFile(reader, logger) {
1184
- const roots = await reader.getRoots();
1185
- const header = await reader.get(roots[0]);
1186
- if (!header) throw logger.Error().Msg("missing header block").AsError();
1187
- const dec = await decode({ bytes: header.bytes, hasher: import_sha2.sha256, codec: dagCodec });
1188
- const fpvalue = dec.value;
1189
- if (fpvalue && !fpvalue.fp) {
1190
- throw logger.Error().Msg("missing fp").AsError();
1191
- }
1192
- return fpvalue.fp;
1193
- }
1194
-
1195
- // src/blockstore/transaction.ts
1196
- var import_block3 = require("@web3-storage/pail/block");
1164
+ // src/blockstore/commit-queue.ts
1197
1165
  var import_cement2 = require("@adviser/cement");
1198
- init_utils();
1199
- var CarTransaction = class extends import_block3.MemoryBlockstore {
1200
- constructor(parent, opts = { add: true, noLoader: false }) {
1201
- super();
1202
- if (opts.add) {
1203
- parent.transactions.add(this);
1204
- }
1205
- this.parent = parent;
1206
- }
1207
- async get(cid) {
1208
- return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
1209
- }
1210
- async superGet(cid) {
1211
- return super.get(cid);
1166
+ var CommitQueue = class {
1167
+ constructor() {
1168
+ this.queue = [];
1169
+ this.processing = false;
1170
+ this._waitIdleItems = /* @__PURE__ */ new Set();
1212
1171
  }
1213
- };
1214
- function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1215
- const logger = ensureLogger(sthis, component, ctx);
1216
- const store = opts.store || {};
1217
- return {
1218
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1219
- applyMeta: (meta, snap) => {
1172
+ waitIdle() {
1173
+ if (this.queue.length === 0 && !this.processing) {
1220
1174
  return Promise.resolve();
1221
- },
1222
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1223
- compact: async (blocks) => {
1224
- return {};
1225
- },
1226
- autoCompact: 100,
1227
- public: false,
1228
- name: void 0,
1229
- threshold: 1e3 * 1e3,
1230
- ...opts,
1231
- logger,
1232
- keyBag: opts.keyBag || {},
1233
- crypto: (0, import_cement2.toCryptoRuntime)(opts.crypto),
1234
- store,
1235
- storeRuntime: toStoreRuntime(store, sthis)
1236
- };
1237
- }
1238
- function blockstoreFactory(sthis, opts) {
1239
- if (opts.name) {
1240
- return new EncryptedBlockstore(sthis, opts);
1241
- } else {
1242
- return new BaseBlockstore(opts);
1243
- }
1244
- }
1245
- var BaseBlockstore = class {
1246
- constructor(ebOpts = {}) {
1247
- this.transactions = /* @__PURE__ */ new Set();
1248
- this.sthis = ensureSuperThis(ebOpts);
1249
- this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1250
- this.logger = this.ebOpts.logger;
1251
- }
1252
- // ready: Promise<void>;
1253
- ready() {
1254
- return Promise.resolve();
1255
- }
1256
- async close() {
1257
- }
1258
- async destroy() {
1259
- }
1260
- async get(cid) {
1261
- if (!cid) throw this.logger.Error().Msg("required cid").AsError();
1262
- for (const f of this.transactions) {
1263
- const v = await f.superGet(cid);
1264
- if (v) return v;
1265
- }
1266
- }
1267
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1268
- async put(cid, block) {
1269
- throw this.logger.Error().Msg("use a transaction to put").AsError();
1270
- }
1271
- // TransactionMeta
1272
- async transaction(fn, _opts) {
1273
- const t = new CarTransaction(this, _opts);
1274
- const done = await fn(t);
1275
- this.lastTxMeta = done;
1276
- return { t, meta: done };
1277
- }
1278
- async *entries() {
1279
- const seen = /* @__PURE__ */ new Set();
1280
- for (const t of this.transactions) {
1281
- for await (const blk of t.entries()) {
1282
- if (seen.has(blk.cid.toString())) continue;
1283
- seen.add(blk.cid.toString());
1284
- yield blk;
1285
- }
1286
- }
1287
- }
1288
- };
1289
- var EncryptedBlockstore = class extends BaseBlockstore {
1290
- constructor(sthis, ebOpts) {
1291
- super(ebOpts);
1292
- this.compacting = false;
1293
- this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
1294
- const { name } = ebOpts;
1295
- if (!name) {
1296
- throw this.logger.Error().Msg("name required").AsError();
1297
- }
1298
- this.name = name;
1299
- this.loader = new Loader(this.name, ebOpts, sthis);
1300
- }
1301
- ready() {
1302
- return this.loader.ready();
1303
- }
1304
- close() {
1305
- return this.loader.close();
1306
- }
1307
- destroy() {
1308
- return this.loader.destroy();
1309
- }
1310
- async get(cid) {
1311
- const got = await super.get(cid);
1312
- if (got) return got;
1313
- if (!this.loader) {
1314
- return;
1315
- }
1316
- return falsyToUndef(await this.loader.getBlock(cid));
1317
- }
1318
- async transaction(fn, opts = { noLoader: false }) {
1319
- const { t, meta: done } = await super.transaction(fn);
1320
- const cars = await this.loader.commit(t, done, opts);
1321
- if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1322
- setTimeout(() => void this.compact(), 10);
1323
- }
1324
- if (cars) {
1325
- this.transactions.delete(t);
1326
- return { meta: done, cars, t };
1327
- }
1328
- throw this.logger.Error().Msg("failed to commit car files").AsError();
1329
- }
1330
- async getFile(car, cid) {
1331
- await this.ready();
1332
- if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
1333
- const reader = await this.loader.loadFileCar(
1334
- car
1335
- /*, isPublic */
1336
- );
1337
- const block = await reader.get(cid);
1338
- if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
1339
- return block.bytes;
1340
- }
1341
- async compact() {
1342
- await this.ready();
1343
- if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
1344
- if (this.loader.carLog.length < 2) return;
1345
- const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
1346
- if (!compactFn || this.compacting) return;
1347
- const blockLog = new CompactionFetcher(this);
1348
- this.compacting = true;
1349
- const meta = await compactFn(blockLog);
1350
- await this.loader?.commit(blockLog.loggedBlocks, meta, {
1351
- compact: true,
1352
- noLoader: true
1353
- });
1354
- this.compacting = false;
1355
- }
1356
- async defaultCompact(blocks, logger) {
1357
- if (!this.loader) {
1358
- throw logger.Error().Msg("no loader").AsError();
1359
- }
1360
- if (!this.lastTxMeta) {
1361
- throw logger.Error().Msg("no lastTxMeta").AsError();
1362
- }
1363
- for await (const blk of this.loader.entries(false)) {
1364
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1365
- }
1366
- for (const t of this.transactions) {
1367
- for await (const blk of t.entries()) {
1368
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1369
- }
1370
- }
1371
- return this.lastTxMeta;
1372
- }
1373
- async *entries() {
1374
- for await (const blk of this.loader.entries()) {
1375
- yield blk;
1376
- }
1377
- }
1378
- };
1379
- var CompactionFetcher = class {
1380
- constructor(blocks) {
1381
- this.blockstore = blocks;
1382
- this.loggedBlocks = new CarTransaction(blocks);
1383
- }
1384
- async get(cid) {
1385
- const block = await this.blockstore.get(cid);
1386
- if (block) this.loggedBlocks.putSync(cid, block.bytes);
1387
- return falsyToUndef(block);
1388
- }
1389
- };
1390
-
1391
- // src/blockstore/commit-queue.ts
1392
- var import_cement3 = require("@adviser/cement");
1393
- var CommitQueue = class {
1394
- constructor() {
1395
- this.queue = [];
1396
- this.processing = false;
1397
- this._waitIdleItems = /* @__PURE__ */ new Set();
1398
- }
1399
- waitIdle() {
1400
- if (this.queue.length === 0 && !this.processing) {
1401
- return Promise.resolve();
1402
- }
1403
- const fn = new import_cement3.Future();
1404
- this._waitIdleItems.add(fn);
1405
- return fn.asPromise();
1175
+ }
1176
+ const fn = new import_cement2.Future();
1177
+ this._waitIdleItems.add(fn);
1178
+ return fn.asPromise();
1406
1179
  }
1407
1180
  async enqueue(fn) {
1408
1181
  return new Promise((resolve, reject) => {
@@ -1439,1132 +1212,1501 @@ var CommitQueue = class {
1439
1212
  }
1440
1213
  };
1441
1214
 
1442
- // src/runtime/key-bag.ts
1443
- var key_bag_exports = {};
1444
- __export(key_bag_exports, {
1445
- KeyBag: () => KeyBag,
1446
- getKeyBag: () => getKeyBag,
1447
- registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
1215
+ // src/runtime/keyed-crypto.ts
1216
+ var keyed_crypto_exports = {};
1217
+ __export(keyed_crypto_exports, {
1218
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1219
+ keyedCryptoFactory: () => keyedCryptoFactory
1448
1220
  });
1449
- var import_cement6 = require("@adviser/cement");
1450
1221
  init_utils();
1451
1222
  var import_base582 = require("multiformats/bases/base58");
1452
- var KeyBag = class {
1453
- constructor(rt) {
1454
- this.rt = rt;
1455
- this._warnOnce = new import_cement6.ResolveOnce();
1456
- this._seq = new import_cement6.ResolveSeq();
1457
- this.logger = ensureLogger(rt.sthis, "KeyBag");
1458
- this.logger.Debug().Msg("KeyBag created");
1459
- }
1460
- async subtleKey(key) {
1461
- const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
1462
- if (extractable) {
1463
- this._warnOnce.once(
1464
- () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
1465
- );
1466
- }
1467
- return await this.rt.crypto.importKey(
1468
- "raw",
1469
- // raw or jwk
1470
- import_base582.base58btc.decode(key),
1471
- // hexStringToUint8Array(key), // raw data
1472
- "AES-GCM",
1473
- extractable,
1474
- ["encrypt", "decrypt"]
1475
- );
1476
- }
1477
- async ensureKeyFromUrl(url, keyFactory) {
1478
- const storeKey = url.getParam("storekey");
1479
- if (storeKey === "insecure") {
1480
- return import_cement6.Result.Ok(url);
1481
- }
1482
- if (!storeKey) {
1483
- const keyName = `@${keyFactory()}@`;
1484
- const ret = await this.getNamedKey(keyName);
1485
- if (ret.isErr()) {
1486
- return ret;
1487
- }
1488
- const urb = url.build().setParam("storekey", keyName);
1489
- return import_cement6.Result.Ok(urb.URI());
1223
+ var import_sha2 = require("multiformats/hashes/sha2");
1224
+ var CBOR = __toESM(require("cborg"), 1);
1225
+ init_types();
1226
+ var generateIV = {
1227
+ random: {
1228
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1229
+ calc: async (ko, crypto, data) => {
1230
+ return crypto.randomBytes(ko.ivLength);
1231
+ },
1232
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1233
+ verify: async (ko, crypto, iv, data) => {
1234
+ return true;
1490
1235
  }
1491
- if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
1492
- const ret = await this.getNamedKey(storeKey);
1493
- if (ret.isErr()) {
1494
- return ret;
1236
+ },
1237
+ hash: {
1238
+ calc: async (ko, crypto, data) => {
1239
+ const hash = await import_sha2.sha256.digest(data);
1240
+ const hashBytes = new Uint8Array(hash.bytes);
1241
+ const hashArray = new Uint8Array(ko.ivLength);
1242
+ for (let i = 0; i < hashBytes.length; i++) {
1243
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
1495
1244
  }
1245
+ return hashArray;
1246
+ },
1247
+ verify: async function(ko, crypto, iv, data) {
1248
+ return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1496
1249
  }
1497
- return import_cement6.Result.Ok(url);
1498
1250
  }
1499
- async toKeyWithFingerPrint(keyStr) {
1500
- const material = import_base582.base58btc.decode(keyStr);
1501
- const key = await this.subtleKey(keyStr);
1502
- const fpr = await this.rt.crypto.digestSHA256(material);
1503
- return import_cement6.Result.Ok({
1504
- key,
1505
- fingerPrint: import_base582.base58btc.encode(new Uint8Array(fpr))
1251
+ };
1252
+ function getGenerateIVFn(url, opts) {
1253
+ const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
1254
+ return generateIV[ivhash] || generateIV["hash"];
1255
+ }
1256
+ var BlockIvKeyIdCodec = class {
1257
+ constructor(ko, iv, opts) {
1258
+ this.code = 3147065;
1259
+ this.name = "Fireproof@encrypted-block:aes-gcm";
1260
+ this.ko = ko;
1261
+ this.iv = iv;
1262
+ this.opts = opts || {};
1263
+ }
1264
+ async encode(data) {
1265
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1266
+ const { iv } = this.ko.algo(calcIv);
1267
+ const fprt = await this.ko.fingerPrint();
1268
+ const keyId = import_base582.base58btc.decode(fprt);
1269
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1270
+ return CBOR.encode({
1271
+ iv,
1272
+ keyId,
1273
+ data: await this.ko._encrypt({ iv, bytes: data })
1506
1274
  });
1507
1275
  }
1508
- async setNamedKey(name, key) {
1509
- return this._seq.add(() => this._setNamedKey(name, key));
1276
+ async decode(abytes) {
1277
+ let bytes;
1278
+ if (abytes instanceof Uint8Array) {
1279
+ bytes = abytes;
1280
+ } else {
1281
+ bytes = new Uint8Array(abytes);
1282
+ }
1283
+ const { iv, keyId, data } = CBOR.decode(bytes);
1284
+ const fprt = await this.ko.fingerPrint();
1285
+ this.ko.logger.Debug().Str("fp", import_base582.base58btc.encode(keyId)).Msg("decode");
1286
+ if (import_base582.base58btc.encode(keyId) !== fprt) {
1287
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base582.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
1288
+ }
1289
+ const result = await this.ko._decrypt({ iv, bytes: data });
1290
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1291
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1292
+ }
1293
+ return result;
1510
1294
  }
1511
- // avoid deadlock
1512
- async _setNamedKey(name, key) {
1513
- const item = {
1514
- name,
1515
- key
1295
+ };
1296
+ var keyedCrypto = class {
1297
+ constructor(url, key, cyopt, sthis) {
1298
+ this.ivLength = 12;
1299
+ this.isEncrypting = true;
1300
+ this.logger = ensureLogger(sthis, "keyedCrypto");
1301
+ this.crypto = cyopt;
1302
+ this.key = key;
1303
+ this.url = url;
1304
+ }
1305
+ fingerPrint() {
1306
+ return Promise.resolve(this.key.fingerPrint);
1307
+ }
1308
+ codec(iv, opts) {
1309
+ return new BlockIvKeyIdCodec(this, iv, opts);
1310
+ }
1311
+ algo(iv) {
1312
+ return {
1313
+ name: "AES-GCM",
1314
+ iv: iv || this.crypto.randomBytes(this.ivLength),
1315
+ tagLength: 128
1516
1316
  };
1517
- const bag = await this.rt.getBag();
1518
- this.logger.Debug().Str("name", name).Msg("setNamedKey");
1519
- await bag.set(name, item);
1520
- return await this.toKeyWithFingerPrint(item.key);
1521
1317
  }
1522
- async getNamedExtractableKey(name, failIfNotFound = false) {
1523
- const ret = await this.getNamedKey(name, failIfNotFound);
1524
- if (ret.isErr()) {
1525
- return ret;
1526
- }
1527
- const named = ret.Ok();
1528
- return import_cement6.Result.Ok({
1529
- ...named,
1530
- extract: async () => {
1531
- const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
1532
- return {
1533
- key: ext,
1534
- keyStr: import_base582.base58btc.encode(ext)
1535
- };
1536
- }
1537
- });
1318
+ async _decrypt(data) {
1319
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
1320
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
1538
1321
  }
1539
- async getNamedKey(name, failIfNotFound = false) {
1540
- const id = this.rt.sthis.nextId(4).str;
1541
- return this._seq.add(async () => {
1542
- const bag = await this.rt.getBag();
1543
- const named = await bag.get(name);
1544
- if (named) {
1545
- const fpr = await this.toKeyWithFingerPrint(named.key);
1546
- this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
1547
- return fpr;
1548
- }
1549
- if (failIfNotFound) {
1550
- this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
1551
- return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
1552
- }
1553
- const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
1554
- this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
1555
- return ret;
1556
- });
1322
+ async _encrypt(data) {
1323
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
1324
+ const a = this.algo(data.iv);
1325
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
1557
1326
  }
1558
1327
  };
1559
- var keyBagProviderFactories = new Map(
1560
- [
1561
- {
1562
- protocol: "file:",
1563
- factory: async (url, sthis) => {
1564
- const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1565
- return new KeyBagProviderFile2(url, sthis);
1566
- }
1567
- },
1568
- {
1569
- protocol: "indexdb:",
1570
- factory: async (url, sthis) => {
1571
- const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1572
- return new KeyBagProviderIndexDB2(url, sthis);
1573
- }
1574
- }
1575
- ].map((i) => [i.protocol, i])
1576
- );
1577
- function registerKeyBagProviderFactory(item) {
1578
- const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
1579
- keyBagProviderFactories.set(protocol, {
1580
- ...item,
1581
- protocol
1582
- });
1583
- }
1584
- function defaultKeyBagOpts(sthis, kbo) {
1585
- if (kbo.keyRuntime) {
1586
- return kbo.keyRuntime;
1328
+ var nullCodec = class {
1329
+ constructor() {
1330
+ this.code = 0;
1331
+ this.name = "Fireproof@unencrypted-block";
1587
1332
  }
1588
- const logger = ensureLogger(sthis, "KeyBag");
1589
- let url;
1590
- if (kbo.url) {
1591
- url = import_cement6.URI.from(kbo.url);
1592
- logger.Debug().Url(url).Msg("from opts");
1593
- } else {
1594
- let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
1595
- if ((0, import_cement6.runtimeFn)().isBrowser) {
1596
- url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
1597
- } else {
1598
- if (!bagFnameOrUrl) {
1599
- const home = sthis.env.get("HOME");
1600
- bagFnameOrUrl = `${home}/.fireproof/keybag`;
1601
- url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
1602
- } else {
1603
- url = import_cement6.URI.from(bagFnameOrUrl);
1604
- }
1605
- }
1606
- logger.Debug().Url(url).Msg("from env");
1333
+ encode(data) {
1334
+ return data;
1607
1335
  }
1608
- const kitem = keyBagProviderFactories.get(url.protocol);
1609
- if (!kitem) {
1610
- throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
1336
+ decode(data) {
1337
+ return data;
1611
1338
  }
1612
- const getBag = async () => kitem.factory(url, sthis);
1613
- if (url.hasParam("masterkey")) {
1614
- throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
1339
+ };
1340
+ var noCrypto = class {
1341
+ constructor(url, cyrt, sthis) {
1342
+ this.ivLength = 0;
1343
+ this.code = 0;
1344
+ this.name = "Fireproof@unencrypted-block";
1345
+ this.isEncrypting = false;
1346
+ this._fingerPrint = "noCrypto:" + Math.random();
1347
+ this.logger = ensureLogger(sthis, "noCrypto");
1348
+ this.crypto = cyrt;
1349
+ this.url = url;
1615
1350
  }
1616
- return {
1617
- url,
1618
- crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
1619
- sthis,
1620
- logger,
1621
- keyLength: kbo.keyLength || 16,
1622
- getBag,
1623
- id: () => {
1624
- return url.toString();
1625
- }
1626
- };
1627
- }
1628
- var _keyBags = new import_cement6.KeyedResolvOnce();
1629
- async function getKeyBag(sthis, kbo = {}) {
1630
- await sthis.start();
1631
- const rt = defaultKeyBagOpts(sthis, kbo);
1632
- return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
1633
- }
1634
-
1635
- // src/blockstore/commitor.ts
1636
- var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
1637
- var import_sha22 = require("multiformats/hashes/sha2");
1638
- var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
1639
- async function encodeCarFile(roots, t, codec3) {
1640
- let size = 0;
1641
- const headerSize = CBW.headerLength({ roots });
1642
- size += headerSize;
1643
- for (const { cid, bytes } of t.entries()) {
1644
- size += CBW.blockLength({ cid, bytes });
1351
+ fingerPrint() {
1352
+ return Promise.resolve(this._fingerPrint);
1645
1353
  }
1646
- const buffer = new Uint8Array(size);
1647
- const writer = CBW.createWriter(buffer, { headerSize });
1648
- for (const r of roots) {
1649
- writer.addRoot(r);
1354
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1355
+ codec(iv) {
1356
+ return new nullCodec();
1650
1357
  }
1651
- for (const { cid, bytes } of t.entries()) {
1652
- writer.write({ cid, bytes });
1358
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1359
+ algo(iv) {
1360
+ return {
1361
+ name: "noCrypto",
1362
+ iv: new Uint8Array(),
1363
+ tagLength: 0
1364
+ };
1653
1365
  }
1654
- writer.close();
1655
- return await encode({ value: writer.bytes, hasher: import_sha22.sha256, codec: codec3 });
1656
- }
1657
- async function createCarFile(encoder, cid, t) {
1658
- return encodeCarFile([cid], t, encoder);
1659
- }
1660
- async function commitFiles(fileStore, walStore, t, done) {
1661
- const { files: roots } = makeFileCarHeader(done);
1662
- const cids = [];
1663
- const codec3 = (await fileStore.keyedCrypto()).codec();
1664
- const cars = await prepareCarFilesFiles(codec3, roots, t);
1665
- for (const car of cars) {
1666
- const { cid, bytes } = car;
1667
- await fileStore.save({ cid, bytes });
1668
- await walStore.enqueueFile(
1669
- cid
1670
- /*, !!opts.public*/
1671
- );
1672
- cids.push(cid);
1366
+ _decrypt() {
1367
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1673
1368
  }
1674
- return cids;
1675
- }
1676
- function makeFileCarHeader(result) {
1677
- const files = [];
1678
- for (const [, meta] of Object.entries(result.files || {})) {
1679
- if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1680
- files.push(meta.cid);
1369
+ _encrypt() {
1370
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1371
+ }
1372
+ };
1373
+ async function keyedCryptoFactory(url, kb, sthis) {
1374
+ const storekey = url.getParam("storekey" /* STORE_KEY */);
1375
+ if (storekey && storekey !== "insecure") {
1376
+ let rkey = await kb.getNamedKey(storekey, true);
1377
+ if (rkey.isErr()) {
1378
+ try {
1379
+ rkey = await kb.toKeyWithFingerPrint(storekey);
1380
+ } catch (e) {
1381
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
1382
+ }
1681
1383
  }
1384
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
1682
1385
  }
1683
- return { ...result, files };
1684
- }
1685
- async function prepareCarFilesFiles(encoder, roots, t) {
1686
- return [await encodeCarFile(roots, t, encoder)];
1687
- }
1688
- function makeCarHeader(meta, cars, compact = false) {
1689
- const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1690
- return { ...coreHeader, meta };
1691
- }
1692
- async function encodeCarHeader(fp) {
1693
- return await encode({
1694
- value: { fp },
1695
- hasher: import_sha22.sha256,
1696
- codec: dagCodec2
1697
- });
1386
+ return new noCrypto(url, kb.rt.crypto, sthis);
1698
1387
  }
1699
- async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
1700
- const fp = makeCarHeader(done, params.carLog, !!opts.compact);
1701
- const rootBlock = await encodeCarHeader(fp);
1702
- const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
1703
- const cids = [];
1704
- for (const car of cars) {
1705
- const { cid, bytes } = car;
1706
- await params.carStore.save({ cid, bytes });
1707
- cids.push(cid);
1388
+
1389
+ // src/blockstore/fragment-gateway.ts
1390
+ var import_cement3 = require("@adviser/cement");
1391
+ var import_base583 = require("multiformats/bases/base58");
1392
+ var import_cborg = require("cborg");
1393
+ init_types();
1394
+ init_utils();
1395
+ function getFragSize(url) {
1396
+ const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
1397
+ let ret = 0;
1398
+ if (fragSize) {
1399
+ ret = parseInt(fragSize);
1708
1400
  }
1709
- const newDbMeta = { cars: cids };
1710
- await params.WALStore.enqueue(newDbMeta, opts);
1711
- await params.metaStore.save(newDbMeta);
1712
- return { cgrp: cids, header: fp };
1401
+ if (isNaN(ret) || ret <= 0) {
1402
+ ret = 0;
1403
+ }
1404
+ return ret;
1713
1405
  }
1714
- async function prepareCarFiles(encoder, threshold, rootBlock, t) {
1715
- const carFiles = [];
1716
- threshold = threshold || 128e3 * 8;
1717
- let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1718
- clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1719
- let newsize = CBW.blockLength(toCIDBlock(rootBlock));
1720
- let cidRootBlock = rootBlock;
1721
- for (const { cid, bytes } of t.entries()) {
1722
- newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
1723
- if (newsize >= threshold) {
1724
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1725
- clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1726
- clonedt.putSync(cid, bytes);
1727
- cidRootBlock = { cid, bytes };
1728
- newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
1729
- } else {
1730
- clonedt.putSync(cid, bytes);
1406
+ async function getFrags(url, innerGW, headerSize, logger) {
1407
+ const fragSize = getFragSize(url);
1408
+ if (!fragSize) {
1409
+ const res = await innerGW.get(url);
1410
+ if (res.isErr()) {
1411
+ return [res];
1731
1412
  }
1413
+ const data = res.unwrap();
1414
+ return [
1415
+ import_cement3.Result.Ok({
1416
+ fid: new Uint8Array(0),
1417
+ ofs: 0,
1418
+ len: data.length,
1419
+ data
1420
+ })
1421
+ ];
1732
1422
  }
1733
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1734
- return carFiles;
1423
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
1424
+ if (firstRaw.isErr()) {
1425
+ return [firstRaw];
1426
+ }
1427
+ const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
1428
+ const blockSize = firstFragment.data.length;
1429
+ const ops = [Promise.resolve(import_cement3.Result.Ok(firstFragment))];
1430
+ const fidStr = import_base583.base58btc.encode(firstFragment.fid);
1431
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
1432
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
1433
+ ops.push(
1434
+ (async (furl, ofs2) => {
1435
+ const raw2 = await innerGW.get(furl);
1436
+ if (raw2.isErr()) {
1437
+ return raw2;
1438
+ }
1439
+ const fragment = (0, import_cborg.decode)(raw2.unwrap());
1440
+ if (import_base583.base58btc.encode(fragment.fid) !== fidStr) {
1441
+ return import_cement3.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
1442
+ }
1443
+ if (fragment.ofs !== ofs2) {
1444
+ return import_cement3.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
1445
+ }
1446
+ return import_cement3.Result.Ok(fragment);
1447
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
1448
+ );
1449
+ }
1450
+ return Promise.all(ops);
1735
1451
  }
1736
-
1737
- // src/blockstore/loader.ts
1738
- var import_sha23 = require("multiformats/hashes/sha2");
1739
-
1740
- // src/blockstore/task-manager.ts
1741
- init_utils();
1742
- var TaskManager = class {
1743
- constructor(sthis, callback) {
1744
- this.eventsWeHandled = /* @__PURE__ */ new Set();
1745
- this.queue = [];
1746
- this.isProcessing = false;
1747
- this.logger = ensureLogger(sthis, "TaskManager");
1748
- this.callback = callback;
1452
+ var FragmentGateway = class {
1453
+ constructor(sthis, innerGW) {
1454
+ this.fidLength = 4;
1455
+ this.headerSize = 32;
1456
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
1457
+ this.logger = this.sthis.logger;
1458
+ this.innerGW = innerGW;
1749
1459
  }
1750
- async handleEvent(cid, parents, dbMeta) {
1751
- for (const parent of parents) {
1752
- this.eventsWeHandled.add(parent.toString());
1460
+ slicer(url, body) {
1461
+ const fragSize = getFragSize(url);
1462
+ if (!fragSize) {
1463
+ return [this.innerGW.put(url, body)];
1753
1464
  }
1754
- this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
1755
- this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
1756
- void this.processQueue();
1757
- }
1758
- async processQueue() {
1759
- if (this.isProcessing) return;
1760
- this.isProcessing = true;
1761
- const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1762
- const first = filteredQueue[0];
1763
- if (!first) {
1764
- return;
1465
+ const blocksize = fragSize - this.headerSize;
1466
+ if (blocksize <= 0) {
1467
+ throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
1765
1468
  }
1766
- try {
1767
- await this.callback(first.dbMeta);
1768
- this.eventsWeHandled.add(first.cid);
1769
- this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1770
- } catch (err) {
1771
- if (first.retries++ > 3) {
1772
- this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
1773
- this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
1774
- }
1775
- await new Promise((resolve) => setTimeout(resolve, 50));
1776
- throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
1777
- } finally {
1778
- this.isProcessing = false;
1779
- if (this.queue.length > 0) {
1780
- void this.processQueue();
1469
+ const ops = [];
1470
+ const fid = this.sthis.nextId(this.fidLength);
1471
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
1472
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
1473
+ const block = (0, import_cborg.encode)({
1474
+ fid: fid.bin,
1475
+ ofs,
1476
+ len: body.length,
1477
+ data: body.slice(ofs, ofs + blocksize)
1478
+ });
1479
+ if (block.length > fragSize) {
1480
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
1781
1481
  }
1482
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
1782
1483
  }
1484
+ return ops;
1783
1485
  }
1784
- };
1785
-
1786
- // src/blockstore/loader.ts
1787
- function carLogIncludesGroup(list, cids) {
1788
- return list.some((arr) => {
1789
- return arr.toString() === cids.toString();
1790
- });
1791
- }
1792
- function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
1793
- const byString = /* @__PURE__ */ new Map();
1794
- for (const cid of list) {
1795
- if (remove.has(cid.toString())) continue;
1796
- byString.set(cid.toString(), cid);
1797
- }
1798
- return [...byString.values()];
1799
- }
1800
- var Loader = class {
1801
- constructor(name, ebOpts, sthis) {
1802
- this.commitQueue = new CommitQueue();
1803
- this.isCompacting = false;
1804
- this.carReaders = /* @__PURE__ */ new Map();
1805
- this.seenCompacted = /* @__PURE__ */ new Set();
1806
- this.processedCars = /* @__PURE__ */ new Set();
1807
- this.carLog = [];
1808
- this.getBlockCache = /* @__PURE__ */ new Map();
1809
- this.seenMeta = /* @__PURE__ */ new Set();
1810
- this.writeLimit = (0, import_p_limit.default)(1);
1811
- this.onceReady = new import_cement7.ResolveOnce();
1812
- this.name = name;
1813
- this.sthis = sthis;
1814
- this.ebOpts = defaultedBlockstoreRuntime(
1815
- sthis,
1816
- {
1817
- ...ebOpts,
1818
- name
1819
- },
1820
- "Loader"
1821
- );
1822
- this.logger = this.ebOpts.logger;
1823
- this.taskManager = new TaskManager(sthis, async (dbMeta) => {
1824
- await this.handleDbMetasFromStore([dbMeta]);
1825
- });
1826
- }
1827
- // readonly id = uuidv4();
1828
- async keyBag() {
1829
- return getKeyBag(this.sthis, this.ebOpts.keyBag);
1486
+ buildUrl(baseUrl, key) {
1487
+ return this.innerGW.buildUrl(baseUrl, key);
1830
1488
  }
1831
- async carStore() {
1832
- return this.ebOpts.storeRuntime.makeDataStore(this);
1489
+ async destroy(iurl) {
1490
+ return this.innerGW.destroy(iurl);
1833
1491
  }
1834
- async fileStore() {
1835
- return this.ebOpts.storeRuntime.makeDataStore(this);
1492
+ async start(url) {
1493
+ this.headerSize = (0, import_cborg.encode)({
1494
+ fid: this.sthis.nextId(this.fidLength).bin,
1495
+ ofs: 1024 * 1024,
1496
+ // 32bit
1497
+ len: 16 * 1024 * 1024,
1498
+ // 32bit
1499
+ data: new Uint8Array(1024)
1500
+ }).length - 1024;
1501
+ return this.innerGW.start(url);
1836
1502
  }
1837
- async WALStore() {
1838
- return this.ebOpts.storeRuntime.makeWALStore(this);
1503
+ async close(url) {
1504
+ return this.innerGW.close(url);
1839
1505
  }
1840
- async metaStore() {
1841
- return this.ebOpts.storeRuntime.makeMetaStore(this);
1506
+ async put(url, body) {
1507
+ await Promise.all(this.slicer(url, body));
1508
+ return import_cement3.Result.Ok(void 0);
1842
1509
  }
1843
- async ready() {
1844
- return this.onceReady.once(async () => {
1845
- const metas = await (await this.metaStore()).load();
1846
- if (this.ebOpts.meta) {
1847
- await this.handleDbMetasFromStore([this.ebOpts.meta]);
1848
- } else if (metas) {
1849
- await this.handleDbMetasFromStore(metas);
1510
+ async get(url) {
1511
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1512
+ let buffer = void 0;
1513
+ for (const rfrag of rfrags) {
1514
+ if (rfrag.isErr()) {
1515
+ return import_cement3.Result.Err(rfrag.Err());
1850
1516
  }
1851
- });
1852
- }
1853
- async close() {
1854
- const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1855
- await Promise.all(toClose.map((store) => store.close()));
1856
- }
1857
- async destroy() {
1858
- const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1859
- await Promise.all(toDestroy.map((store) => store.destroy()));
1860
- }
1861
- // async snapToCar(carCid: AnyLink | string) {
1862
- // await this.ready
1863
- // if (typeof carCid === 'string') {
1864
- // carCid = CID.parse(carCid)
1865
- // }
1866
- // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
1867
- // this.carLog = [carCid, ...carHeader.cars]
1868
- // await this.getMoreReaders(carHeader.cars)
1869
- // await this._applyCarHeader(carHeader, true)
1870
- // }
1871
- async handleDbMetasFromStore(metas) {
1872
- this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
1873
- for (const meta of metas) {
1874
- await this.writeLimit(async () => {
1875
- await this.mergeDbMetaIntoClock(meta);
1876
- });
1517
+ const frag = rfrag.Ok();
1518
+ buffer = buffer || new Uint8Array(frag.len);
1519
+ buffer.set(frag.data, frag.ofs);
1877
1520
  }
1521
+ return import_cement3.Result.Ok(buffer || new Uint8Array(0));
1878
1522
  }
1879
- async mergeDbMetaIntoClock(meta) {
1880
- if (this.isCompacting) {
1881
- throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1523
+ async subscribe(url, callback) {
1524
+ if (this.innerGW.subscribe) {
1525
+ return this.innerGW.subscribe(url, callback);
1526
+ } else {
1527
+ return import_cement3.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
1882
1528
  }
1883
- if (this.seenMeta.has(meta.cars.toString())) return;
1884
- this.seenMeta.add(meta.cars.toString());
1885
- if (carLogIncludesGroup(this.carLog, meta.cars)) {
1886
- return;
1887
- }
1888
- const carHeader = await this.loadCarHeaderFromMeta(meta);
1889
- carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1890
- await this.getMoreReaders(carHeader.cars.flat());
1891
- this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1892
- await this.ebOpts.applyMeta?.(carHeader.meta);
1893
- }
1894
- // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1895
- // const { key } = meta;
1896
- // if (key) {
1897
- // await this.setKey(key);
1898
- // }
1899
- // }
1900
- async loadCarHeaderFromMeta({ cars: cids }) {
1901
- const reader = await this.loadCar(cids[0]);
1902
- return await parseCarFile(reader, this.logger);
1903
- }
1904
- // async _getKey(): Promise<string | undefined> {
1905
- // if (this.key) return this.key;
1906
- // // generate a random key
1907
- // if (!this.ebOpts.public) {
1908
- // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1909
- // }
1910
- // return this.key || undefined;
1911
- // }
1912
- async commitFiles(t, done) {
1913
- await this.ready();
1914
- const fstore = await this.fileStore();
1915
- const wstore = await this.WALStore();
1916
- return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1917
1529
  }
1918
- async loadFileCar(cid) {
1919
- return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1530
+ async delete(url) {
1531
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1532
+ for (const rfrag of rfrags) {
1533
+ if (rfrag.isErr()) {
1534
+ return import_cement3.Result.Err(rfrag.Err());
1535
+ }
1536
+ const frag = rfrag.Ok();
1537
+ let fragUrl;
1538
+ if (rfrags.length > 1) {
1539
+ const fidStr = import_base583.base58btc.encode(frag.fid);
1540
+ fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
1541
+ } else {
1542
+ fragUrl = url;
1543
+ }
1544
+ await this.innerGW.delete(fragUrl);
1545
+ }
1546
+ return import_cement3.Result.Ok(void 0);
1920
1547
  }
1921
- async commit(t, done, opts = { noLoader: false, compact: false }) {
1922
- await this.ready();
1923
- const fstore = await this.fileStore();
1924
- const params = {
1925
- encoder: (await fstore.keyedCrypto()).codec(),
1926
- carLog: this.carLog,
1927
- carStore: fstore,
1928
- WALStore: await this.WALStore(),
1929
- metaStore: await this.metaStore(),
1930
- threshold: this.ebOpts.threshold
1931
- };
1932
- return this.commitQueue.enqueue(async () => {
1933
- await this.cacheTransaction(t);
1934
- const ret = await commit(params, t, done, opts);
1935
- await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1936
- return ret.cgrp;
1548
+ };
1549
+
1550
+ // src/blockstore/meta-key-helper.ts
1551
+ var import_dag_json = require("@ipld/dag-json");
1552
+ var import_clock = require("@web3-storage/pail/clock");
1553
+ var import_multiformats = require("multiformats");
1554
+ var import_base64 = require("multiformats/bases/base64");
1555
+ var import_cement7 = require("@adviser/cement");
1556
+
1557
+ // src/runtime/key-bag.ts
1558
+ var key_bag_exports = {};
1559
+ __export(key_bag_exports, {
1560
+ KeyBag: () => KeyBag,
1561
+ defaultKeyBagOpts: () => defaultKeyBagOpts,
1562
+ getKeyBag: () => getKeyBag,
1563
+ registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
1564
+ });
1565
+ var import_cement6 = require("@adviser/cement");
1566
+ init_utils();
1567
+ var import_base584 = require("multiformats/bases/base58");
1568
+ init_types();
1569
+ var KeyBag = class {
1570
+ constructor(rt) {
1571
+ this.rt = rt;
1572
+ this._warnOnce = new import_cement6.ResolveOnce();
1573
+ this._seq = new import_cement6.ResolveSeq();
1574
+ this.logger = ensureLogger(rt.sthis, "KeyBag", {
1575
+ id: rt.id()
1937
1576
  });
1577
+ this.logger.Debug().Msg("KeyBag created");
1938
1578
  }
1939
- async updateCarLog(cids, fp, compact) {
1940
- if (compact) {
1941
- const previousCompactCid = fp.compact[fp.compact.length - 1];
1942
- fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1943
- this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1944
- await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1945
- } else {
1946
- this.carLog.unshift(cids);
1579
+ async subtleKey(key) {
1580
+ const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
1581
+ if (extractable) {
1582
+ this._warnOnce.once(
1583
+ () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
1584
+ );
1947
1585
  }
1586
+ return await this.rt.crypto.importKey(
1587
+ "raw",
1588
+ // raw or jwk
1589
+ import_base584.base58btc.decode(key),
1590
+ // hexStringToUint8Array(key), // raw data
1591
+ "AES-GCM",
1592
+ extractable,
1593
+ ["encrypt", "decrypt"]
1594
+ );
1948
1595
  }
1949
- async cacheTransaction(t) {
1950
- for await (const block of t.entries()) {
1951
- const sBlock = block.cid.toString();
1952
- if (!this.getBlockCache.has(sBlock)) {
1953
- this.getBlockCache.set(sBlock, block);
1596
+ async ensureKeyFromUrl(url, keyFactory) {
1597
+ const storeKey = url.getParam("storekey" /* STORE_KEY */);
1598
+ if (storeKey === "insecure") {
1599
+ return import_cement6.Result.Ok(url);
1600
+ }
1601
+ if (!storeKey) {
1602
+ const keyName = `@${keyFactory()}@`;
1603
+ const ret = await this.getNamedKey(keyName);
1604
+ if (ret.isErr()) {
1605
+ return ret;
1954
1606
  }
1607
+ const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
1608
+ return import_cement6.Result.Ok(urb.URI());
1955
1609
  }
1956
- }
1957
- async cacheCarReader(carCidStr, reader) {
1958
- if (this.processedCars.has(carCidStr)) return;
1959
- this.processedCars.add(carCidStr);
1960
- for await (const block of reader.blocks()) {
1961
- const sBlock = block.cid.toString();
1962
- if (!this.getBlockCache.has(sBlock)) {
1963
- this.getBlockCache.set(sBlock, block);
1610
+ if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
1611
+ const ret = await this.getNamedKey(storeKey);
1612
+ if (ret.isErr()) {
1613
+ return ret;
1964
1614
  }
1965
1615
  }
1616
+ return import_cement6.Result.Ok(url);
1966
1617
  }
1967
- async removeCidsForCompact(cid) {
1968
- const carHeader = await this.loadCarHeaderFromMeta({
1969
- cars: [cid]
1618
+ async toKeyWithFingerPrint(keyStr) {
1619
+ const material = import_base584.base58btc.decode(keyStr);
1620
+ const key = await this.subtleKey(keyStr);
1621
+ const fpr = await this.rt.crypto.digestSHA256(material);
1622
+ return import_cement6.Result.Ok({
1623
+ key,
1624
+ fingerPrint: import_base584.base58btc.encode(new Uint8Array(fpr))
1970
1625
  });
1971
- for (const cids of carHeader.compact) {
1972
- for (const cid2 of cids) {
1973
- await (await this.carStore()).remove(cid2);
1974
- }
1975
- }
1976
1626
  }
1977
- // async flushCars() {
1978
- // await this.ready
1979
- // // for each cid in car log, make a dbMeta
1980
- // for (const cid of this.carLog) {
1981
- // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1982
- // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1983
- // }
1984
- // }
1985
- async *entries(cache2 = true) {
1986
- await this.ready();
1987
- if (cache2) {
1988
- for (const [, block] of this.getBlockCache) {
1989
- yield block;
1627
+ async setNamedKey(name, key) {
1628
+ return this._seq.add(() => this._setNamedKey(name, key));
1629
+ }
1630
+ // avoid deadlock
1631
+ async _setNamedKey(name, key) {
1632
+ const item = {
1633
+ name,
1634
+ key
1635
+ };
1636
+ const bag = await this.rt.getBag();
1637
+ this.logger.Debug().Str("name", name).Msg("setNamedKey");
1638
+ await bag.set(name, item);
1639
+ return await this.toKeyWithFingerPrint(item.key);
1640
+ }
1641
+ async getNamedExtractableKey(name, failIfNotFound = false) {
1642
+ const ret = await this.getNamedKey(name, failIfNotFound);
1643
+ if (ret.isErr()) {
1644
+ return ret;
1645
+ }
1646
+ const named = ret.Ok();
1647
+ return import_cement6.Result.Ok({
1648
+ ...named,
1649
+ extract: async () => {
1650
+ const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
1651
+ return {
1652
+ key: ext,
1653
+ keyStr: import_base584.base58btc.encode(ext)
1654
+ };
1990
1655
  }
1991
- } else {
1992
- for (const [, block] of this.getBlockCache) {
1993
- yield block;
1656
+ });
1657
+ }
1658
+ async getNamedKey(name, failIfNotFound = false) {
1659
+ const id = this.rt.sthis.nextId(4).str;
1660
+ return this._seq.add(async () => {
1661
+ const bag = await this.rt.getBag();
1662
+ const named = await bag.get(name);
1663
+ if (named) {
1664
+ const fpr = await this.toKeyWithFingerPrint(named.key);
1665
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
1666
+ return fpr;
1994
1667
  }
1995
- for (const cids of this.carLog) {
1996
- for (const cid of cids) {
1997
- const reader = await this.loadCar(cid);
1998
- if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1999
- for await (const block of reader.blocks()) {
2000
- const sCid = block.cid.toString();
2001
- if (!this.getBlockCache.has(sCid)) {
2002
- yield block;
2003
- }
2004
- }
2005
- }
1668
+ if (failIfNotFound) {
1669
+ this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
1670
+ return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
2006
1671
  }
2007
- }
1672
+ const ret = await this._setNamedKey(name, import_base584.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
1673
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
1674
+ return ret;
1675
+ });
2008
1676
  }
2009
- async getBlock(cid) {
2010
- await this.ready();
2011
- const sCid = cid.toString();
2012
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2013
- const getCarCid = async (carCid) => {
2014
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2015
- const reader = await this.loadCar(carCid);
2016
- if (!reader) {
2017
- throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1677
+ };
1678
+ var keyBagProviderFactories = new Map(
1679
+ [
1680
+ {
1681
+ protocol: "file:",
1682
+ factory: async (url, sthis) => {
1683
+ const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1684
+ return new KeyBagProviderFile2(url, sthis);
2018
1685
  }
2019
- await this.cacheCarReader(carCid.toString(), reader).catch(() => {
2020
- return;
2021
- });
2022
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2023
- throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
2024
- };
2025
- const getCompactCarCids = async (carCid) => {
2026
- const reader = await this.loadCar(carCid);
2027
- if (!reader) {
2028
- throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1686
+ },
1687
+ {
1688
+ protocol: "indexdb:",
1689
+ factory: async (url, sthis) => {
1690
+ const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1691
+ return new KeyBagProviderIndexDB2(url, sthis);
2029
1692
  }
2030
- const header = await parseCarFile(reader, this.logger);
2031
- const compacts = header.compact;
2032
- let got2;
2033
- const batchSize2 = 5;
2034
- for (let i = 0; i < compacts.length; i += batchSize2) {
2035
- const promises = [];
2036
- for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
2037
- for (const cid2 of compacts[j]) {
2038
- promises.push(getCarCid(cid2));
2039
- }
2040
- }
2041
- try {
2042
- got2 = await Promise.any(promises);
2043
- } catch {
2044
- }
2045
- if (got2) break;
2046
- }
2047
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2048
- throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
2049
- };
2050
- let got;
2051
- const batchSize = 5;
2052
- for (let i = 0; i < this.carLog.length; i += batchSize) {
2053
- const batch = this.carLog.slice(i, i + batchSize);
2054
- const promises = batch.flatMap((slice) => slice.map(getCarCid));
2055
- try {
2056
- got = await Promise.any(promises);
2057
- } catch {
2058
- }
2059
- if (got) break;
2060
1693
  }
2061
- if (!got) {
2062
- try {
2063
- got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
2064
- } catch {
1694
+ ].map((i) => [i.protocol, i])
1695
+ );
1696
+ function registerKeyBagProviderFactory(item) {
1697
+ const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
1698
+ keyBagProviderFactories.set(protocol, {
1699
+ ...item,
1700
+ protocol
1701
+ });
1702
+ }
1703
+ function defaultKeyBagOpts(sthis, kbo) {
1704
+ kbo = kbo || {};
1705
+ if (kbo.keyRuntime) {
1706
+ return kbo.keyRuntime;
1707
+ }
1708
+ const logger = ensureLogger(sthis, "KeyBag");
1709
+ let url;
1710
+ if (kbo.url) {
1711
+ url = import_cement6.URI.from(kbo.url);
1712
+ logger.Debug().Url(url).Msg("from opts");
1713
+ } else {
1714
+ let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
1715
+ if ((0, import_cement6.runtimeFn)().isBrowser) {
1716
+ url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
1717
+ } else {
1718
+ if (!bagFnameOrUrl) {
1719
+ const home = sthis.env.get("HOME");
1720
+ bagFnameOrUrl = `${home}/.fireproof/keybag`;
1721
+ url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
1722
+ } else {
1723
+ url = import_cement6.URI.from(bagFnameOrUrl);
2065
1724
  }
2066
1725
  }
2067
- return got;
1726
+ logger.Debug().Url(url).Msg("from env");
2068
1727
  }
2069
- async loadCar(cid) {
2070
- if (!this.carStore) {
2071
- throw this.logger.Error().Msg("car store not initialized").AsError();
1728
+ let keyProviderFactory;
1729
+ switch (url.protocol) {
1730
+ case "file:":
1731
+ keyProviderFactory = async () => {
1732
+ const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1733
+ return new KeyBagProviderFile2(url, sthis);
1734
+ };
1735
+ break;
1736
+ case "indexdb:":
1737
+ keyProviderFactory = async () => {
1738
+ const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1739
+ return new KeyBagProviderIndexDB2(url, sthis);
1740
+ };
1741
+ break;
1742
+ case "memory:":
1743
+ keyProviderFactory = async () => {
1744
+ const { KeyBagProviderMemory: KeyBagProviderMemory2 } = await Promise.resolve().then(() => (init_key_bag_memory(), key_bag_memory_exports));
1745
+ return new KeyBagProviderMemory2(url, sthis);
1746
+ };
1747
+ break;
1748
+ default:
1749
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
1750
+ }
1751
+ if (url.hasParam("masterkey")) {
1752
+ throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
1753
+ }
1754
+ return {
1755
+ url,
1756
+ crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
1757
+ sthis,
1758
+ logger,
1759
+ keyLength: kbo.keyLength || 16,
1760
+ getBag: keyProviderFactory,
1761
+ id: () => {
1762
+ return url.toString();
2072
1763
  }
2073
- const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
2074
- return loaded;
1764
+ };
1765
+ }
1766
+ var _keyBags = new import_cement6.KeyedResolvOnce();
1767
+ async function getKeyBag(sthis, kbo = {}) {
1768
+ await sthis.start();
1769
+ const rt = defaultKeyBagOpts(sthis, kbo);
1770
+ return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
1771
+ }
1772
+
1773
+ // src/blockstore/meta-key-helper.ts
1774
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
1775
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
1776
+ if (!crdtEntries.length) {
1777
+ sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
1778
+ return [];
2075
1779
  }
2076
- async makeDecoderAndCarReader(cid, local, remote) {
2077
- const cidsString = cid.toString();
2078
- let loadedCar = void 0;
2079
- let activeStore = local;
2080
- try {
2081
- this.logger.Debug().Str("cid", cidsString).Msg("loading car");
2082
- loadedCar = await local.load(cid);
2083
- this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
2084
- } catch (e) {
2085
- if (remote) {
2086
- const remoteCar = await remote.load(cid);
2087
- if (remoteCar) {
2088
- this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
2089
- await local.save(remoteCar);
2090
- loadedCar = remoteCar;
2091
- activeStore = remote;
1780
+ if (!crdtEntries.map) {
1781
+ sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
1782
+ return [];
1783
+ }
1784
+ return Promise.all(
1785
+ crdtEntries.map(async (crdtEntry) => {
1786
+ const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
1787
+ const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
1788
+ return {
1789
+ eventCid: eventBlock.cid,
1790
+ parents: crdtEntry.parents,
1791
+ dbMeta
1792
+ };
1793
+ })
1794
+ );
1795
+ }
1796
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
1797
+ try {
1798
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
1799
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
1800
+ if (keyInfo.length) {
1801
+ const dbMeta = keyInfo[0].dbMeta;
1802
+ if (dbMeta.key) {
1803
+ const kb = await getKeyBag(sthis);
1804
+ const keyName = getStoreKeyName(uri);
1805
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
1806
+ if (res.isErr()) {
1807
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
1808
+ throw res.Err();
2092
1809
  }
2093
- } else {
2094
- this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
2095
1810
  }
1811
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
1812
+ return import_cement7.Result.Ok(dbMeta);
2096
1813
  }
2097
- if (!loadedCar) {
2098
- throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
2099
- }
2100
- const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha23.sha256, codec: (await activeStore.keyedCrypto()).codec() });
2101
- const rawReader = await import_car.CarReader.fromBytes(bytes.value);
2102
- const readerP = Promise.resolve(rawReader);
2103
- const cachedReaderP = readerP.then(async (reader) => {
2104
- await this.cacheCarReader(cidsString, reader).catch((e) => {
2105
- this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
2106
- return;
2107
- });
2108
- return reader;
2109
- });
2110
- this.carReaders.set(cidsString, cachedReaderP);
2111
- return readerP;
1814
+ sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
1815
+ return import_cement7.Result.Ok(void 0);
1816
+ } catch (error) {
1817
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
1818
+ return import_cement7.Result.Err(error);
2112
1819
  }
2113
- //What if instead it returns an Array of CarHeader
2114
- async storesLoadCar(cid, local, remote) {
2115
- const cidsString = cid.toString();
2116
- let dacr = this.carReaders.get(cidsString);
2117
- if (!dacr) {
2118
- dacr = this.makeDecoderAndCarReader(cid, local, remote);
2119
- this.carReaders.set(cidsString, dacr);
1820
+ }
1821
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
1822
+ try {
1823
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
1824
+ const keyName = getStoreKeyName(uri);
1825
+ const kb = await getKeyBag(sthis);
1826
+ const res = await kb.getNamedExtractableKey(keyName, true);
1827
+ if (res.isErr()) {
1828
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
1829
+ throw res.Err();
2120
1830
  }
2121
- return dacr;
1831
+ const keyData = await res.Ok().extract();
1832
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
1833
+ const { dbMeta, parents } = dbMetas[0];
1834
+ const parentLinks = parents.map((p) => import_multiformats.CID.parse(p));
1835
+ dbMeta.key = keyData.keyStr;
1836
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
1837
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
1838
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
1839
+ return import_cement7.Result.Ok(encoded);
1840
+ } catch (error) {
1841
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
1842
+ return import_cement7.Result.Err(error);
2122
1843
  }
2123
- async getMoreReaders(cids) {
2124
- const limit = (0, import_p_limit.default)(5);
2125
- const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
2126
- await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1844
+ }
1845
+ function getStoreKeyName(url) {
1846
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
1847
+ const idx = url.getParam("index");
1848
+ if (idx) {
1849
+ storeKeyName.push(idx);
2127
1850
  }
2128
- };
1851
+ storeKeyName.push("data");
1852
+ return `@${storeKeyName.join(":")}@`;
1853
+ }
1854
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
1855
+ const event = await import_clock.EventBlock.create(
1856
+ {
1857
+ dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
1858
+ },
1859
+ parents
1860
+ );
1861
+ return event;
1862
+ }
1863
+ async function encodeEventsWithParents(sthis, events, parents) {
1864
+ const crdtEntries = events.map((event) => {
1865
+ const base64String = import_base64.base64pad.encode(event.bytes);
1866
+ return {
1867
+ cid: event.cid.toString(),
1868
+ data: base64String,
1869
+ parents: parents.map((p) => p.toString())
1870
+ };
1871
+ });
1872
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
1873
+ }
2129
1874
 
2130
- // src/runtime/keyed-crypto.ts
2131
- var keyed_crypto_exports = {};
2132
- __export(keyed_crypto_exports, {
2133
- BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
2134
- keyedCryptoFactory: () => keyedCryptoFactory
1875
+ // src/blockstore/store.ts
1876
+ var import_p_retry = __toESM(require("p-retry"), 1);
1877
+ var import_p_map = __toESM(require("p-map"), 1);
1878
+
1879
+ // src/blockstore/loader.ts
1880
+ var import_p_limit = __toESM(require("p-limit"), 1);
1881
+ var import_car = require("@ipld/car");
1882
+ var import_cement9 = require("@adviser/cement");
1883
+
1884
+ // src/runtime/wait-pr-multiformats/block.ts
1885
+ var block_exports = {};
1886
+ __export(block_exports, {
1887
+ Block: () => Block,
1888
+ create: () => create,
1889
+ createUnsafe: () => createUnsafe,
1890
+ decode: () => decode3,
1891
+ encode: () => encode3
2135
1892
  });
1893
+ var import_multiformats2 = require("multiformats");
1894
+ var import_block = require("multiformats/block");
1895
+ var Block = import_block.Block;
1896
+ async function decode3({
1897
+ bytes,
1898
+ codec: codec3,
1899
+ hasher: hasher7
1900
+ }) {
1901
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
1902
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
1903
+ const value = await Promise.resolve(codec3.decode(bytes));
1904
+ const hash = await hasher7.digest(bytes);
1905
+ const cid = import_multiformats2.CID.create(1, codec3.code, hash);
1906
+ return new import_block.Block({ value, bytes, cid });
1907
+ }
1908
+ async function encode3({
1909
+ value,
1910
+ codec: codec3,
1911
+ hasher: hasher7
1912
+ }) {
1913
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
1914
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
1915
+ const bytes = await Promise.resolve(codec3.encode(value));
1916
+ const hash = await hasher7.digest(bytes);
1917
+ const cid = import_multiformats2.CID.create(1, codec3.code, hash);
1918
+ return new import_block.Block({ value, bytes, cid });
1919
+ }
1920
+ async function create({
1921
+ bytes,
1922
+ cid,
1923
+ hasher: hasher7,
1924
+ codec: codec3
1925
+ }) {
1926
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
1927
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
1928
+ const value = await Promise.resolve(codec3.decode(bytes));
1929
+ const hash = await hasher7.digest(bytes);
1930
+ if (!import_multiformats2.bytes.equals(cid.multihash.bytes, hash.bytes)) {
1931
+ throw new Error("CID hash does not match bytes");
1932
+ }
1933
+ return createUnsafe({
1934
+ bytes,
1935
+ cid,
1936
+ value,
1937
+ codec: codec3
1938
+ });
1939
+ }
1940
+ async function createUnsafe({
1941
+ bytes,
1942
+ cid,
1943
+ value: maybeValue,
1944
+ codec: codec3
1945
+ }) {
1946
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
1947
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
1948
+ return new Block({
1949
+ cid,
1950
+ bytes,
1951
+ value
1952
+ });
1953
+ }
1954
+
1955
+ // src/blockstore/loader-helpers.ts
1956
+ var import_sha22 = require("multiformats/hashes/sha2");
1957
+ var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
1958
+ async function parseCarFile(reader, logger) {
1959
+ const roots = await reader.getRoots();
1960
+ const header = await reader.get(roots[0]);
1961
+ if (!header) throw logger.Error().Msg("missing header block").AsError();
1962
+ const dec = await decode3({ bytes: header.bytes, hasher: import_sha22.sha256, codec: dagCodec });
1963
+ const fpvalue = dec.value;
1964
+ if (fpvalue && !fpvalue.fp) {
1965
+ throw logger.Error().Msg("missing fp").AsError();
1966
+ }
1967
+ return fpvalue.fp;
1968
+ }
1969
+
1970
+ // src/blockstore/transaction.ts
1971
+ var import_block3 = require("@web3-storage/pail/block");
1972
+ init_types();
1973
+ var import_cement8 = require("@adviser/cement");
2136
1974
  init_utils();
2137
- var import_base583 = require("multiformats/bases/base58");
2138
- var import_sha24 = require("multiformats/hashes/sha2");
2139
- var CBOR = __toESM(require("cborg"), 1);
2140
- var generateIV = {
2141
- random: {
1975
+ var CarTransaction = class extends import_block3.MemoryBlockstore {
1976
+ constructor(parent, opts = { add: true, noLoader: false }) {
1977
+ super();
1978
+ if (opts.add) {
1979
+ parent.transactions.add(this);
1980
+ }
1981
+ this.parent = parent;
1982
+ }
1983
+ async get(cid) {
1984
+ return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
1985
+ }
1986
+ async superGet(cid) {
1987
+ return super.get(cid);
1988
+ }
1989
+ };
1990
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1991
+ const logger = ensureLogger(sthis, component, ctx);
1992
+ return {
2142
1993
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
2143
- calc: async (ko, crypto, data) => {
2144
- return crypto.randomBytes(ko.ivLength);
1994
+ applyMeta: (meta, snap) => {
1995
+ return Promise.resolve();
2145
1996
  },
2146
1997
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
2147
- verify: async (ko, crypto, iv, data) => {
2148
- return true;
1998
+ compact: async (blocks) => {
1999
+ return {};
2000
+ },
2001
+ autoCompact: 100,
2002
+ public: false,
2003
+ // name: undefined,
2004
+ threshold: 1e3 * 1e3,
2005
+ ...opts,
2006
+ logger,
2007
+ keyBag: opts.keyBag || {},
2008
+ crypto: (0, import_cement8.toCryptoRuntime)(opts.crypto),
2009
+ storeUrls: opts.storeUrls,
2010
+ // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
2011
+ // store,
2012
+ storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
2013
+ };
2014
+ }
2015
+ function blockstoreFactory(sthis, opts) {
2016
+ return new EncryptedBlockstore(sthis, opts);
2017
+ }
2018
+ var BaseBlockstore = class {
2019
+ constructor(ebOpts) {
2020
+ this.transactions = /* @__PURE__ */ new Set();
2021
+ this.sthis = ensureSuperThis(ebOpts);
2022
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
2023
+ this.sthis = ensureSuperThis(ebOpts);
2024
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
2025
+ this.logger = this.ebOpts.logger;
2026
+ }
2027
+ // readonly name?: string;
2028
+ // ready: Promise<void>;
2029
+ ready() {
2030
+ return Promise.resolve();
2031
+ }
2032
+ async close() {
2033
+ }
2034
+ async destroy() {
2035
+ }
2036
+ async compact() {
2037
+ }
2038
+ async get(cid) {
2039
+ if (!cid) throw this.logger.Error().Msg("required cid").AsError();
2040
+ for (const f of this.transactions) {
2041
+ const v = await f.superGet(cid);
2042
+ if (v) return v;
2149
2043
  }
2150
- },
2151
- hash: {
2152
- calc: async (ko, crypto, data) => {
2153
- const hash = await import_sha24.sha256.digest(data);
2154
- const hashBytes = new Uint8Array(hash.bytes);
2155
- const hashArray = new Uint8Array(ko.ivLength);
2156
- for (let i = 0; i < hashBytes.length; i++) {
2157
- hashArray[i % ko.ivLength] ^= hashBytes[i];
2044
+ }
2045
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2046
+ async put(cid, block) {
2047
+ throw this.logger.Error().Msg("use a transaction to put").AsError();
2048
+ }
2049
+ // TransactionMeta
2050
+ async transaction(fn, _opts) {
2051
+ this.logger.Debug().Msg("enter transaction");
2052
+ const t = new CarTransaction(this, _opts);
2053
+ this.logger.Debug().Msg("post CarTransaction");
2054
+ const done = await fn(t);
2055
+ this.logger.Debug().Msg("post fn");
2056
+ this.lastTxMeta = done;
2057
+ return { t, meta: done };
2058
+ }
2059
+ openTransaction(opts = { add: true, noLoader: false }) {
2060
+ return new CarTransaction(this, opts);
2061
+ }
2062
+ async commitTransaction(t, done, opts) {
2063
+ if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
2064
+ const cars = await this.loader?.commit(t, done, opts);
2065
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
2066
+ setTimeout(() => void this.compact(), 10);
2067
+ }
2068
+ if (cars) {
2069
+ this.transactions.delete(t);
2070
+ return { meta: done, cars, t };
2071
+ }
2072
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
2073
+ }
2074
+ async *entries() {
2075
+ const seen = /* @__PURE__ */ new Set();
2076
+ for (const t of this.transactions) {
2077
+ for await (const blk of t.entries()) {
2078
+ if (seen.has(blk.cid.toString())) continue;
2079
+ seen.add(blk.cid.toString());
2080
+ yield blk;
2158
2081
  }
2159
- return hashArray;
2160
- },
2161
- verify: async function(ko, crypto, iv, data) {
2162
- return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
2163
2082
  }
2164
2083
  }
2165
2084
  };
2166
- function getGenerateIVFn(url, opts) {
2167
- const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
2168
- return generateIV[ivhash] || generateIV["hash"];
2169
- }
2170
- var BlockIvKeyIdCodec = class {
2171
- constructor(ko, iv, opts) {
2172
- this.code = 3147065;
2173
- this.name = "Fireproof@encrypted-block:aes-gcm";
2174
- this.ko = ko;
2175
- this.iv = iv;
2176
- this.opts = opts || {};
2177
- }
2178
- async encode(data) {
2179
- const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
2180
- const { iv } = this.ko.algo(calcIv);
2181
- const fprt = await this.ko.fingerPrint();
2182
- const keyId = import_base583.base58btc.decode(fprt);
2183
- this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
2184
- return CBOR.encode({
2185
- iv,
2186
- keyId,
2187
- data: await this.ko._encrypt({ iv, bytes: data })
2085
+ var EncryptedBlockstore = class extends BaseBlockstore {
2086
+ constructor(sthis, ebOpts) {
2087
+ super(ebOpts);
2088
+ this.compacting = false;
2089
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
2090
+ this: 1
2188
2091
  });
2092
+ this.loader = new Loader(sthis, ebOpts);
2189
2093
  }
2190
- async decode(abytes) {
2191
- let bytes;
2192
- if (abytes instanceof Uint8Array) {
2193
- bytes = abytes;
2194
- } else {
2195
- bytes = new Uint8Array(abytes);
2094
+ ready() {
2095
+ return this.loader.ready();
2096
+ }
2097
+ close() {
2098
+ return this.loader.close();
2099
+ }
2100
+ destroy() {
2101
+ return this.loader.destroy();
2102
+ }
2103
+ async get(cid) {
2104
+ const got = await super.get(cid);
2105
+ if (got) return got;
2106
+ if (!this.loader) {
2107
+ return;
2196
2108
  }
2197
- const { iv, keyId, data } = CBOR.decode(bytes);
2198
- const fprt = await this.ko.fingerPrint();
2199
- this.ko.logger.Debug().Str("fp", import_base583.base58btc.encode(keyId)).Msg("decode");
2200
- if (import_base583.base58btc.encode(keyId) !== fprt) {
2201
- throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base583.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
2109
+ return falsyToUndef(await this.loader.getBlock(cid));
2110
+ }
2111
+ async transaction(fn, opts = { noLoader: false }) {
2112
+ this.logger.Debug().Msg("enter transaction");
2113
+ const { t, meta: done } = await super.transaction(fn);
2114
+ this.logger.Debug().Msg("post super.transaction");
2115
+ const cars = await this.loader.commit(t, done, opts);
2116
+ this.logger.Debug().Msg("post this.loader.commit");
2117
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
2118
+ setTimeout(() => void this.compact(), 10);
2202
2119
  }
2203
- const result = await this.ko._decrypt({ iv, bytes: data });
2204
- if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
2205
- throw this.ko.logger.Error().Msg("iv missmatch").AsError();
2120
+ if (cars) {
2121
+ this.transactions.delete(t);
2122
+ return { meta: done, cars, t };
2206
2123
  }
2207
- return result;
2124
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
2208
2125
  }
2209
- };
2210
- var keyedCrypto = class {
2211
- constructor(url, key, cyopt, sthis) {
2212
- this.ivLength = 12;
2213
- this.isEncrypting = true;
2214
- this.logger = ensureLogger(sthis, "keyedCrypto");
2215
- this.crypto = cyopt;
2216
- this.key = key;
2217
- this.url = url;
2126
+ async getFile(car, cid) {
2127
+ await this.ready();
2128
+ if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
2129
+ const reader = await this.loader.loadFileCar(
2130
+ car
2131
+ /*, isPublic */
2132
+ );
2133
+ const block = await reader.get(cid);
2134
+ if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
2135
+ return block.bytes;
2218
2136
  }
2219
- fingerPrint() {
2220
- return Promise.resolve(this.key.fingerPrint);
2137
+ async compact() {
2138
+ await this.ready();
2139
+ if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
2140
+ if (this.loader.carLog.length < 2) return;
2141
+ const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
2142
+ if (!compactFn || this.compacting) return;
2143
+ const blockLog = new CompactionFetcher(this);
2144
+ this.compacting = true;
2145
+ const meta = await compactFn(blockLog);
2146
+ await this.loader?.commit(blockLog.loggedBlocks, meta, {
2147
+ compact: true,
2148
+ noLoader: true
2149
+ });
2150
+ this.compacting = false;
2221
2151
  }
2222
- codec(iv, opts) {
2223
- return new BlockIvKeyIdCodec(this, iv, opts);
2152
+ async defaultCompact(blocks, logger) {
2153
+ if (!this.loader) {
2154
+ throw logger.Error().Msg("no loader").AsError();
2155
+ }
2156
+ if (!this.lastTxMeta) {
2157
+ throw logger.Error().Msg("no lastTxMeta").AsError();
2158
+ }
2159
+ for await (const blk of this.loader.entries(false)) {
2160
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
2161
+ }
2162
+ for (const t of this.transactions) {
2163
+ for await (const blk of t.entries()) {
2164
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
2165
+ }
2166
+ }
2167
+ return this.lastTxMeta;
2224
2168
  }
2225
- algo(iv) {
2226
- return {
2227
- name: "AES-GCM",
2228
- iv: iv || this.crypto.randomBytes(this.ivLength),
2229
- tagLength: 128
2230
- };
2169
+ async *entries() {
2170
+ for await (const blk of this.loader.entries()) {
2171
+ yield blk;
2172
+ }
2231
2173
  }
2232
- async _decrypt(data) {
2233
- this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
2234
- return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
2174
+ };
2175
+ var CompactionFetcher = class {
2176
+ constructor(blocks) {
2177
+ this.blockstore = blocks;
2178
+ this.loggedBlocks = new CarTransaction(blocks);
2235
2179
  }
2236
- async _encrypt(data) {
2237
- this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
2238
- const a = this.algo(data.iv);
2239
- return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
2180
+ async get(cid) {
2181
+ const block = await this.blockstore.get(cid);
2182
+ if (block) this.loggedBlocks.putSync(cid, block.bytes);
2183
+ return falsyToUndef(block);
2240
2184
  }
2241
2185
  };
2242
- var nullCodec = class {
2243
- constructor() {
2244
- this.code = 0;
2245
- this.name = "Fireproof@unencrypted-block";
2186
+
2187
+ // src/blockstore/commitor.ts
2188
+ var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
2189
+ var import_sha23 = require("multiformats/hashes/sha2");
2190
+ var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
2191
+ async function encodeCarFile(roots, t, codec3) {
2192
+ let size = 0;
2193
+ const headerSize = CBW.headerLength({ roots });
2194
+ size += headerSize;
2195
+ for (const { cid, bytes } of t.entries()) {
2196
+ size += CBW.blockLength({ cid, bytes });
2246
2197
  }
2247
- encode(data) {
2248
- return data;
2198
+ const buffer = new Uint8Array(size);
2199
+ const writer = CBW.createWriter(buffer, { headerSize });
2200
+ for (const r of roots) {
2201
+ writer.addRoot(r);
2249
2202
  }
2250
- decode(data) {
2251
- return data;
2203
+ for (const { cid, bytes } of t.entries()) {
2204
+ writer.write({ cid, bytes });
2252
2205
  }
2253
- };
2254
- var noCrypto = class {
2255
- constructor(url, cyrt, sthis) {
2256
- this.ivLength = 0;
2257
- this.code = 0;
2258
- this.name = "Fireproof@unencrypted-block";
2259
- this.isEncrypting = false;
2260
- this._fingerPrint = "noCrypto:" + Math.random();
2261
- this.logger = ensureLogger(sthis, "noCrypto");
2262
- this.crypto = cyrt;
2263
- this.url = url;
2206
+ writer.close();
2207
+ return await encode3({ value: writer.bytes, hasher: import_sha23.sha256, codec: codec3 });
2208
+ }
2209
+ async function createCarFile(encoder, cid, t) {
2210
+ return encodeCarFile([cid], t, encoder);
2211
+ }
2212
+ async function commitFiles(fileStore, walStore, t, done) {
2213
+ const { files: roots } = makeFileCarHeader(done);
2214
+ const cids = [];
2215
+ const codec3 = (await fileStore.keyedCrypto()).codec();
2216
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
2217
+ for (const car of cars) {
2218
+ const { cid, bytes } = car;
2219
+ await fileStore.save({ cid, bytes });
2220
+ await walStore.enqueueFile(
2221
+ cid
2222
+ /*, !!opts.public*/
2223
+ );
2224
+ cids.push(cid);
2264
2225
  }
2265
- fingerPrint() {
2266
- return Promise.resolve(this._fingerPrint);
2226
+ return cids;
2227
+ }
2228
+ function makeFileCarHeader(result) {
2229
+ const files = [];
2230
+ for (const [, meta] of Object.entries(result.files || {})) {
2231
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
2232
+ files.push(meta.cid);
2233
+ }
2267
2234
  }
2268
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
2269
- codec(iv) {
2270
- return new nullCodec();
2235
+ return { ...result, files };
2236
+ }
2237
+ async function prepareCarFilesFiles(encoder, roots, t) {
2238
+ return [await encodeCarFile(roots, t, encoder)];
2239
+ }
2240
+ function makeCarHeader(meta, cars, compact = false) {
2241
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
2242
+ return { ...coreHeader, meta };
2243
+ }
2244
+ async function encodeCarHeader(fp) {
2245
+ return await encode3({
2246
+ value: { fp },
2247
+ hasher: import_sha23.sha256,
2248
+ codec: dagCodec2
2249
+ });
2250
+ }
2251
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
2252
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
2253
+ const rootBlock = await encodeCarHeader(fp);
2254
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
2255
+ const cids = [];
2256
+ for (const car of cars) {
2257
+ const { cid, bytes } = car;
2258
+ await params.carStore.save({ cid, bytes });
2259
+ cids.push(cid);
2271
2260
  }
2272
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
2273
- algo(iv) {
2274
- return {
2275
- name: "noCrypto",
2276
- iv: new Uint8Array(),
2277
- tagLength: 0
2278
- };
2261
+ const newDbMeta = { cars: cids };
2262
+ await params.WALStore.enqueue(newDbMeta, opts);
2263
+ await params.metaStore.save(newDbMeta);
2264
+ return { cgrp: cids, header: fp };
2265
+ }
2266
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
2267
+ const carFiles = [];
2268
+ threshold = threshold || 128e3 * 8;
2269
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
2270
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
2271
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
2272
+ let cidRootBlock = rootBlock;
2273
+ for (const { cid, bytes } of t.entries()) {
2274
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
2275
+ if (newsize >= threshold) {
2276
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
2277
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
2278
+ clonedt.putSync(cid, bytes);
2279
+ cidRootBlock = { cid, bytes };
2280
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
2281
+ } else {
2282
+ clonedt.putSync(cid, bytes);
2283
+ }
2279
2284
  }
2280
- _decrypt() {
2281
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
2285
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
2286
+ return carFiles;
2287
+ }
2288
+
2289
+ // src/blockstore/loader.ts
2290
+ var import_sha24 = require("multiformats/hashes/sha2");
2291
+
2292
+ // src/blockstore/task-manager.ts
2293
+ init_utils();
2294
+ var TaskManager = class {
2295
+ constructor(sthis, callback) {
2296
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
2297
+ this.queue = [];
2298
+ this.isProcessing = false;
2299
+ this.logger = ensureLogger(sthis, "TaskManager");
2300
+ this.callback = callback;
2282
2301
  }
2283
- _encrypt() {
2284
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
2302
+ async handleEvent(cid, parents, dbMeta) {
2303
+ for (const parent of parents) {
2304
+ this.eventsWeHandled.add(parent.toString());
2305
+ }
2306
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
2307
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
2308
+ void this.processQueue();
2285
2309
  }
2286
- };
2287
- async function keyedCryptoFactory(url, kb, sthis) {
2288
- const storekey = url.getParam("storekey");
2289
- if (storekey && storekey !== "insecure") {
2290
- let rkey = await kb.getNamedKey(storekey, true);
2291
- if (rkey.isErr()) {
2292
- try {
2293
- rkey = await kb.toKeyWithFingerPrint(storekey);
2294
- } catch (e) {
2295
- throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
2310
+ async processQueue() {
2311
+ if (this.isProcessing) return;
2312
+ this.isProcessing = true;
2313
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
2314
+ const first = filteredQueue[0];
2315
+ if (!first) {
2316
+ return;
2317
+ }
2318
+ try {
2319
+ await this.callback(first.dbMeta);
2320
+ this.eventsWeHandled.add(first.cid);
2321
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
2322
+ } catch (err) {
2323
+ if (first.retries++ > 3) {
2324
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
2325
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
2326
+ }
2327
+ await new Promise((resolve) => setTimeout(resolve, 50));
2328
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
2329
+ } finally {
2330
+ this.isProcessing = false;
2331
+ if (this.queue.length > 0) {
2332
+ void this.processQueue();
2296
2333
  }
2297
2334
  }
2298
- return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
2299
2335
  }
2300
- return new noCrypto(url, kb.rt.crypto, sthis);
2301
- }
2336
+ };
2302
2337
 
2303
- // src/blockstore/fragment-gateway.ts
2304
- var import_cement8 = require("@adviser/cement");
2305
- var import_base584 = require("multiformats/bases/base58");
2306
- var import_cborg = require("cborg");
2307
- init_utils();
2308
- function getFragSize(url) {
2309
- const fragSize = url.getParam("fragSize");
2310
- let ret = 0;
2311
- if (fragSize) {
2312
- ret = parseInt(fragSize);
2313
- }
2314
- if (isNaN(ret) || ret <= 0) {
2315
- ret = 0;
2338
+ // src/blockstore/loader.ts
2339
+ function carLogIncludesGroup(list, cids) {
2340
+ return list.some((arr) => {
2341
+ return arr.toString() === cids.toString();
2342
+ });
2343
+ }
2344
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
2345
+ const byString = /* @__PURE__ */ new Map();
2346
+ for (const cid of list) {
2347
+ if (remove.has(cid.toString())) continue;
2348
+ byString.set(cid.toString(), cid);
2316
2349
  }
2317
- return ret;
2350
+ return [...byString.values()];
2318
2351
  }
2319
- async function getFrags(url, innerGW, headerSize, logger) {
2320
- const fragSize = getFragSize(url);
2321
- if (!fragSize) {
2322
- const res = await innerGW.get(url);
2323
- if (res.isErr()) {
2324
- return [res];
2325
- }
2326
- const data = res.unwrap();
2327
- return [
2328
- import_cement8.Result.Ok({
2329
- fid: new Uint8Array(0),
2330
- ofs: 0,
2331
- len: data.length,
2332
- data
2352
+ var Loader = class {
2353
+ constructor(sthis, ebOpts) {
2354
+ this.commitQueue = new CommitQueue();
2355
+ this.isCompacting = false;
2356
+ this.carReaders = /* @__PURE__ */ new Map();
2357
+ this.seenCompacted = /* @__PURE__ */ new Set();
2358
+ this.processedCars = /* @__PURE__ */ new Set();
2359
+ this.carLog = [];
2360
+ this.getBlockCache = /* @__PURE__ */ new Map();
2361
+ this.seenMeta = /* @__PURE__ */ new Set();
2362
+ this.writeLimit = (0, import_p_limit.default)(1);
2363
+ this._carStore = new import_cement9.ResolveOnce();
2364
+ this._fileStore = new import_cement9.ResolveOnce();
2365
+ this._WALStore = new import_cement9.ResolveOnce();
2366
+ this._metaStore = new import_cement9.ResolveOnce();
2367
+ this.onceReady = new import_cement9.ResolveOnce();
2368
+ this.sthis = sthis;
2369
+ this.ebOpts = defaultedBlockstoreRuntime(
2370
+ sthis,
2371
+ {
2372
+ ...ebOpts
2373
+ // name,
2374
+ },
2375
+ "Loader"
2376
+ );
2377
+ this.logger = this.ebOpts.logger;
2378
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
2379
+ await this.handleDbMetasFromStore([dbMeta]);
2380
+ });
2381
+ }
2382
+ async carStore() {
2383
+ return this._carStore.once(
2384
+ async () => this.ebOpts.storeRuntime.makeDataStore({
2385
+ sthis: this.sthis,
2386
+ url: this.ebOpts.storeUrls.data,
2387
+ keybag: await this.keyBag()
2333
2388
  })
2334
- ];
2389
+ );
2335
2390
  }
2336
- const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
2337
- if (firstRaw.isErr()) {
2338
- return [firstRaw];
2391
+ async fileStore() {
2392
+ return this._fileStore.once(
2393
+ async () => this.ebOpts.storeRuntime.makeDataStore({
2394
+ sthis: this.sthis,
2395
+ url: this.ebOpts.storeUrls.file,
2396
+ keybag: await this.keyBag()
2397
+ })
2398
+ );
2339
2399
  }
2340
- const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
2341
- const blockSize = firstFragment.data.length;
2342
- const ops = [Promise.resolve(import_cement8.Result.Ok(firstFragment))];
2343
- const fidStr = import_base584.base58btc.encode(firstFragment.fid);
2344
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
2345
- for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
2346
- ops.push(
2347
- (async (furl, ofs2) => {
2348
- const raw2 = await innerGW.get(furl);
2349
- if (raw2.isErr()) {
2350
- return raw2;
2351
- }
2352
- const fragment = (0, import_cborg.decode)(raw2.unwrap());
2353
- if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
2354
- return import_cement8.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
2355
- }
2356
- if (fragment.ofs !== ofs2) {
2357
- return import_cement8.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
2358
- }
2359
- return import_cement8.Result.Ok(fragment);
2360
- })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
2400
+ async WALStore() {
2401
+ return this._WALStore.once(
2402
+ async () => this.ebOpts.storeRuntime.makeWALStore({
2403
+ sthis: this.sthis,
2404
+ url: this.ebOpts.storeUrls.wal,
2405
+ keybag: await this.keyBag()
2406
+ })
2361
2407
  );
2362
2408
  }
2363
- return Promise.all(ops);
2364
- }
2365
- var FragmentGateway = class {
2366
- constructor(sthis, innerGW) {
2367
- this.fidLength = 4;
2368
- this.headerSize = 32;
2369
- this.sthis = ensureSuperLog(sthis, "FragmentGateway");
2370
- this.logger = this.sthis.logger;
2371
- this.innerGW = innerGW;
2409
+ async metaStore() {
2410
+ return this._metaStore.once(
2411
+ async () => this.ebOpts.storeRuntime.makeMetaStore({
2412
+ sthis: this.sthis,
2413
+ url: this.ebOpts.storeUrls.meta,
2414
+ keybag: await this.keyBag()
2415
+ })
2416
+ );
2372
2417
  }
2373
- slicer(url, body) {
2374
- const fragSize = getFragSize(url);
2375
- if (!fragSize) {
2376
- return [this.innerGW.put(url, body)];
2377
- }
2378
- const blocksize = fragSize - this.headerSize;
2379
- if (blocksize <= 0) {
2380
- throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
2381
- }
2382
- const ops = [];
2383
- const fid = this.sthis.nextId(this.fidLength);
2384
- const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
2385
- for (let ofs = 0; ofs < body.length; ofs += blocksize) {
2386
- const block = (0, import_cborg.encode)({
2387
- fid: fid.bin,
2388
- ofs,
2389
- len: body.length,
2390
- data: body.slice(ofs, ofs + blocksize)
2391
- });
2392
- if (block.length > fragSize) {
2393
- throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
2418
+ keyBag() {
2419
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
2420
+ }
2421
+ async ready() {
2422
+ return this.onceReady.once(async () => {
2423
+ const metas = await (await this.metaStore()).load();
2424
+ if (this.ebOpts.meta) {
2425
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
2426
+ } else if (metas) {
2427
+ await this.handleDbMetasFromStore(metas);
2394
2428
  }
2395
- ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
2429
+ });
2430
+ }
2431
+ async close() {
2432
+ await this.commitQueue.waitIdle();
2433
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
2434
+ await Promise.all(toClose.map((store) => store.close()));
2435
+ }
2436
+ async destroy() {
2437
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
2438
+ await Promise.all(toDestroy.map((store) => store.destroy()));
2439
+ }
2440
+ // async snapToCar(carCid: AnyLink | string) {
2441
+ // await this.ready
2442
+ // if (typeof carCid === 'string') {
2443
+ // carCid = CID.parse(carCid)
2444
+ // }
2445
+ // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
2446
+ // this.carLog = [carCid, ...carHeader.cars]
2447
+ // await this.getMoreReaders(carHeader.cars)
2448
+ // await this._applyCarHeader(carHeader, true)
2449
+ // }
2450
+ async handleDbMetasFromStore(metas) {
2451
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
2452
+ for (const meta of metas) {
2453
+ await this.writeLimit(async () => {
2454
+ await this.mergeDbMetaIntoClock(meta);
2455
+ });
2396
2456
  }
2397
- return ops;
2398
2457
  }
2399
- buildUrl(baseUrl, key) {
2400
- return this.innerGW.buildUrl(baseUrl, key);
2458
+ async mergeDbMetaIntoClock(meta) {
2459
+ if (this.isCompacting) {
2460
+ throw this.logger.Error().Msg("cannot merge while compacting").AsError();
2461
+ }
2462
+ if (this.seenMeta.has(meta.cars.toString())) return;
2463
+ this.seenMeta.add(meta.cars.toString());
2464
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
2465
+ return;
2466
+ }
2467
+ const carHeader = await this.loadCarHeaderFromMeta(meta);
2468
+ carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
2469
+ await this.getMoreReaders(carHeader.cars.flat());
2470
+ this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
2471
+ await this.ebOpts.applyMeta?.(carHeader.meta);
2401
2472
  }
2402
- async destroy(iurl) {
2403
- return this.innerGW.destroy(iurl);
2473
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
2474
+ // const { key } = meta;
2475
+ // if (key) {
2476
+ // await this.setKey(key);
2477
+ // }
2478
+ // }
2479
+ async loadCarHeaderFromMeta({ cars: cids }) {
2480
+ const reader = await this.loadCar(cids[0]);
2481
+ return await parseCarFile(reader, this.logger);
2404
2482
  }
2405
- async start(url) {
2406
- this.headerSize = (0, import_cborg.encode)({
2407
- fid: this.sthis.nextId(this.fidLength).bin,
2408
- ofs: 1024 * 1024,
2409
- // 32bit
2410
- len: 16 * 1024 * 1024,
2411
- // 32bit
2412
- data: new Uint8Array(1024)
2413
- }).length - 1024;
2414
- return this.innerGW.start(url);
2483
+ // async _getKey(): Promise<string | undefined> {
2484
+ // if (this.key) return this.key;
2485
+ // // generate a random key
2486
+ // if (!this.ebOpts.public) {
2487
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
2488
+ // }
2489
+ // return this.key || undefined;
2490
+ // }
2491
+ async commitFiles(t, done) {
2492
+ await this.ready();
2493
+ const fstore = await this.fileStore();
2494
+ const wstore = await this.WALStore();
2495
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
2415
2496
  }
2416
- async close(url) {
2417
- return this.innerGW.close(url);
2497
+ async loadFileCar(cid) {
2498
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
2418
2499
  }
2419
- async put(url, body) {
2420
- await Promise.all(this.slicer(url, body));
2421
- return import_cement8.Result.Ok(void 0);
2500
+ async commit(t, done, opts = { noLoader: false, compact: false }) {
2501
+ await this.ready();
2502
+ const carStore = await this.carStore();
2503
+ const params = {
2504
+ encoder: (await carStore.keyedCrypto()).codec(),
2505
+ carLog: this.carLog,
2506
+ carStore,
2507
+ WALStore: await this.WALStore(),
2508
+ metaStore: await this.metaStore(),
2509
+ threshold: this.ebOpts.threshold
2510
+ };
2511
+ return this.commitQueue.enqueue(async () => {
2512
+ await this.cacheTransaction(t);
2513
+ const ret = await commit(params, t, done, opts);
2514
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
2515
+ return ret.cgrp;
2516
+ });
2422
2517
  }
2423
- async get(url) {
2424
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
2425
- let buffer = void 0;
2426
- for (const rfrag of rfrags) {
2427
- if (rfrag.isErr()) {
2428
- return import_cement8.Result.Err(rfrag.Err());
2429
- }
2430
- const frag = rfrag.Ok();
2431
- buffer = buffer || new Uint8Array(frag.len);
2432
- buffer.set(frag.data, frag.ofs);
2518
+ async updateCarLog(cids, fp, compact) {
2519
+ if (compact) {
2520
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
2521
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
2522
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
2523
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
2524
+ } else {
2525
+ this.carLog.unshift(cids);
2433
2526
  }
2434
- return import_cement8.Result.Ok(buffer || new Uint8Array(0));
2435
2527
  }
2436
- async subscribe(url, callback) {
2437
- if (this.innerGW.subscribe) {
2438
- return this.innerGW.subscribe(url, callback);
2439
- } else {
2440
- return import_cement8.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
2528
+ async cacheTransaction(t) {
2529
+ for await (const block of t.entries()) {
2530
+ const sBlock = block.cid.toString();
2531
+ if (!this.getBlockCache.has(sBlock)) {
2532
+ this.getBlockCache.set(sBlock, block);
2533
+ }
2441
2534
  }
2442
2535
  }
2443
- async delete(url) {
2444
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
2445
- for (const rfrag of rfrags) {
2446
- if (rfrag.isErr()) {
2447
- return import_cement8.Result.Err(rfrag.Err());
2536
+ async cacheCarReader(carCidStr, reader) {
2537
+ if (this.processedCars.has(carCidStr)) return;
2538
+ this.processedCars.add(carCidStr);
2539
+ for await (const block of reader.blocks()) {
2540
+ const sBlock = block.cid.toString();
2541
+ if (!this.getBlockCache.has(sBlock)) {
2542
+ this.getBlockCache.set(sBlock, block);
2448
2543
  }
2449
- const frag = rfrag.Ok();
2450
- const fidStr = import_base584.base58btc.encode(frag.fid);
2451
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
2452
- await this.innerGW.delete(fragUrl);
2453
2544
  }
2454
- return import_cement8.Result.Ok(void 0);
2455
2545
  }
2456
- };
2457
-
2458
- // src/blockstore/meta-key-helper.ts
2459
- var import_dag_json = require("@ipld/dag-json");
2460
- var import_clock = require("@web3-storage/pail/clock");
2461
- var import_multiformats2 = require("multiformats");
2462
- var import_base64 = require("multiformats/bases/base64");
2463
- var import_cement9 = require("@adviser/cement");
2464
- async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
2465
- const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
2466
- if (!crdtEntries.length) {
2467
- sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
2468
- return [];
2546
+ async removeCidsForCompact(cid) {
2547
+ const carHeader = await this.loadCarHeaderFromMeta({
2548
+ cars: [cid]
2549
+ });
2550
+ for (const cids of carHeader.compact) {
2551
+ for (const cid2 of cids) {
2552
+ await (await this.carStore()).remove(cid2);
2553
+ }
2554
+ }
2469
2555
  }
2470
- if (!crdtEntries.map) {
2471
- sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
2472
- return [];
2556
+ // async flushCars() {
2557
+ // await this.ready
2558
+ // // for each cid in car log, make a dbMeta
2559
+ // for (const cid of this.carLog) {
2560
+ // const dbMeta = { car: cid, key: this.key || null } as DbMeta
2561
+ // await this.remoteWAL!.enqueue(dbMeta, { public: false })
2562
+ // }
2563
+ // }
2564
+ async *entries(cache2 = true) {
2565
+ await this.ready();
2566
+ if (cache2) {
2567
+ for (const [, block] of this.getBlockCache) {
2568
+ yield block;
2569
+ }
2570
+ } else {
2571
+ for (const [, block] of this.getBlockCache) {
2572
+ yield block;
2573
+ }
2574
+ for (const cids of this.carLog) {
2575
+ for (const cid of cids) {
2576
+ const reader = await this.loadCar(cid);
2577
+ if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
2578
+ for await (const block of reader.blocks()) {
2579
+ const sCid = block.cid.toString();
2580
+ if (!this.getBlockCache.has(sCid)) {
2581
+ yield block;
2582
+ }
2583
+ }
2584
+ }
2585
+ }
2586
+ }
2473
2587
  }
2474
- return Promise.all(
2475
- crdtEntries.map(async (crdtEntry) => {
2476
- const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
2477
- const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
2478
- return {
2479
- eventCid: eventBlock.cid,
2480
- parents: crdtEntry.parents,
2481
- dbMeta
2482
- };
2483
- })
2484
- );
2485
- }
2486
- async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
2487
- try {
2488
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
2489
- const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
2490
- if (keyInfo.length) {
2491
- const dbMeta = keyInfo[0].dbMeta;
2492
- if (dbMeta.key) {
2493
- const kb = await getKeyBag(sthis);
2494
- const keyName = getStoreKeyName(uri);
2495
- const res = await kb.setNamedKey(keyName, dbMeta.key);
2496
- if (res.isErr()) {
2497
- sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
2498
- throw res.Err();
2588
+ async getBlock(cid) {
2589
+ await this.ready();
2590
+ const sCid = cid.toString();
2591
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2592
+ const getCarCid = async (carCid) => {
2593
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2594
+ const reader = await this.loadCar(carCid);
2595
+ if (!reader) {
2596
+ throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
2597
+ }
2598
+ await this.cacheCarReader(carCid.toString(), reader).catch(() => {
2599
+ return;
2600
+ });
2601
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2602
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
2603
+ };
2604
+ const getCompactCarCids = async (carCid) => {
2605
+ const reader = await this.loadCar(carCid);
2606
+ if (!reader) {
2607
+ throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
2608
+ }
2609
+ const header = await parseCarFile(reader, this.logger);
2610
+ const compacts = header.compact;
2611
+ let got2;
2612
+ const batchSize2 = 5;
2613
+ for (let i = 0; i < compacts.length; i += batchSize2) {
2614
+ const promises = [];
2615
+ for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
2616
+ for (const cid2 of compacts[j]) {
2617
+ promises.push(getCarCid(cid2));
2618
+ }
2619
+ }
2620
+ try {
2621
+ got2 = await Promise.any(promises);
2622
+ } catch {
2499
2623
  }
2624
+ if (got2) break;
2500
2625
  }
2501
- sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
2502
- return import_cement9.Result.Ok(dbMeta);
2626
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
2627
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
2628
+ };
2629
+ let got;
2630
+ const batchSize = 5;
2631
+ for (let i = 0; i < this.carLog.length; i += batchSize) {
2632
+ const batch = this.carLog.slice(i, i + batchSize);
2633
+ const promises = batch.flatMap((slice) => slice.map(getCarCid));
2634
+ try {
2635
+ got = await Promise.any(promises);
2636
+ } catch {
2637
+ }
2638
+ if (got) break;
2503
2639
  }
2504
- sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
2505
- return import_cement9.Result.Ok(void 0);
2506
- } catch (error) {
2507
- sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
2508
- return import_cement9.Result.Err(error);
2640
+ if (!got) {
2641
+ try {
2642
+ got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
2643
+ } catch {
2644
+ }
2645
+ }
2646
+ return got;
2509
2647
  }
2510
- }
2511
- async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
2512
- try {
2513
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
2514
- const keyName = getStoreKeyName(uri);
2515
- const kb = await getKeyBag(sthis);
2516
- const res = await kb.getNamedExtractableKey(keyName, true);
2517
- if (res.isErr()) {
2518
- sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
2519
- throw res.Err();
2648
+ async loadCar(cid) {
2649
+ if (!this.carStore) {
2650
+ throw this.logger.Error().Msg("car store not initialized").AsError();
2520
2651
  }
2521
- const keyData = await res.Ok().extract();
2522
- const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
2523
- const { dbMeta, parents } = dbMetas[0];
2524
- const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
2525
- dbMeta.key = keyData.keyStr;
2526
- const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
2527
- const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
2528
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
2529
- return import_cement9.Result.Ok(encoded);
2530
- } catch (error) {
2531
- sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
2532
- return import_cement9.Result.Err(error);
2652
+ const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
2653
+ return loaded;
2533
2654
  }
2534
- }
2535
- function getStoreKeyName(url) {
2536
- const storeKeyName = [url.getParam("localName") || url.getParam("name")];
2537
- const idx = url.getParam("index");
2538
- if (idx) {
2539
- storeKeyName.push(idx);
2655
+ async makeDecoderAndCarReader(cid, local, remote) {
2656
+ const cidsString = cid.toString();
2657
+ let loadedCar = void 0;
2658
+ let activeStore = local;
2659
+ try {
2660
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
2661
+ loadedCar = await local.load(cid);
2662
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
2663
+ } catch (e) {
2664
+ if (remote) {
2665
+ const remoteCar = await remote.load(cid);
2666
+ if (remoteCar) {
2667
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
2668
+ await local.save(remoteCar);
2669
+ loadedCar = remoteCar;
2670
+ activeStore = remote;
2671
+ }
2672
+ } else {
2673
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
2674
+ }
2675
+ }
2676
+ if (!loadedCar) {
2677
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
2678
+ }
2679
+ const bytes = await decode3({ bytes: loadedCar.bytes, hasher: import_sha24.sha256, codec: (await activeStore.keyedCrypto()).codec() });
2680
+ const rawReader = await import_car.CarReader.fromBytes(bytes.value);
2681
+ const readerP = Promise.resolve(rawReader);
2682
+ const cachedReaderP = readerP.then(async (reader) => {
2683
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
2684
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
2685
+ return;
2686
+ });
2687
+ return reader;
2688
+ });
2689
+ this.carReaders.set(cidsString, cachedReaderP);
2690
+ return readerP;
2540
2691
  }
2541
- storeKeyName.push("data");
2542
- return `@${storeKeyName.join(":")}@`;
2543
- }
2544
- async function createDbMetaEventBlock(sthis, dbMeta, parents) {
2545
- const event = await import_clock.EventBlock.create(
2546
- {
2547
- dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
2548
- },
2549
- parents
2550
- );
2551
- return event;
2552
- }
2553
- async function encodeEventsWithParents(sthis, events, parents) {
2554
- const crdtEntries = events.map((event) => {
2555
- const base64String = import_base64.base64pad.encode(event.bytes);
2556
- return {
2557
- cid: event.cid.toString(),
2558
- data: base64String,
2559
- parents: parents.map((p) => p.toString())
2560
- };
2561
- });
2562
- return sthis.txt.encode(JSON.stringify(crdtEntries));
2563
- }
2692
+ //What if instead it returns an Array of CarHeader
2693
+ async storesLoadCar(cid, local, remote) {
2694
+ const cidsString = cid.toString();
2695
+ let dacr = this.carReaders.get(cidsString);
2696
+ if (!dacr) {
2697
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
2698
+ this.carReaders.set(cidsString, dacr);
2699
+ }
2700
+ return dacr;
2701
+ }
2702
+ async getMoreReaders(cids) {
2703
+ const limit = (0, import_p_limit.default)(5);
2704
+ const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
2705
+ await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
2706
+ }
2707
+ };
2564
2708
 
2565
2709
  // src/blockstore/store.ts
2566
- var import_p_retry = __toESM(require("p-retry"), 1);
2567
- var import_p_map = __toESM(require("p-map"), 1);
2568
2710
  function guardVersion(url) {
2569
2711
  if (!url.hasParam("version")) {
2570
2712
  return import_cement10.Result.Err(`missing version: ${url.toString()}`);
@@ -2572,16 +2714,21 @@ function guardVersion(url) {
2572
2714
  return import_cement10.Result.Ok(url);
2573
2715
  }
2574
2716
  var BaseStoreImpl = class {
2575
- constructor(name, url, opts, sthis, logger) {
2717
+ constructor(sthis, url, opts, logger) {
2576
2718
  this._onStarted = [];
2577
2719
  this._onClosed = [];
2578
- this.name = name;
2579
2720
  this._url = url;
2580
2721
  this.keybag = opts.keybag;
2722
+ this.loader = opts.loader;
2581
2723
  this.sthis = sthis;
2582
- this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
2724
+ const name = this._url.getParam("name" /* NAME */);
2725
+ if (!name) {
2726
+ throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
2727
+ }
2728
+ this.name = name;
2729
+ this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
2730
+ this.realGateway = opts.gateway;
2583
2731
  this.gateway = new FragmentGateway(this.sthis, opts.gateway);
2584
- this.loader = opts.loader;
2585
2732
  }
2586
2733
  url() {
2587
2734
  return this._url;
@@ -2596,20 +2743,20 @@ var BaseStoreImpl = class {
2596
2743
  return;
2597
2744
  }
2598
2745
  async keyedCrypto() {
2599
- return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
2746
+ return keyedCryptoFactory(this._url, this.keybag, this.sthis);
2600
2747
  }
2601
2748
  async start() {
2602
2749
  this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
2603
- this._url = this._url.build().setParam("store", this.storeType).URI();
2750
+ this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
2604
2751
  const res = await this.gateway.start(this._url);
2605
2752
  if (res.isErr()) {
2606
2753
  this.logger.Error().Result("gw-start", res).Msg("started-gateway");
2607
2754
  return res;
2608
2755
  }
2609
2756
  this._url = res.Ok();
2610
- const kb = await this.keybag();
2757
+ const kb = await this.keybag;
2611
2758
  const skRes = await kb.ensureKeyFromUrl(this._url, () => {
2612
- const idx = this._url.getParam("index");
2759
+ const idx = this._url.getParam("index" /* INDEX */);
2613
2760
  const storeKeyName = [this.name];
2614
2761
  if (idx) {
2615
2762
  storeKeyName.push(idx);
@@ -2642,8 +2789,8 @@ var BaseStoreImpl = class {
2642
2789
  };
2643
2790
  var MetaStoreImpl = class extends BaseStoreImpl {
2644
2791
  // remote: boolean;
2645
- constructor(sthis, name, url, opts) {
2646
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
2792
+ constructor(sthis, url, opts) {
2793
+ super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
2647
2794
  this.storeType = "meta";
2648
2795
  this.subscribers = /* @__PURE__ */ new Map();
2649
2796
  this.parents = [];
@@ -2714,13 +2861,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
2714
2861
  return import_cement10.Result.Ok(void 0);
2715
2862
  }
2716
2863
  async destroy() {
2864
+ this.logger.Debug().Msg("destroy");
2717
2865
  return this.gateway.destroy(this.url());
2718
2866
  }
2719
2867
  };
2720
2868
  var DataStoreImpl = class extends BaseStoreImpl {
2721
2869
  // readonly tag: string = "car-base";
2722
- constructor(sthis, name, url, opts) {
2723
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
2870
+ constructor(sthis, url, opts) {
2871
+ super(
2872
+ sthis,
2873
+ url,
2874
+ {
2875
+ ...opts
2876
+ },
2877
+ ensureLogger(sthis, "DataStoreImpl")
2878
+ );
2724
2879
  this.storeType = "data";
2725
2880
  }
2726
2881
  async load(cid) {
@@ -2761,23 +2916,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
2761
2916
  return import_cement10.Result.Ok(void 0);
2762
2917
  }
2763
2918
  destroy() {
2919
+ this.logger.Debug().Msg("destroy");
2764
2920
  return this.gateway.destroy(this.url());
2765
2921
  }
2766
2922
  };
2767
2923
  var WALStoreImpl = class extends BaseStoreImpl {
2768
- constructor(loader, url, opts) {
2769
- super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
2924
+ constructor(sthis, url, opts) {
2925
+ super(
2926
+ sthis,
2927
+ url,
2928
+ {
2929
+ ...opts
2930
+ },
2931
+ ensureLogger(sthis, "WALStoreImpl")
2932
+ );
2770
2933
  this.storeType = "wal";
2934
+ // readonly tag: string = "rwal-base";
2935
+ // readonly loader: Loadable;
2771
2936
  this._ready = new import_cement10.ResolveOnce();
2772
2937
  this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
2773
2938
  this.processing = void 0;
2774
2939
  this.processQueue = new CommitQueue();
2775
- this.loader = loader;
2776
2940
  }
2777
2941
  async ready() {
2778
2942
  return this._ready.once(async () => {
2779
2943
  const walState = await this.load().catch((e) => {
2780
- this.logger.Error().Any("error", e).Msg("error loading wal");
2944
+ this.logger.Error().Err(e).Msg("error loading wal");
2781
2945
  return void 0;
2782
2946
  });
2783
2947
  if (!walState) {
@@ -2810,7 +2974,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2810
2974
  }
2811
2975
  async process() {
2812
2976
  await this.ready();
2813
- if (!this.loader.remoteCarStore) return;
2977
+ if (!this.loader?.remoteCarStore) return;
2814
2978
  await this.processQueue.enqueue(async () => {
2815
2979
  try {
2816
2980
  await this._doProcess();
@@ -2823,6 +2987,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2823
2987
  });
2824
2988
  }
2825
2989
  async _doProcess() {
2990
+ if (!this.loader) return;
2826
2991
  if (!this.loader.remoteCarStore) return;
2827
2992
  const operations = [...this.walState.operations];
2828
2993
  const noLoaderOps = [...this.walState.noLoaderOps];
@@ -2840,6 +3005,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2840
3005
  noLoaderOps,
2841
3006
  async (dbMeta) => {
2842
3007
  await retryableUpload(async () => {
3008
+ if (!this.loader) return;
2843
3009
  for (const cid of dbMeta.cars) {
2844
3010
  const car = await (await this.loader.carStore()).load(cid);
2845
3011
  if (!car) {
@@ -2859,6 +3025,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2859
3025
  operations,
2860
3026
  async (dbMeta) => {
2861
3027
  await retryableUpload(async () => {
3028
+ if (!this.loader) return;
2862
3029
  for (const cid of dbMeta.cars) {
2863
3030
  const car = await (await this.loader.carStore()).load(cid);
2864
3031
  if (!car) {
@@ -2878,6 +3045,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2878
3045
  fileOperations,
2879
3046
  async ({ cid: fileCid, public: publicFile }) => {
2880
3047
  await retryableUpload(async () => {
3048
+ if (!this.loader) return;
2881
3049
  const fileBlock = await (await this.loader.fileStore()).load(fileCid);
2882
3050
  if (!fileBlock) {
2883
3051
  throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
@@ -2891,6 +3059,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2891
3059
  if (operations.length) {
2892
3060
  const lastOp = operations[operations.length - 1];
2893
3061
  await retryableUpload(async () => {
3062
+ if (!this.loader) return;
2894
3063
  await this.loader.remoteMetaStore?.save(lastOp);
2895
3064
  }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
2896
3065
  }
@@ -2916,6 +3085,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2916
3085
  }
2917
3086
  try {
2918
3087
  return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
3088
+ return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
2919
3089
  } catch (e) {
2920
3090
  throw this.logger.Error().Err(e).Msg("error parse").AsError();
2921
3091
  }
@@ -2942,73 +3112,98 @@ var WALStoreImpl = class extends BaseStoreImpl {
2942
3112
  return import_cement10.Result.Ok(void 0);
2943
3113
  }
2944
3114
  destroy() {
3115
+ this.logger.Debug().Msg("destroy");
2945
3116
  return this.gateway.destroy(this.url());
2946
3117
  }
2947
3118
  };
2948
3119
 
2949
3120
  // src/blockstore/store-factory.ts
3121
+ init_types();
3122
+
3123
+ // src/blockstore/register-store-protocol.ts
3124
+ var import_cement14 = require("@adviser/cement");
3125
+ init_version();
3126
+
3127
+ // src/runtime/gateways/memory/gateway.ts
3128
+ var import_cement11 = require("@adviser/cement");
3129
+ init_types();
3130
+
3131
+ // src/runtime/gateways/memory/version.ts
3132
+ var MEMORY_VERSION = "v0.19-memory";
3133
+
3134
+ // src/runtime/gateways/memory/gateway.ts
2950
3135
  init_utils();
2951
- function ensureIsIndex(url, isIndex) {
2952
- if (isIndex) {
2953
- return url.build().setParam("index", isIndex).URI();
3136
+ var MemoryGateway = class {
3137
+ constructor(memorys) {
3138
+ this.memorys = memorys;
2954
3139
  }
2955
- return url.build().delParam("index").URI();
2956
- }
2957
- function ensureName(name, url) {
2958
- if (!url.hasParam("name")) {
2959
- return url.build().setParam("name", name).URI();
3140
+ buildUrl(baseUrl, key) {
3141
+ return Promise.resolve(import_cement11.Result.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
2960
3142
  }
2961
- return url;
2962
- }
3143
+ start(baseUrl) {
3144
+ return Promise.resolve(import_cement11.Result.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
3145
+ }
3146
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
3147
+ close(baseUrl) {
3148
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3149
+ }
3150
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
3151
+ destroy(baseUrl) {
3152
+ this.memorys.clear();
3153
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3154
+ }
3155
+ put(url, body) {
3156
+ this.memorys.set(url.toString(), body);
3157
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3158
+ }
3159
+ // get could return a NotFoundError if the key is not found
3160
+ get(url) {
3161
+ const x = this.memorys.get(url.toString());
3162
+ if (x === void 0) {
3163
+ return Promise.resolve(import_cement11.Result.Err(new NotFoundError("not found")));
3164
+ }
3165
+ return Promise.resolve(import_cement11.Result.Ok(x));
3166
+ }
3167
+ delete(url) {
3168
+ this.memorys.delete(url.toString());
3169
+ return Promise.resolve(import_cement11.Result.Ok(void 0));
3170
+ }
3171
+ };
3172
+ var MemoryTestGateway = class {
3173
+ constructor(memorys) {
3174
+ this.memorys = memorys;
3175
+ }
3176
+ async get(url, key) {
3177
+ return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
3178
+ }
3179
+ };
3180
+
3181
+ // src/blockstore/register-store-protocol.ts
2963
3182
  var storeFactory = /* @__PURE__ */ new Map();
2964
- function buildURL(optURL, loader) {
2965
- const storeOpts = loader.ebOpts.store;
2966
- const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
2967
- let obuUrl;
2968
- if (obuItem && obuItem.overrideBaseURL) {
2969
- obuUrl = import_cement13.URI.from(obuItem.overrideBaseURL);
2970
- }
2971
- const ret = ensureIsIndex(
2972
- import_cement13.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2973
- storeOpts.isIndex
2974
- );
2975
- return ret;
2976
- }
2977
- var onceGateway = new import_cement13.KeyedResolvOnce();
2978
- async function getGatewayFromURL(url, sthis) {
2979
- return onceGateway.get(url.toString()).once(async () => {
2980
- const item = storeFactory.get(url.protocol);
2981
- if (item) {
2982
- const ret = {
2983
- gateway: await item.gateway(sthis),
2984
- test: await item.test(sthis)
2985
- };
2986
- const res = await ret.gateway.start(url);
2987
- if (res.isErr()) {
2988
- sthis.logger.Error().Result("start", res).Msg("start failed");
2989
- return void 0;
2990
- }
2991
- return ret;
3183
+ function getDefaultURI(sthis, protocol) {
3184
+ if (protocol) {
3185
+ if (!protocol.endsWith(":")) {
3186
+ protocol += ":";
2992
3187
  }
2993
- sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2994
- return void 0;
2995
- });
3188
+ const gfi = storeFactory.get(protocol);
3189
+ if (gfi) {
3190
+ return gfi.defaultURI(sthis);
3191
+ }
3192
+ }
3193
+ const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
3194
+ if (!found) {
3195
+ throw sthis.logger.Error().Msg("no default found").AsError();
3196
+ }
3197
+ return found.defaultURI(sthis);
2996
3198
  }
2997
3199
  function registerStoreProtocol(item) {
2998
3200
  let protocol = item.protocol;
2999
3201
  if (!protocol.endsWith(":")) {
3000
3202
  protocol += ":";
3001
3203
  }
3002
- if (storeFactory.has(protocol)) {
3003
- if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
3004
- throw new Error(`we need a logger here`);
3005
- return () => {
3006
- };
3007
- }
3008
- }
3009
- if (item.overrideBaseURL) {
3204
+ if (item.isDefault) {
3010
3205
  Array.from(storeFactory.values()).forEach((items) => {
3011
- items.overrideBaseURL = void 0;
3206
+ items.isDefault = false;
3012
3207
  });
3013
3208
  }
3014
3209
  storeFactory.set(protocol, item);
@@ -3016,134 +3211,186 @@ function registerStoreProtocol(item) {
3016
3211
  storeFactory.delete(protocol);
3017
3212
  };
3018
3213
  }
3019
- var onceDataStoreFactory = new import_cement13.KeyedResolvOnce();
3020
- async function dataStoreFactory(loader) {
3021
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
3022
- const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
3023
- return onceDataStoreFactory.get(url.toString()).once(async () => {
3024
- const gateway = await getGatewayFromURL(url, sthis);
3025
- if (!gateway) {
3026
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3027
- }
3028
- const store = new DataStoreImpl(sthis, loader.name, url, {
3029
- gateway: gateway.gateway,
3030
- keybag: () => getKeyBag(loader.sthis, {
3031
- ...loader.ebOpts.keyBag
3032
- })
3033
- });
3034
- return store;
3214
+ function getGatewayFactoryItem(protocol) {
3215
+ return storeFactory.get(protocol);
3216
+ }
3217
+ function fileGatewayFactoryItem() {
3218
+ return {
3219
+ protocol: "file:",
3220
+ isDefault: true,
3221
+ defaultURI: (sthis) => {
3222
+ return import_cement14.BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
3223
+ },
3224
+ gateway: async (sthis) => {
3225
+ const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3226
+ return new FileGateway2(sthis);
3227
+ },
3228
+ test: async (sthis) => {
3229
+ const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3230
+ return new FileTestStore2(sthis);
3231
+ }
3232
+ };
3233
+ }
3234
+ if ((0, import_cement14.runtimeFn)().isBrowser) {
3235
+ registerStoreProtocol({
3236
+ protocol: "indexdb:",
3237
+ isDefault: true,
3238
+ defaultURI: () => {
3239
+ return import_cement14.BuildURI.from("indexdb://").pathname("fp").URI();
3240
+ },
3241
+ gateway: async (logger) => {
3242
+ const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3243
+ return new IndexDBGateway2(logger);
3244
+ },
3245
+ test: async (logger) => {
3246
+ const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3247
+ return new IndexDBTestStore2(logger);
3248
+ }
3035
3249
  });
3250
+ } else {
3251
+ registerStoreProtocol(fileGatewayFactoryItem());
3036
3252
  }
3037
- var onceMetaStoreFactory = new import_cement13.KeyedResolvOnce();
3038
- async function metaStoreFactory(loader) {
3039
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
3040
- const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
3041
- return onceMetaStoreFactory.get(url.toString()).once(async () => {
3042
- sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
3043
- const gateway = await getGatewayFromURL(url, sthis);
3044
- if (!gateway) {
3045
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3046
- }
3047
- const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
3048
- gateway: gateway.gateway,
3049
- keybag: () => getKeyBag(loader.sthis, {
3050
- ...loader.ebOpts.keyBag
3051
- })
3052
- });
3053
- return store;
3253
+ var memory = /* @__PURE__ */ new Map();
3254
+ registerStoreProtocol({
3255
+ protocol: "memory:",
3256
+ isDefault: false,
3257
+ defaultURI: () => {
3258
+ return import_cement14.BuildURI.from("memory://").pathname("ram").URI();
3259
+ },
3260
+ gateway: async () => {
3261
+ return new MemoryGateway(memory);
3262
+ },
3263
+ test: async () => {
3264
+ return new MemoryTestGateway(memory);
3265
+ }
3266
+ });
3267
+
3268
+ // src/blockstore/store-factory.ts
3269
+ var onceGateway = new import_cement15.KeyedResolvOnce();
3270
+ var gatewayInstances = new import_cement15.KeyedResolvOnce();
3271
+ async function getStartedGateway(sthis, url) {
3272
+ return onceGateway.get(url.toString()).once(async () => {
3273
+ const item = getGatewayFactoryItem(url.protocol);
3274
+ if (item) {
3275
+ const ret = {
3276
+ url,
3277
+ ...await gatewayInstances.get(url.protocol).once(async () => ({
3278
+ gateway: await item.gateway(sthis),
3279
+ test: await item.test(sthis)
3280
+ }))
3281
+ };
3282
+ const res = await ret.gateway.start(url);
3283
+ if (res.isErr()) {
3284
+ return import_cement15.Result.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
3285
+ }
3286
+ ret.url = res.Ok();
3287
+ return import_cement15.Result.Ok(ret);
3288
+ }
3289
+ return import_cement15.Result.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
3054
3290
  });
3055
3291
  }
3056
- var onceRemoteWalFactory = new import_cement13.KeyedResolvOnce();
3057
- async function remoteWalFactory(loader) {
3058
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
3059
- const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
3060
- return onceRemoteWalFactory.get(url.toString()).once(async () => {
3061
- const gateway = await getGatewayFromURL(url, sthis);
3062
- if (!gateway) {
3063
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3064
- }
3065
- sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
3066
- const store = new WALStoreImpl(loader, url, {
3067
- gateway: gateway.gateway,
3068
- keybag: () => getKeyBag(loader.sthis, {
3069
- ...loader.ebOpts.keyBag
3070
- })
3071
- });
3072
- return store;
3292
+ async function dataStoreFactory(sfi) {
3293
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
3294
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
3295
+ if (rgateway.isErr()) {
3296
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
3297
+ }
3298
+ const gateway = rgateway.Ok();
3299
+ const store = new DataStoreImpl(sfi.sthis, gateway.url, {
3300
+ gateway: gateway.gateway,
3301
+ keybag: sfi.keybag
3302
+ });
3303
+ return store;
3304
+ }
3305
+ async function metaStoreFactory(sfi) {
3306
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
3307
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
3308
+ if (rgateway.isErr()) {
3309
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
3310
+ }
3311
+ const gateway = rgateway.Ok();
3312
+ const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
3313
+ gateway: gateway.gateway,
3314
+ keybag: sfi.keybag
3073
3315
  });
3316
+ return store;
3317
+ }
3318
+ async function WALStoreFactory(sfi) {
3319
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
3320
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
3321
+ if (rgateway.isErr()) {
3322
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
3323
+ }
3324
+ const gateway = rgateway.Ok();
3325
+ const store = new WALStoreImpl(sfi.sthis, gateway.url, {
3326
+ gateway: gateway.gateway,
3327
+ keybag: sfi.keybag
3328
+ });
3329
+ return store;
3074
3330
  }
3075
3331
  async function testStoreFactory(url, sthis) {
3076
- sthis = ensureSuperLog(sthis, "testStoreFactory");
3077
- const gateway = await getGatewayFromURL(url, sthis);
3078
- if (!gateway) {
3332
+ const rgateway = await getStartedGateway(sthis, url);
3333
+ if (!rgateway) {
3079
3334
  throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
3080
3335
  }
3081
- return gateway.test;
3336
+ return rgateway.Ok().test;
3082
3337
  }
3083
- async function ensureStart(store, logger) {
3338
+ async function ensureStart(store) {
3084
3339
  const ret = await store.start();
3085
3340
  if (ret.isErr()) {
3086
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
3341
+ throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
3087
3342
  }
3088
- logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
3343
+ store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
3089
3344
  return store;
3090
3345
  }
3091
- function toStoreRuntime(opts, sthis) {
3092
- const logger = ensureLogger(sthis, "toStoreRuntime", {});
3346
+ function ensureStoreEnDeFile(ende) {
3347
+ ende = ende || {};
3093
3348
  return {
3094
- makeMetaStore: async (loader) => {
3095
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
3096
- return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
3097
- },
3098
- makeDataStore: async (loader) => {
3099
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
3100
- return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
3101
- },
3102
- makeWALStore: async (loader) => {
3103
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
3104
- return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
3105
- },
3106
- encodeFile: opts.encodeFile || encodeFile,
3107
- decodeFile: opts.decodeFile || decodeFile
3349
+ encodeFile: ende.encodeFile || encodeFile,
3350
+ decodeFile: ende.decodeFile || decodeFile
3108
3351
  };
3109
3352
  }
3110
- registerStoreProtocol({
3111
- protocol: "file:",
3112
- gateway: async (sthis) => {
3113
- const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3114
- return new FileGateway2(sthis);
3115
- },
3116
- test: async (sthis) => {
3117
- const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
3118
- return new FileTestStore2(sthis);
3119
- }
3120
- });
3121
- registerStoreProtocol({
3122
- protocol: "indexdb:",
3123
- gateway: async (sthis) => {
3124
- const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3125
- return new IndexDBGateway2(sthis);
3126
- },
3127
- test: async (sthis) => {
3128
- const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
3129
- return new IndexDBTestStore2(sthis);
3130
- }
3131
- });
3353
+ function toStoreRuntime(sthis, endeOpts = {}) {
3354
+ return {
3355
+ makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
3356
+ // async (loader: Loadable) => {
3357
+ // logger
3358
+ // .Debug()
3359
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
3360
+ // .Msg("makeMetaStore");
3361
+ // return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
3362
+ // },
3363
+ makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
3364
+ // async (loader: Loadable) => {
3365
+ // logger
3366
+ // .Debug()
3367
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
3368
+ // .Msg("makeDataStore");
3369
+ // return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
3370
+ // },
3371
+ makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
3372
+ // async (loader: Loadable) => {
3373
+ // logger
3374
+ // .Debug()
3375
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
3376
+ // .Msg("makeRemoteWAL");
3377
+ // return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
3378
+ // },
3379
+ ...ensureStoreEnDeFile(endeOpts)
3380
+ };
3381
+ }
3382
+
3383
+ // src/blockstore/connection-base.ts
3384
+ init_types();
3132
3385
 
3133
3386
  // src/blockstore/store-remote.ts
3134
3387
  async function RemoteDataStore(sthis, name, url, opts) {
3135
- const ds = new DataStoreImpl(sthis, name, url, opts);
3388
+ const ds = new DataStoreImpl(sthis, url, opts);
3136
3389
  await ds.start();
3137
3390
  return ds;
3138
3391
  }
3139
3392
  async function RemoteMetaStore(sthis, name, url, opts) {
3140
- const ms = new MetaStoreImpl(
3141
- sthis,
3142
- name,
3143
- url,
3144
- opts
3145
- /* , true*/
3146
- );
3393
+ const ms = new MetaStoreImpl(sthis, url, opts);
3147
3394
  await ms.start();
3148
3395
  return ms;
3149
3396
  }
@@ -3168,14 +3415,17 @@ var ConnectionBase = class {
3168
3415
  if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
3169
3416
  this.loader = loader;
3170
3417
  await this.onConnect();
3171
- const metaUrl = this.url.build().defParam("store", "meta").URI();
3172
- const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
3173
- if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
3418
+ const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
3419
+ const rgateway = await getStartedGateway(loader.sthis, metaUrl);
3420
+ if (rgateway.isErr())
3421
+ throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
3422
+ const name = metaUrl.toString();
3174
3423
  const dbName = metaUrl.getParam("name");
3175
- if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
3176
- const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
3424
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
3425
+ const gateway = rgateway.Ok();
3426
+ const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
3177
3427
  gateway: gateway.gateway,
3178
- keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
3428
+ keybag: await loader.keyBag(),
3179
3429
  loader
3180
3430
  });
3181
3431
  this.loader.remoteMetaStore = remote;
@@ -3188,14 +3438,15 @@ var ConnectionBase = class {
3188
3438
  async connectStorage_X({ loader }) {
3189
3439
  if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
3190
3440
  this.loader = loader;
3191
- const dataUrl = this.url.build().defParam("store", "data").URI();
3192
- const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
3193
- if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
3441
+ const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
3442
+ const rgateway = await getStartedGateway(loader.sthis, dataUrl);
3443
+ if (rgateway.isErr())
3444
+ throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
3194
3445
  const name = dataUrl.getParam("name");
3195
3446
  if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
3196
3447
  loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
3197
- gateway: gateway.gateway,
3198
- keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
3448
+ gateway: rgateway.Ok().gateway,
3449
+ keybag: await loader.keyBag()
3199
3450
  });
3200
3451
  loader.remoteFileStore = loader.remoteCarStore;
3201
3452
  }
@@ -3235,6 +3486,13 @@ var ConnectionBase = class {
3235
3486
  };
3236
3487
 
3237
3488
  // src/crdt-helpers.ts
3489
+ var import_link = require("multiformats/link");
3490
+ var import_sha25 = require("multiformats/hashes/sha2");
3491
+ var codec = __toESM(require("@ipld/dag-cbor"), 1);
3492
+ var import_crdt = require("@web3-storage/pail/crdt");
3493
+ var import_clock2 = require("@web3-storage/pail/clock");
3494
+ var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
3495
+ init_types();
3238
3496
  function time(tag) {
3239
3497
  }
3240
3498
  function timeEnd(tag) {
@@ -3282,7 +3540,7 @@ async function writeDocContent(store, blocks, update, logger) {
3282
3540
  await processFiles(store, blocks, update.value, logger);
3283
3541
  value = { doc: update.value };
3284
3542
  }
3285
- const block = await encode({ value, hasher: import_sha25.sha256, codec });
3543
+ const block = await encode3({ value, hasher: import_sha25.sha256, codec });
3286
3544
  blocks.putSync(block.cid, block.bytes);
3287
3545
  return block.cid;
3288
3546
  }
@@ -3373,7 +3631,7 @@ function readFileset(blocks, files, isPublic = false) {
3373
3631
  async function getValueFromLink(blocks, link, logger) {
3374
3632
  const block = await blocks.get(link);
3375
3633
  if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
3376
- const { value } = await decode({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
3634
+ const { value } = await decode3({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
3377
3635
  const cvalue = {
3378
3636
  ...value,
3379
3637
  cid: link
@@ -3494,16 +3752,19 @@ async function doCompact(blockLog, head, logger) {
3494
3752
  async function getBlock(blocks, cidString) {
3495
3753
  const block = await blocks.get((0, import_link.parse)(cidString));
3496
3754
  if (!block) throw new Error(`Missing block ${cidString}`);
3497
- const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
3755
+ const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
3498
3756
  return new Block({ cid, value, bytes: block.bytes });
3499
3757
  }
3500
3758
 
3759
+ // src/indexer.ts
3760
+ init_types();
3761
+
3501
3762
  // src/indexer-helpers.ts
3502
3763
  var import_sha26 = require("multiformats/hashes/sha2");
3503
3764
  var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
3504
3765
  var import_charwise = __toESM(require("charwise"), 1);
3505
3766
  var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
3506
- var import_utils16 = require("prolly-trees/utils");
3767
+ var import_utils15 = require("prolly-trees/utils");
3507
3768
  var import_cache = require("prolly-trees/cache");
3508
3769
  var IndexTree = class {
3509
3770
  };
@@ -3511,17 +3772,17 @@ function refCompare(aRef, bRef) {
3511
3772
  if (Number.isNaN(aRef)) return -1;
3512
3773
  if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
3513
3774
  if (aRef === Infinity) return 1;
3514
- return (0, import_utils16.simpleCompare)(aRef, bRef);
3775
+ return (0, import_utils15.simpleCompare)(aRef, bRef);
3515
3776
  }
3516
3777
  function compare(a, b) {
3517
3778
  const [aKey, aRef] = a;
3518
3779
  const [bKey, bRef] = b;
3519
- const comp = (0, import_utils16.simpleCompare)(aKey, bKey);
3780
+ const comp = (0, import_utils15.simpleCompare)(aKey, bKey);
3520
3781
  if (comp !== 0) return comp;
3521
3782
  return refCompare(aRef, bRef);
3522
3783
  }
3523
- var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
3524
- var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils16.simpleCompare };
3784
+ var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
3785
+ var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils15.simpleCompare };
3525
3786
  function indexEntriesForChanges(changes, mapFn) {
3526
3787
  const indexEntries = [];
3527
3788
  changes.forEach(({ id: key, value, del }) => {
@@ -3552,7 +3813,8 @@ function makeProllyGetBlock(blocks) {
3552
3813
  return create({ cid, bytes, hasher: import_sha26.sha256, codec: codec2 });
3553
3814
  };
3554
3815
  }
3555
- async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
3816
+ async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
3817
+ logger.Debug().Msg("enter bulkIndex");
3556
3818
  if (!indexEntries.length) return inIndex;
3557
3819
  if (!inIndex.root) {
3558
3820
  if (!inIndex.cid) {
@@ -3569,18 +3831,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
3569
3831
  returnNode = node;
3570
3832
  }
3571
3833
  if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
3834
+ logger.Debug().Msg("exit !root bulkIndex");
3572
3835
  return { root: returnNode, cid: returnRootBlock.cid };
3573
3836
  } else {
3574
3837
  inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
3575
3838
  }
3576
3839
  }
3840
+ logger.Debug().Msg("pre bulk bulkIndex");
3577
3841
  const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
3578
3842
  if (root3) {
3843
+ logger.Debug().Msg("pre root put bulkIndex");
3579
3844
  for await (const block of newBlocks) {
3580
3845
  await tblocks.put(block.cid, block.bytes);
3581
3846
  }
3582
3847
  return { root: root3, cid: (await root3.block).cid };
3583
3848
  } else {
3849
+ logger.Debug().Msg("pre !root bulkIndex");
3584
3850
  return { root: void 0, cid: void 0 };
3585
3851
  }
3586
3852
  }
@@ -3621,17 +3887,17 @@ function encodeKey(key) {
3621
3887
 
3622
3888
  // src/indexer.ts
3623
3889
  init_utils();
3624
- function index(sthis, { _crdt }, name, mapFn, meta) {
3625
- if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3626
- if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
3627
- if (_crdt.indexers.has(name)) {
3628
- const idx = _crdt.indexers.get(name);
3890
+ function index(refDb, name, mapFn, meta) {
3891
+ if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3892
+ if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
3893
+ if (refDb.crdt.indexers.has(name)) {
3894
+ const idx = refDb.crdt.indexers.get(name);
3629
3895
  idx.applyMapFn(name, mapFn, meta);
3630
3896
  } else {
3631
- const idx = new Index(sthis, _crdt, name, mapFn, meta);
3632
- _crdt.indexers.set(name, idx);
3897
+ const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
3898
+ refDb.crdt.indexers.set(name, idx);
3633
3899
  }
3634
- return _crdt.indexers.get(name);
3900
+ return refDb.crdt.indexers.get(name);
3635
3901
  }
3636
3902
  var Index = class {
3637
3903
  constructor(sthis, crdt, name, mapFn, meta) {
@@ -3650,18 +3916,9 @@ var Index = class {
3650
3916
  return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
3651
3917
  });
3652
3918
  }
3653
- close() {
3654
- return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
3655
- });
3656
- }
3657
- destroy() {
3658
- return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
3659
- });
3660
- }
3661
3919
  applyMapFn(name, mapFn, meta) {
3662
3920
  if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3663
3921
  if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
3664
- this.name = name;
3665
3922
  try {
3666
3923
  if (meta) {
3667
3924
  if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
@@ -3709,9 +3966,13 @@ var Index = class {
3709
3966
  }
3710
3967
  }
3711
3968
  async query(opts = {}) {
3969
+ this.logger.Debug().Msg("enter query");
3712
3970
  await this.ready();
3971
+ this.logger.Debug().Msg("post ready query");
3713
3972
  await this._updateIndex();
3973
+ this.logger.Debug().Msg("post _updateIndex query");
3714
3974
  await this._hydrateIndex();
3975
+ this.logger.Debug().Msg("post _hydrateIndex query");
3715
3976
  if (!this.byKey.root) {
3716
3977
  return await applyQuery(this.crdt, { result: [] }, opts);
3717
3978
  }
@@ -3767,13 +4028,16 @@ var Index = class {
3767
4028
  }
3768
4029
  async _updateIndex() {
3769
4030
  await this.ready();
4031
+ this.logger.Debug().Msg("enter _updateIndex");
3770
4032
  if (this.initError) throw this.initError;
3771
4033
  if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
3772
4034
  let result, head;
3773
4035
  if (!this.indexHead || this.indexHead.length === 0) {
3774
4036
  ({ result, head } = await this.crdt.allDocs());
4037
+ this.logger.Debug().Msg("enter crdt.allDocs");
3775
4038
  } else {
3776
4039
  ({ result, head } = await this.crdt.changes(this.indexHead));
4040
+ this.logger.Debug().Msg("enter crdt.changes");
3777
4041
  }
3778
4042
  if (result.length === 0) {
3779
4043
  this.indexHead = head;
@@ -3806,9 +4070,22 @@ var Index = class {
3806
4070
  if (result.length === 0) {
3807
4071
  return indexerMeta;
3808
4072
  }
4073
+ this.logger.Debug().Msg("pre this.blockstore.transaction");
3809
4074
  const { meta } = await this.blockstore.transaction(async (tblocks) => {
3810
- this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
3811
- this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
4075
+ this.byId = await bulkIndex(
4076
+ this.logger,
4077
+ tblocks,
4078
+ this.byId,
4079
+ removeIdIndexEntries.concat(byIdIndexEntries),
4080
+ byIdOpts
4081
+ );
4082
+ this.byKey = await bulkIndex(
4083
+ this.logger,
4084
+ tblocks,
4085
+ this.byKey,
4086
+ staleKeyIndexEntries.concat(indexEntries),
4087
+ byKeyOpts
4088
+ );
3812
4089
  this.indexHead = head;
3813
4090
  if (this.byId.cid && this.byKey.cid) {
3814
4091
  const idxMeta = {
@@ -3820,8 +4097,10 @@ var Index = class {
3820
4097
  };
3821
4098
  indexerMeta.indexes?.set(this.name, idxMeta);
3822
4099
  }
4100
+ this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
3823
4101
  return indexerMeta;
3824
4102
  });
4103
+ this.logger.Debug().Msg("post this.blockstore.transaction");
3825
4104
  return meta;
3826
4105
  }
3827
4106
  };
@@ -3829,7 +4108,8 @@ var Index = class {
3829
4108
  // src/crdt-clock.ts
3830
4109
  var import_clock3 = require("@web3-storage/pail/clock");
3831
4110
  var import_crdt2 = require("@web3-storage/pail/crdt");
3832
- var import_cement14 = require("@adviser/cement");
4111
+ var import_cement16 = require("@adviser/cement");
4112
+ init_types();
3833
4113
 
3834
4114
  // src/apply-head-queue.ts
3835
4115
  function applyHeadQueue(worker, logger) {
@@ -3887,7 +4167,7 @@ var CRDTClock = class {
3887
4167
  this.zoomers = /* @__PURE__ */ new Set();
3888
4168
  this.watchers = /* @__PURE__ */ new Set();
3889
4169
  this.emptyWatchers = /* @__PURE__ */ new Set();
3890
- this._ready = new import_cement14.ResolveOnce();
4170
+ this._ready = new import_cement16.ResolveOnce();
3891
4171
  this.blockstore = blockstore;
3892
4172
  this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
3893
4173
  this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
@@ -3950,21 +4230,23 @@ var CRDTClock = class {
3950
4230
  throw this.logger.Error().Msg("missing blockstore").AsError();
3951
4231
  }
3952
4232
  await validateBlocks(this.logger, newHead, this.blockstore);
3953
- const { meta } = await this.blockstore.transaction(
3954
- async (tblocks) => {
3955
- const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3956
- const result = await (0, import_crdt2.root)(tblocks, advancedHead);
3957
- for (const { cid, bytes } of [
3958
- ...result.additions
3959
- // ...result.removals
3960
- ]) {
3961
- tblocks.putSync(cid, bytes);
3962
- }
3963
- return { head: advancedHead };
3964
- },
3965
- { noLoader, add: false }
3966
- );
3967
- this.setHead(meta.head);
4233
+ if (!this.transaction) {
4234
+ this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
4235
+ }
4236
+ const tblocks = this.transaction;
4237
+ const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
4238
+ const result = await (0, import_crdt2.root)(tblocks, advancedHead);
4239
+ for (const { cid, bytes } of [
4240
+ ...result.additions
4241
+ // ...result.removals
4242
+ ]) {
4243
+ tblocks.putSync(cid, bytes);
4244
+ }
4245
+ if (!noLoader) {
4246
+ await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
4247
+ this.transaction = void 0;
4248
+ }
4249
+ this.setHead(advancedHead);
3968
4250
  }
3969
4251
  };
3970
4252
  function sortClockHead(clockHead) {
@@ -3997,15 +4279,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
3997
4279
  // src/crdt.ts
3998
4280
  init_utils();
3999
4281
  var CRDT = class {
4000
- constructor(sthis, name, opts = {}) {
4282
+ constructor(sthis, opts) {
4001
4283
  this.indexers = /* @__PURE__ */ new Map();
4002
- this.onceReady = new import_cement15.ResolveOnce();
4284
+ this.onceReady = new import_cement17.ResolveOnce();
4003
4285
  this.sthis = sthis;
4004
- this.name = name;
4005
4286
  this.logger = ensureLogger(sthis, "CRDT");
4006
4287
  this.opts = opts;
4007
4288
  this.blockstore = blockstoreFactory(sthis, {
4008
- name,
4009
4289
  applyMeta: async (meta) => {
4010
4290
  const crdtMeta = meta;
4011
4291
  if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
@@ -4015,23 +4295,27 @@ var CRDT = class {
4015
4295
  await doCompact(blocks, this.clock.head, this.logger);
4016
4296
  return { head: this.clock.head };
4017
4297
  },
4018
- autoCompact: this.opts.autoCompact || 100,
4019
- store: { ...this.opts.store, isIndex: void 0 },
4020
- public: this.opts.public,
4021
- meta: this.opts.meta,
4022
- threshold: this.opts.threshold
4298
+ // autoCompact: this.opts.autoCompact || 100,
4299
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
4300
+ storeUrls: this.opts.storeUrls.data,
4301
+ keyBag: this.opts.keyBag,
4302
+ // public: this.opts.public,
4303
+ meta: this.opts.meta
4304
+ // threshold: this.opts.threshold,
4023
4305
  });
4024
4306
  this.indexBlockstore = blockstoreFactory(sthis, {
4025
- name,
4307
+ // name: opts.name,
4026
4308
  applyMeta: async (meta) => {
4027
4309
  const idxCarMeta = meta;
4028
4310
  if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
4029
- for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
4030
- index(this.sthis, { _crdt: this }, name2, void 0, idx);
4311
+ for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
4312
+ index({ crdt: this }, name, void 0, idx);
4031
4313
  }
4032
4314
  },
4033
- store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
4034
- public: this.opts.public
4315
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
4316
+ storeUrls: this.opts.storeUrls.idx,
4317
+ keyBag: this.opts.keyBag
4318
+ // public: this.opts.public,
4035
4319
  });
4036
4320
  this.clock = new CRDTClock(this.blockstore);
4037
4321
  this.clock.onZoom(() => {
@@ -4113,52 +4397,166 @@ var CRDT = class {
4113
4397
  };
4114
4398
 
4115
4399
  // src/database.ts
4400
+ init_types();
4116
4401
  init_utils();
4117
- var Database = class {
4118
- constructor(name, opts) {
4119
- this.opts = {};
4402
+ var databases = new import_cement18.KeyedResolvOnce();
4403
+ function toSortedArray(set) {
4404
+ if (!set) return [];
4405
+ return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
4406
+ }
4407
+ function keyConfigOpts(sthis, name, opts) {
4408
+ return JSON.stringify(
4409
+ toSortedArray({
4410
+ name,
4411
+ stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
4412
+ })
4413
+ );
4414
+ }
4415
+ function isDatabase(db) {
4416
+ return db instanceof DatabaseImpl || db instanceof DatabaseShell;
4417
+ }
4418
+ function DatabaseFactory(name, opts) {
4419
+ const sthis = ensureSuperThis(opts);
4420
+ return new DatabaseShell(
4421
+ databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
4422
+ const db = new DatabaseImpl(sthis, {
4423
+ name,
4424
+ meta: opts?.meta,
4425
+ keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
4426
+ storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
4427
+ storeEnDe: {
4428
+ encodeFile,
4429
+ decodeFile,
4430
+ ...opts?.storeEnDe
4431
+ }
4432
+ });
4433
+ db.onClosed(() => {
4434
+ databases.unget(key);
4435
+ });
4436
+ return db;
4437
+ })
4438
+ );
4439
+ }
4440
+ var DatabaseShell = class {
4441
+ constructor(ref) {
4442
+ this.ref = ref;
4443
+ ref.addShell(this);
4444
+ }
4445
+ get id() {
4446
+ return this.ref.id;
4447
+ }
4448
+ get logger() {
4449
+ return this.ref.logger;
4450
+ }
4451
+ get sthis() {
4452
+ return this.ref.sthis;
4453
+ }
4454
+ get crdt() {
4455
+ return this.ref.crdt;
4456
+ }
4457
+ name() {
4458
+ return this.ref.name();
4459
+ }
4460
+ onClosed(fn) {
4461
+ return this.ref.onClosed(fn);
4462
+ }
4463
+ close() {
4464
+ return this.ref.shellClose(this);
4465
+ }
4466
+ destroy() {
4467
+ return this.ref.destroy();
4468
+ }
4469
+ ready() {
4470
+ return this.ref.ready();
4471
+ }
4472
+ get(id) {
4473
+ return this.ref.get(id);
4474
+ }
4475
+ put(doc) {
4476
+ return this.ref.put(doc);
4477
+ }
4478
+ del(id) {
4479
+ return this.ref.del(id);
4480
+ }
4481
+ changes(since, opts) {
4482
+ return this.ref.changes(since, opts);
4483
+ }
4484
+ allDocs(opts) {
4485
+ return this.ref.allDocs(opts);
4486
+ }
4487
+ allDocuments() {
4488
+ return this.ref.allDocuments();
4489
+ }
4490
+ subscribe(listener, updates) {
4491
+ return this.ref.subscribe(listener, updates);
4492
+ }
4493
+ query(field, opts) {
4494
+ return this.ref.query(field, opts);
4495
+ }
4496
+ compact() {
4497
+ return this.ref.compact();
4498
+ }
4499
+ };
4500
+ var DatabaseImpl = class {
4501
+ constructor(sthis, opts) {
4120
4502
  this._listening = false;
4121
4503
  this._listeners = /* @__PURE__ */ new Set();
4122
4504
  this._noupdate_listeners = /* @__PURE__ */ new Set();
4123
- this._ready = new import_cement16.ResolveOnce();
4124
- this.name = name;
4125
- this.opts = opts || this.opts;
4126
- this.sthis = ensureSuperThis(this.opts);
4505
+ // readonly blockstore: BaseBlockstore;
4506
+ this.shells = /* @__PURE__ */ new Set();
4507
+ this._onClosedFns = /* @__PURE__ */ new Set();
4508
+ this._ready = new import_cement18.ResolveOnce();
4509
+ this.opts = opts;
4510
+ this.sthis = sthis;
4511
+ this.id = sthis.timeOrderedNextId().str;
4127
4512
  this.logger = ensureLogger(this.sthis, "Database");
4128
- this._crdt = new CRDT(this.sthis, name, this.opts);
4129
- this.blockstore = this._crdt.blockstore;
4513
+ this.crdt = new CRDT(this.sthis, this.opts);
4130
4514
  this._writeQueue = writeQueue(async (updates) => {
4131
- return await this._crdt.bulk(updates);
4515
+ return await this.crdt.bulk(updates);
4132
4516
  });
4133
- this._crdt.clock.onTock(() => {
4517
+ this.crdt.clock.onTock(() => {
4134
4518
  this._no_update_notify();
4135
4519
  });
4136
4520
  }
4137
- static {
4138
- this.databases = /* @__PURE__ */ new Map();
4521
+ addShell(shell) {
4522
+ this.shells.add(shell);
4523
+ }
4524
+ onClosed(fn) {
4525
+ this._onClosedFns.add(fn);
4139
4526
  }
4140
4527
  async close() {
4141
- await this.ready();
4142
- await this._crdt.close();
4143
- await this.blockstore.close();
4528
+ throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
4529
+ }
4530
+ async shellClose(db) {
4531
+ if (!this.shells.has(db)) {
4532
+ throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
4533
+ }
4534
+ this.shells.delete(db);
4535
+ if (this.shells.size === 0) {
4536
+ await this.ready();
4537
+ await this.crdt.close();
4538
+ this._onClosedFns.forEach((fn) => fn());
4539
+ }
4144
4540
  }
4145
4541
  async destroy() {
4146
4542
  await this.ready();
4147
- await this._crdt.destroy();
4148
- await this.blockstore.destroy();
4543
+ await this.crdt.destroy();
4149
4544
  }
4150
4545
  async ready() {
4151
- return this._ready.once(async () => {
4546
+ const ret = await this._ready.once(async () => {
4152
4547
  await this.sthis.start();
4153
- await this._crdt.ready();
4154
- await this.blockstore.ready();
4548
+ await this.crdt.ready();
4155
4549
  });
4550
+ return ret;
4551
+ }
4552
+ name() {
4553
+ return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
4156
4554
  }
4157
4555
  async get(id) {
4158
- if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
4556
+ if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
4159
4557
  await this.ready();
4160
4558
  this.logger.Debug().Str("id", id).Msg("get");
4161
- const got = await this._crdt.get(id).catch((e) => {
4559
+ const got = await this.crdt.get(id).catch((e) => {
4162
4560
  throw new NotFoundError(`Not found: ${id} - ${e.message}`);
4163
4561
  });
4164
4562
  if (!got) throw new NotFoundError(`Not found: ${id}`);
@@ -4177,34 +4575,34 @@ var Database = class {
4177
4575
  _id: docId
4178
4576
  }
4179
4577
  });
4180
- return { id: docId, clock: result?.head, name: this.name };
4578
+ return { id: docId, clock: result?.head, name: this.name() };
4181
4579
  }
4182
4580
  async del(id) {
4183
4581
  await this.ready();
4184
4582
  this.logger.Debug().Str("id", id).Msg("del");
4185
4583
  const result = await this._writeQueue.push({ id, del: true });
4186
- return { id, clock: result?.head, name: this.name };
4584
+ return { id, clock: result?.head, name: this.name() };
4187
4585
  }
4188
4586
  async changes(since = [], opts = {}) {
4189
4587
  await this.ready();
4190
4588
  this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
4191
- const { result, head } = await this._crdt.changes(since, opts);
4589
+ const { result, head } = await this.crdt.changes(since, opts);
4192
4590
  const rows = result.map(({ id: key, value, del, clock }) => ({
4193
4591
  key,
4194
4592
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
4195
4593
  clock
4196
4594
  }));
4197
- return { rows, clock: head, name: this.name };
4595
+ return { rows, clock: head, name: this.name() };
4198
4596
  }
4199
4597
  async allDocs(opts = {}) {
4200
4598
  await this.ready();
4201
4599
  this.logger.Debug().Msg("allDocs");
4202
- const { result, head } = await this._crdt.allDocs();
4600
+ const { result, head } = await this.crdt.allDocs();
4203
4601
  const rows = result.map(({ id: key, value, del }) => ({
4204
4602
  key,
4205
4603
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
4206
4604
  }));
4207
- return { rows, clock: head, name: this.name };
4605
+ return { rows, clock: head, name: this.name() };
4208
4606
  }
4209
4607
  async allDocuments() {
4210
4608
  return this.allDocs();
@@ -4214,7 +4612,7 @@ var Database = class {
4214
4612
  if (updates) {
4215
4613
  if (!this._listening) {
4216
4614
  this._listening = true;
4217
- this._crdt.clock.onTick((updates2) => {
4615
+ this.crdt.clock.onTick((updates2) => {
4218
4616
  void this._notify(updates2);
4219
4617
  });
4220
4618
  }
@@ -4233,13 +4631,13 @@ var Database = class {
4233
4631
  async query(field, opts = {}) {
4234
4632
  await this.ready();
4235
4633
  this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
4236
- const _crdt = this._crdt;
4237
- const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
4634
+ const _crdt = this.crdt;
4635
+ const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
4238
4636
  return await idx.query(opts);
4239
4637
  }
4240
4638
  async compact() {
4241
4639
  await this.ready();
4242
- await this._crdt.compact();
4640
+ await this.crdt.compact();
4243
4641
  }
4244
4642
  async _notify(updates) {
4245
4643
  await this.ready();
@@ -4263,23 +4661,62 @@ var Database = class {
4263
4661
  }
4264
4662
  }
4265
4663
  };
4266
- function toSortedArray(set) {
4267
- if (!set) return [];
4268
- return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
4664
+ function defaultURI(sthis, curi, uri, store, ctx) {
4665
+ ctx = ctx || {};
4666
+ const ret = (curi ? import_cement18.URI.from(curi) : uri).build().setParam("store" /* STORE */, store);
4667
+ if (!ret.hasParam("name" /* NAME */)) {
4668
+ const name = sthis.pathOps.basename(ret.URI().pathname);
4669
+ if (!name) {
4670
+ throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
4671
+ }
4672
+ ret.setParam("name" /* NAME */, name);
4673
+ }
4674
+ if (ctx.idx) {
4675
+ ret.defParam("index" /* INDEX */, "idx");
4676
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
4677
+ } else {
4678
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
4679
+ }
4680
+ if (store === "data") {
4681
+ if (ctx.file) {
4682
+ } else {
4683
+ ret.defParam("suffix" /* SUFFIX */, ".car");
4684
+ }
4685
+ }
4686
+ return ret.URI();
4269
4687
  }
4270
- function fireproof(name, opts) {
4271
- const key = JSON.stringify(
4272
- toSortedArray({
4273
- name,
4274
- stores: toSortedArray(opts?.store?.stores)
4275
- })
4276
- );
4277
- let db = Database.databases.get(key);
4278
- if (!db) {
4279
- db = new Database(name, opts);
4280
- Database.databases.set(key, db);
4688
+ function toStoreURIRuntime(sthis, name, sopts) {
4689
+ sopts = sopts || {};
4690
+ if (!sopts.base) {
4691
+ const fp_env = sthis.env.get("FP_STORAGE_URL");
4692
+ if (fp_env) {
4693
+ sopts = { ...sopts, base: import_cement18.BuildURI.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
4694
+ } else {
4695
+ sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
4696
+ }
4697
+ }
4698
+ const bbase = import_cement18.BuildURI.from(sopts.base);
4699
+ if (name) {
4700
+ bbase.setParam("name" /* NAME */, name);
4281
4701
  }
4282
- return db;
4702
+ const base = bbase.URI();
4703
+ return {
4704
+ idx: {
4705
+ data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
4706
+ file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
4707
+ meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
4708
+ wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
4709
+ },
4710
+ data: {
4711
+ data: defaultURI(sthis, sopts.data?.data, base, "data"),
4712
+ file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
4713
+ meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
4714
+ wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
4715
+ }
4716
+ };
4717
+ }
4718
+ function fireproof(name, opts) {
4719
+ return DatabaseFactory(name, opts);
4283
4720
  }
4284
4721
  function makeName(fnString) {
4285
4722
  const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
@@ -4298,6 +4735,9 @@ function makeName(fnString) {
4298
4735
  }
4299
4736
  }
4300
4737
 
4738
+ // src/index.ts
4739
+ init_types();
4740
+
4301
4741
  // src/runtime/index.ts
4302
4742
  var runtime_exports = {};
4303
4743
  __export(runtime_exports, {
@@ -4310,7 +4750,7 @@ __export(runtime_exports, {
4310
4750
  kb: () => key_bag_exports,
4311
4751
  kc: () => keyed_crypto_exports,
4312
4752
  mf: () => wait_pr_multiformats_exports,
4313
- runtimeFn: () => import_cement17.runtimeFn,
4753
+ runtimeFn: () => import_cement19.runtimeFn,
4314
4754
  toArrayBuffer: () => toArrayBuffer
4315
4755
  });
4316
4756
  init_utils2();
@@ -4326,7 +4766,7 @@ __export(wait_pr_multiformats_exports, {
4326
4766
  var codec_interface_exports = {};
4327
4767
 
4328
4768
  // src/runtime/index.ts
4329
- var import_cement17 = require("@adviser/cement");
4769
+ var import_cement19 = require("@adviser/cement");
4330
4770
  init_version();
4331
4771
  init_version2();
4332
4772
 
@@ -4335,6 +4775,6 @@ init_utils();
4335
4775
 
4336
4776
  // src/version.ts
4337
4777
  var PACKAGE_VERSION = Object.keys({
4338
- "0.19.100": "xxxx"
4778
+ "0.19.101": "xxxx"
4339
4779
  })[0];
4340
4780
  //# sourceMappingURL=index.cjs.map