@fireproof/core 0.19.111 → 0.19.112-dev-web

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/{chunk-OFGPKRCM.js → chunk-GZANCVTS.js} +3 -1
  2. package/chunk-GZANCVTS.js.map +1 -0
  3. package/{chunk-WS3YRPIA.js → chunk-LNFBDD6E.js} +4 -4
  4. package/chunk-LNFBDD6E.js.map +1 -0
  5. package/deno.json +1 -2
  6. package/{gateway-H7UD6TNB.js → gateway@skip-esm-O655UEIP.js} +3 -3
  7. package/gateway@skip-esm-O655UEIP.js.map +1 -0
  8. package/{gateway-5FCWPX5W.js → gateway@skip-iife-OZ2V32XH.js} +5 -5
  9. package/gateway@skip-iife-OZ2V32XH.js.map +1 -0
  10. package/index.cjs +32 -30
  11. package/index.cjs.map +1 -1
  12. package/index.global.js +417 -178
  13. package/index.global.js.map +1 -1
  14. package/index.js +11 -11
  15. package/index.js.map +1 -1
  16. package/{key-bag-file-WADZBHYG.js → key-bag-file-4TYN2H7F.js} +3 -3
  17. package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-JEOAS4WM.js} +3 -3
  18. package/{mem-filesystem-YPPJV7Q2.js → mem-filesystem@skip-iife-CJI7IIKV.js} +4 -4
  19. package/mem-filesystem@skip-iife-CJI7IIKV.js.map +1 -0
  20. package/metafile-cjs.json +1 -1
  21. package/metafile-esm.json +1 -1
  22. package/metafile-iife.json +1 -1
  23. package/{node-filesystem-INX4ZTHE.js → node-filesystem@skip-iife-O74VAABQ.js} +4 -4
  24. package/node-filesystem@skip-iife-O74VAABQ.js.map +1 -0
  25. package/package.json +15 -6
  26. package/tests/blockstore/keyed-crypto.test.ts +2 -2
  27. package/tests/blockstore/store.test.ts +3 -5
  28. package/tests/fireproof/all-gateway.test.ts +11 -9
  29. package/{utils-QO2HIWGI.js → utils-L7MUZUJX.js} +3 -3
  30. package/web/bundle-not-impl-UH74NK5L.js +5 -0
  31. package/web/bundle-not-impl-UH74NK5L.js.map +1 -0
  32. package/web/chunk-2DC5ZIR4.js +7 -0
  33. package/web/chunk-2DC5ZIR4.js.map +1 -0
  34. package/web/chunk-Q5W7UNMP.js +292 -0
  35. package/web/chunk-Q5W7UNMP.js.map +1 -0
  36. package/web/chunk-S4HRSKEO.js +75 -0
  37. package/web/chunk-S4HRSKEO.js.map +1 -0
  38. package/web/gateway@skip-esm-GI5PRACF.js +165 -0
  39. package/web/gateway@skip-esm-GI5PRACF.js.map +1 -0
  40. package/web/index.cjs +4138 -0
  41. package/web/index.cjs.map +1 -0
  42. package/web/index.d.cts +1139 -0
  43. package/web/index.d.ts +1139 -0
  44. package/web/index.js +3478 -0
  45. package/web/index.js.map +1 -0
  46. package/web/key-bag-file-4SJQGORQ.js +54 -0
  47. package/web/key-bag-file-4SJQGORQ.js.map +1 -0
  48. package/web/key-bag-indexdb-GSQOUUVQ.js +50 -0
  49. package/web/key-bag-indexdb-GSQOUUVQ.js.map +1 -0
  50. package/web/metafile-cjs.json +1 -0
  51. package/web/metafile-esm.json +1 -0
  52. package/web/utils-EFZJNXH5.js +14 -0
  53. package/web/utils-EFZJNXH5.js.map +1 -0
  54. package/chunk-OFGPKRCM.js.map +0 -1
  55. package/chunk-WS3YRPIA.js.map +0 -1
  56. package/gateway-5FCWPX5W.js.map +0 -1
  57. package/gateway-H7UD6TNB.js.map +0 -1
  58. package/mem-filesystem-YPPJV7Q2.js.map +0 -1
  59. package/node-filesystem-INX4ZTHE.js.map +0 -1
  60. /package/{key-bag-file-WADZBHYG.js.map → key-bag-file-4TYN2H7F.js.map} +0 -0
  61. /package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-JEOAS4WM.js.map} +0 -0
  62. /package/{utils-QO2HIWGI.js.map → utils-L7MUZUJX.js.map} +0 -0
package/web/index.cjs ADDED
@@ -0,0 +1,4138 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __esm = (fn, res) => function __init() {
9
+ return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
10
+ };
11
+ var __commonJS = (cb, mod) => function __require() {
12
+ return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
13
+ };
14
+ var __export = (target, all) => {
15
+ for (var name in all)
16
+ __defProp(target, name, { get: all[name], enumerable: true });
17
+ };
18
+ var __copyProps = (to, from, except, desc) => {
19
+ if (from && typeof from === "object" || typeof from === "function") {
20
+ for (let key of __getOwnPropNames(from))
21
+ if (!__hasOwnProp.call(to, key) && key !== except)
22
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
23
+ }
24
+ return to;
25
+ };
26
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
27
+ // If the importer is in node compatibility mode or this is not an ESM
28
+ // file that has been converted to a CommonJS file using a Babel-
29
+ // compatible transform (i.e. "__esModule" has not been set), then set
30
+ // "default" to the CommonJS "module.exports" for node compatibility.
31
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
32
+ mod
33
+ ));
34
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
35
+
36
+ // src/utils.ts
37
+ function presetEnv() {
38
+ const penv = new Map([
39
+ // ["FP_DEBUG", "xxx"],
40
+ // ["FP_ENV", "development"],
41
+ ...Array.from(
42
+ Object.entries(
43
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
44
+ globalThis[Symbol.for("FP_PRESET_ENV")] || {}
45
+ )
46
+ )
47
+ // .map(([k, v]) => [k, v as string])
48
+ ]);
49
+ return penv;
50
+ }
51
+ function ensureSuperThis(osthis) {
52
+ const env = (0, import_cement.envFactory)({
53
+ symbol: osthis?.env?.symbol || "FP_ENV",
54
+ presetEnv: osthis?.env?.presetEnv || presetEnv()
55
+ });
56
+ return new superThis({
57
+ logger: osthis?.logger || globalLogger,
58
+ env,
59
+ crypto: osthis?.crypto || (0, import_cement.toCryptoRuntime)(),
60
+ ctx: osthis?.ctx || {},
61
+ pathOps,
62
+ txt: osthis?.txt || txtOps
63
+ });
64
+ }
65
+ function ensureSuperLog(sthis, componentName, ctx) {
66
+ return sthis.clone({
67
+ logger: ensureLogger(sthis, componentName, ctx)
68
+ });
69
+ }
70
+ function ensureLogger(sthis, componentName, ctx) {
71
+ let logger = globalLogger;
72
+ if ((0, import_cement.IsLogger)(sthis)) {
73
+ logger = sthis;
74
+ } else if (sthis && (0, import_cement.IsLogger)(sthis.logger)) {
75
+ logger = sthis.logger;
76
+ }
77
+ const cLogger = logger.With().Module(componentName);
78
+ const debug = [];
79
+ let exposeStack = false;
80
+ if (ctx) {
81
+ if ("debug" in ctx) {
82
+ if (typeof ctx.debug === "string" && ctx.debug.length > 0) {
83
+ debug.push(ctx.debug);
84
+ } else {
85
+ debug.push(componentName);
86
+ }
87
+ delete ctx.debug;
88
+ }
89
+ if ("exposeStack" in ctx) {
90
+ exposeStack = true;
91
+ delete ctx.exposeStack;
92
+ }
93
+ if ("this" in ctx) {
94
+ cLogger.Str("this", sthis.nextId(4).str);
95
+ delete ctx.this;
96
+ }
97
+ for (const [key, value] of Object.entries(ctx)) {
98
+ switch (typeof value) {
99
+ case "string":
100
+ cLogger.Str(key, value);
101
+ break;
102
+ case "number":
103
+ cLogger.Uint64(key, value);
104
+ break;
105
+ default:
106
+ if (value instanceof Date) {
107
+ cLogger.Str(key, value.toISOString());
108
+ } else if ((0, import_cement.isURL)(value)) {
109
+ cLogger.Str(key, value.toString());
110
+ } else if (typeof value === "function") {
111
+ cLogger.Ref(key, value);
112
+ } else {
113
+ cLogger.Any(key, value);
114
+ }
115
+ break;
116
+ }
117
+ }
118
+ }
119
+ registerFP_DEBUG.once(async () => {
120
+ sthis.env.onSet(
121
+ (key, value) => {
122
+ switch (key) {
123
+ case "FP_FORMAT": {
124
+ switch (value) {
125
+ case "jsonice":
126
+ logger.SetFormatter(new import_cement.JSONFormatter(logger.TxtEnDe(), 2));
127
+ break;
128
+ case "yaml":
129
+ logger.SetFormatter(new import_cement.YAMLFormatter(logger.TxtEnDe(), 2));
130
+ break;
131
+ case "json":
132
+ default:
133
+ logger.SetFormatter(new import_cement.JSONFormatter(logger.TxtEnDe()));
134
+ break;
135
+ }
136
+ break;
137
+ }
138
+ case "FP_DEBUG":
139
+ logger.SetDebug(value || []);
140
+ break;
141
+ case "FP_STACK":
142
+ logger.SetExposeStack(!!value);
143
+ break;
144
+ }
145
+ },
146
+ "FP_FORMAT",
147
+ "FP_DEBUG",
148
+ "FP_STACK"
149
+ );
150
+ }).finally(() => {
151
+ });
152
+ if (debug.length > 0) {
153
+ logger.SetDebug(debug);
154
+ }
155
+ if (exposeStack) {
156
+ logger.SetExposeStack(true);
157
+ }
158
+ const out = cLogger.Logger();
159
+ return out;
160
+ }
161
+ function getStore(url, sthis, joiner) {
162
+ const store = url.getParam("store");
163
+ switch (store) {
164
+ case "data":
165
+ case "wal":
166
+ case "meta":
167
+ break;
168
+ default:
169
+ throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
170
+ }
171
+ let name = store;
172
+ if (url.hasParam("index")) {
173
+ name = joiner(url.getParam("index") || "idx", name);
174
+ }
175
+ return { store, name };
176
+ }
177
+ function getKey(url, logger) {
178
+ const result = url.getParam("key");
179
+ if (!result) throw logger.Error().Str("url", url.toString()).Msg(`key not found`).AsError();
180
+ return result;
181
+ }
182
+ function getName(sthis, url) {
183
+ let result = url.getParam("name");
184
+ if (!result) {
185
+ result = sthis.pathOps.dirname(url.pathname);
186
+ if (result.length === 0) {
187
+ throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
188
+ }
189
+ }
190
+ return result;
191
+ }
192
+ async function exceptionWrapper(fn) {
193
+ return fn().catch((e) => import_cement.Result.Err(e));
194
+ }
195
+ function isNotFoundError(e) {
196
+ if (import_cement.Result.Is(e)) {
197
+ if (e.isOk()) return false;
198
+ e = e.Err();
199
+ }
200
+ if (e.code === "ENOENT") return true;
201
+ return false;
202
+ }
203
+ function dataDir(sthis, name, base) {
204
+ if (!base) {
205
+ if (!(0, import_cement.runtimeFn)().isBrowser) {
206
+ const home = sthis.env.get("HOME") || "./";
207
+ base = sthis.env.get("FP_STORAGE_URL") || `file://${sthis.pathOps.join(home, ".fireproof")}`;
208
+ } else {
209
+ base = sthis.env.get("FP_STORAGE_URL") || `indexdb://fp`;
210
+ }
211
+ }
212
+ return import_cement.URI.from(base.toString()).build().setParam("name", name || "").URI();
213
+ }
214
+ function UInt8ArrayEqual(a, b) {
215
+ if (a.length !== b.length) {
216
+ return false;
217
+ }
218
+ for (let i = 0; i < a.length; i++) {
219
+ if (a[i] !== b[i]) {
220
+ return false;
221
+ }
222
+ }
223
+ return true;
224
+ }
225
+ var import_cement, import_base58, globalLogger, registerFP_DEBUG, superThis, pathOpsImpl, pathOps, txtOps, NotFoundError;
226
+ var init_utils = __esm({
227
+ "src/utils.ts"() {
228
+ "use strict";
229
+ import_cement = require("@adviser/cement");
230
+ import_base58 = require("multiformats/bases/base58");
231
+ globalLogger = new import_cement.LoggerImpl();
232
+ registerFP_DEBUG = new import_cement.ResolveOnce();
233
+ superThis = class _superThis {
234
+ constructor(opts) {
235
+ this.logger = opts.logger;
236
+ this.env = opts.env;
237
+ this.crypto = opts.crypto;
238
+ this.pathOps = opts.pathOps;
239
+ this.txt = opts.txt;
240
+ this.ctx = { ...opts.ctx };
241
+ }
242
+ nextId(bytes = 6) {
243
+ const bin = this.crypto.randomBytes(bytes);
244
+ return {
245
+ str: import_base58.base58btc.encode(bin),
246
+ bin
247
+ };
248
+ }
249
+ timeOrderedNextId(now) {
250
+ now = typeof now === "number" ? now : (/* @__PURE__ */ new Date()).getTime();
251
+ const t = (281474976710656 + now).toString(16).replace(/^1/, "");
252
+ const bin = this.crypto.randomBytes(10);
253
+ bin[1] = bin[1] & 240 | (bin[1] | 8 && 11);
254
+ const hex = Array.from(bin).map((i) => i.toString(16).padStart(2, "0")).join("");
255
+ return {
256
+ str: `${t.slice(0, 8)}-${t.slice(8)}-7${hex.slice(0, 3)}-${hex.slice(3, 7)}-${hex.slice(7, 19)}`
257
+ };
258
+ }
259
+ start() {
260
+ return Promise.resolve();
261
+ }
262
+ clone(override) {
263
+ return new _superThis({
264
+ logger: override.logger || this.logger,
265
+ env: (0, import_cement.envFactory)(override.env) || this.env,
266
+ crypto: override.crypto || this.crypto,
267
+ pathOps: override.pathOps || this.pathOps,
268
+ txt: override.txt || this.txt,
269
+ ctx: { ...this.ctx, ...override.ctx }
270
+ });
271
+ }
272
+ };
273
+ pathOpsImpl = class {
274
+ join(...paths) {
275
+ return paths.map((i) => i.replace(/\/+$/, "")).join("/");
276
+ }
277
+ dirname(path) {
278
+ return path.split("/").slice(0, -1).join("/");
279
+ }
280
+ // homedir() {
281
+ // throw new Error("SysContainer:homedir is not available in seeded state");
282
+ // }
283
+ };
284
+ pathOps = new pathOpsImpl();
285
+ txtOps = {
286
+ // eslint-disable-next-line no-restricted-globals
287
+ encode: (input) => new TextEncoder().encode(input),
288
+ // eslint-disable-next-line no-restricted-globals
289
+ decode: (input) => new TextDecoder().decode(input)
290
+ };
291
+ NotFoundError = class extends Error {
292
+ constructor() {
293
+ super(...arguments);
294
+ this.code = "ENOENT";
295
+ }
296
+ };
297
+ }
298
+ });
299
+
300
+ // src/bundle-not-impl.ts
301
+ var require_bundle_not_impl = __commonJS({
302
+ "src/bundle-not-impl.ts"() {
303
+ "use strict";
304
+ var err = new Error("store-file not implemented");
305
+ console.error(err.stack);
306
+ throw err;
307
+ }
308
+ });
309
+
310
+ // src/runtime/gateways/file/utils.ts
311
+ var utils_exports = {};
312
+ __export(utils_exports, {
313
+ getFileName: () => getFileName,
314
+ getFileSystem: () => getFileSystem,
315
+ getPath: () => getPath,
316
+ toArrayBuffer: () => toArrayBuffer
317
+ });
318
+ async function getFileSystem(url) {
319
+ const name = url.getParam("fs");
320
+ let fs;
321
+ switch (name) {
322
+ case "mem":
323
+ {
324
+ const { MemFileSystem } = await Promise.resolve().then(() => __toESM(require_bundle_not_impl(), 1));
325
+ fs = new MemFileSystem();
326
+ }
327
+ break;
328
+ // case 'deno': {
329
+ // const { DenoFileSystem } = await import("./deno-filesystem.js");
330
+ // fs = new DenoFileSystem();
331
+ // break;
332
+ // }
333
+ case "node": {
334
+ const { NodeFileSystem } = await Promise.resolve().then(() => __toESM(require_bundle_not_impl(), 1));
335
+ fs = new NodeFileSystem();
336
+ break;
337
+ }
338
+ case "sys":
339
+ default: {
340
+ return getFileSystem(url.build().setParam("fs", "node").URI());
341
+ }
342
+ }
343
+ return fs.start();
344
+ }
345
+ function getPath(url, sthis) {
346
+ const basePath = url.pathname;
347
+ const name = url.getParam("name");
348
+ if (name) {
349
+ const version = url.getParam("version");
350
+ if (!version) throw sthis.logger.Error().Url(url).Msg(`version not found`).AsError();
351
+ return sthis.pathOps.join(basePath, version, name);
352
+ }
353
+ return sthis.pathOps.join(basePath);
354
+ }
355
+ function getFileName(url, sthis) {
356
+ const key = url.getParam("key");
357
+ if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
358
+ const res = getStore(url, sthis, (...a) => a.join("-"));
359
+ switch (res.store) {
360
+ case "data":
361
+ return sthis.pathOps.join(res.name, key + ".car");
362
+ case "wal":
363
+ case "meta":
364
+ return sthis.pathOps.join(res.name, key + ".json");
365
+ default:
366
+ throw sthis.logger.Error().Url(url).Msg(`unsupported store type`).AsError();
367
+ }
368
+ }
369
+ function toArrayBuffer(buffer) {
370
+ if (typeof buffer === "string") {
371
+ buffer = Buffer.from(buffer);
372
+ }
373
+ const ab = new ArrayBuffer(buffer.length);
374
+ const view = new Uint8Array(ab);
375
+ for (let i = 0; i < buffer.length; ++i) {
376
+ view[i] = buffer[i];
377
+ }
378
+ return view;
379
+ }
380
+ var init_utils2 = __esm({
381
+ "src/runtime/gateways/file/utils.ts"() {
382
+ "use strict";
383
+ init_utils();
384
+ }
385
+ });
386
+
387
+ // src/runtime/key-bag-file.ts
388
+ var key_bag_file_exports = {};
389
+ __export(key_bag_file_exports, {
390
+ KeyBagProviderFile: () => KeyBagProviderFile
391
+ });
392
+ var KeyBagProviderFile;
393
+ var init_key_bag_file = __esm({
394
+ "src/runtime/key-bag-file.ts"() {
395
+ "use strict";
396
+ init_utils();
397
+ KeyBagProviderFile = class {
398
+ async _prepare(id) {
399
+ await this.sthis.start();
400
+ let sysFS;
401
+ switch (this.url.protocol) {
402
+ case "file:": {
403
+ const { getFileSystem: getFileSystem2 } = await Promise.resolve().then(() => (init_utils2(), utils_exports));
404
+ sysFS = await getFileSystem2(this.url);
405
+ break;
406
+ }
407
+ default:
408
+ throw this.logger.Error().Url(this.url).Msg("unsupported protocol").AsError();
409
+ }
410
+ const dirName = this.url.pathname;
411
+ await sysFS.mkdir(dirName, { recursive: true });
412
+ return {
413
+ dirName,
414
+ sysFS,
415
+ fName: this.sthis.pathOps.join(dirName, `${id.replace(/[^a-zA-Z0-9]/g, "_")}.json`)
416
+ };
417
+ }
418
+ constructor(url, sthis) {
419
+ this.url = url;
420
+ this.sthis = sthis;
421
+ this.logger = sthis.logger;
422
+ }
423
+ async get(id) {
424
+ const ctx = await this._prepare(id);
425
+ try {
426
+ const p = await ctx.sysFS.readfile(ctx.fName);
427
+ const ki = JSON.parse(this.sthis.txt.decode(p));
428
+ return ki;
429
+ } catch (e) {
430
+ if (isNotFoundError(e)) {
431
+ return void 0;
432
+ }
433
+ throw this.logger.Error().Err(e).Str("file", ctx.dirName).Msg("read bag failed").AsError();
434
+ }
435
+ }
436
+ async set(id, item) {
437
+ const ctx = await this._prepare(id);
438
+ const p = this.sthis.txt.encode(JSON.stringify(item, null, 2));
439
+ await ctx.sysFS.writefile(ctx.fName, p);
440
+ }
441
+ };
442
+ }
443
+ });
444
+
445
+ // src/runtime/key-bag-indexdb.ts
446
+ var key_bag_indexdb_exports = {};
447
+ __export(key_bag_indexdb_exports, {
448
+ KeyBagProviderIndexDB: () => KeyBagProviderIndexDB
449
+ });
450
+ var import_idb, import_cement4, KeyBagProviderIndexDB;
451
+ var init_key_bag_indexdb = __esm({
452
+ "src/runtime/key-bag-indexdb.ts"() {
453
+ "use strict";
454
+ import_idb = require("idb");
455
+ init_utils2();
456
+ import_cement4 = require("@adviser/cement");
457
+ KeyBagProviderIndexDB = class {
458
+ constructor(url, sthis) {
459
+ this._db = new import_cement4.ResolveOnce();
460
+ this.sthis = sthis;
461
+ this.logger = sthis.logger;
462
+ this.url = url;
463
+ this.dbName = getPath(this.url, this.sthis);
464
+ }
465
+ async _prepare() {
466
+ return this._db.once(async () => {
467
+ return await (0, import_idb.openDB)(this.dbName, 1, {
468
+ upgrade(db) {
469
+ ["bag"].map((store) => {
470
+ db.createObjectStore(store, {
471
+ autoIncrement: false
472
+ });
473
+ });
474
+ }
475
+ });
476
+ });
477
+ }
478
+ async get(id) {
479
+ const db = await this._prepare();
480
+ const tx = db.transaction(["bag"], "readonly");
481
+ const keyItem = await tx.objectStore("bag").get(id);
482
+ await tx.done;
483
+ if (!keyItem) {
484
+ return void 0;
485
+ }
486
+ return keyItem;
487
+ }
488
+ async set(id, item) {
489
+ const db = await this._prepare();
490
+ const tx = db.transaction(["bag"], "readwrite");
491
+ await tx.objectStore("bag").put(item, id);
492
+ await tx.done;
493
+ }
494
+ };
495
+ }
496
+ });
497
+
498
+ // src/runtime/gateways/indexdb/version.ts
499
+ var INDEXDB_VERSION;
500
+ var init_version = __esm({
501
+ "src/runtime/gateways/indexdb/version.ts"() {
502
+ "use strict";
503
+ INDEXDB_VERSION = "v0.19-indexdb";
504
+ }
505
+ });
506
+
507
+ // src/runtime/gateways/indexdb/gateway@skip-esm.ts
508
+ var gateway_skip_esm_exports = {};
509
+ __export(gateway_skip_esm_exports, {
510
+ IndexDBGateway: () => IndexDBGateway,
511
+ IndexDBTestStore: () => IndexDBTestStore,
512
+ getIndexDBName: () => getIndexDBName
513
+ });
514
+ function ensureVersion(url) {
515
+ return url.build().defParam("version", INDEXDB_VERSION).URI();
516
+ }
517
+ function sanitzeKey(key) {
518
+ if (key.length === 1) {
519
+ key = key[0];
520
+ }
521
+ return key;
522
+ }
523
+ async function connectIdb(url, sthis) {
524
+ const dbName = getIndexDBName(url, sthis);
525
+ const once = await onceIndexDB.get(dbName.fullDb).once(async () => {
526
+ const db = await (0, import_idb2.openDB)(dbName.fullDb, 1, {
527
+ upgrade(db2) {
528
+ ["version", "data", "wal", "meta", "idx.data", "idx.wal", "idx.meta"].map((store) => {
529
+ db2.createObjectStore(store, {
530
+ autoIncrement: false
531
+ });
532
+ });
533
+ }
534
+ });
535
+ const found = await db.get("version", "version");
536
+ const version = ensureVersion(url).getParam("version");
537
+ if (!found) {
538
+ await db.put("version", { version }, "version");
539
+ } else if (found.version !== version) {
540
+ sthis.logger.Warn().Str("url", url.toString()).Str("version", version).Str("found", found.version).Msg("version mismatch");
541
+ }
542
+ return { db, dbName, version, url };
543
+ });
544
+ return {
545
+ ...once,
546
+ url: url.build().setParam("version", once.version).URI()
547
+ };
548
+ }
549
+ function joinDBName(...names) {
550
+ return names.map((i) => i.replace(/^[^a-zA-Z0-9]+/g, "").replace(/[^a-zA-Z0-9]+/g, "_")).filter((i) => i.length).join(".");
551
+ }
552
+ function getIndexDBName(iurl, sthis) {
553
+ const url = ensureVersion(iurl);
554
+ const fullDb = url.pathname.replace(/^\/+/, "").replace(/\?.*$/, "");
555
+ const dbName = url.getParam("name");
556
+ if (!dbName) throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
557
+ const result = joinDBName(fullDb, dbName);
558
+ const objStore = getStore(url, sthis, joinDBName).name;
559
+ const connectionKey = [result, objStore].join(":");
560
+ return {
561
+ fullDb: result,
562
+ objStore,
563
+ connectionKey,
564
+ dbName
565
+ };
566
+ }
567
+ var import_idb2, import_cement10, onceIndexDB, IndexDBGateway, IndexDBTestStore;
568
+ var init_gateway_skip_esm = __esm({
569
+ "src/runtime/gateways/indexdb/gateway@skip-esm.ts"() {
570
+ "use strict";
571
+ import_idb2 = require("idb");
572
+ import_cement10 = require("@adviser/cement");
573
+ init_version();
574
+ init_utils();
575
+ onceIndexDB = new import_cement10.KeyedResolvOnce();
576
+ IndexDBGateway = class {
577
+ constructor(sthis) {
578
+ this._db = {};
579
+ this.logger = ensureLogger(sthis, "IndexDBGateway");
580
+ this.sthis = sthis;
581
+ }
582
+ async start(baseURL) {
583
+ return (0, import_cement10.exception2Result)(async () => {
584
+ this.logger.Debug().Url(baseURL).Msg("starting");
585
+ await this.sthis.start();
586
+ const ic = await connectIdb(baseURL, this.sthis);
587
+ this._db = ic.db;
588
+ this.logger.Debug().Url(ic.url).Msg("started");
589
+ return ic.url;
590
+ });
591
+ }
592
+ async close() {
593
+ return import_cement10.Result.Ok(void 0);
594
+ }
595
+ async destroy(baseUrl) {
596
+ return (0, import_cement10.exception2Result)(async () => {
597
+ const type = getStore(baseUrl, this.sthis, joinDBName).name;
598
+ const idb = this._db;
599
+ const trans = idb.transaction(type, "readwrite");
600
+ const object_store = trans.objectStore(type);
601
+ const toDelete = [];
602
+ for (let cursor = await object_store.openCursor(); cursor; cursor = await cursor.continue()) {
603
+ toDelete.push(cursor.primaryKey);
604
+ }
605
+ for (const key of toDelete) {
606
+ await trans.db.delete(type, key);
607
+ }
608
+ await trans.done;
609
+ });
610
+ }
611
+ buildUrl(baseUrl, key) {
612
+ return Promise.resolve(import_cement10.Result.Ok(baseUrl.build().setParam("key", key).URI()));
613
+ }
614
+ async get(url) {
615
+ return exceptionWrapper(async () => {
616
+ const key = getKey(url, this.logger);
617
+ const store = getStore(url, this.sthis, joinDBName).name;
618
+ this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("getting");
619
+ const tx = this._db.transaction([store], "readonly");
620
+ const bytes = await tx.objectStore(store).get(sanitzeKey(key));
621
+ await tx.done;
622
+ if (!bytes) {
623
+ return import_cement10.Result.Err(new NotFoundError(`missing ${key}`));
624
+ }
625
+ return import_cement10.Result.Ok(bytes);
626
+ });
627
+ }
628
+ async put(url, value) {
629
+ return (0, import_cement10.exception2Result)(async () => {
630
+ const key = getKey(url, this.logger);
631
+ const store = getStore(url, this.sthis, joinDBName).name;
632
+ this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
633
+ const tx = this._db.transaction([store], "readwrite");
634
+ await tx.objectStore(store).put(value, sanitzeKey(key));
635
+ await tx.done;
636
+ });
637
+ }
638
+ async delete(url) {
639
+ return (0, import_cement10.exception2Result)(async () => {
640
+ const key = getKey(url, this.logger);
641
+ const store = getStore(url, this.sthis, joinDBName).name;
642
+ this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
643
+ const tx = this._db.transaction([store], "readwrite");
644
+ await tx.objectStore(store).delete(sanitzeKey(key));
645
+ await tx.done;
646
+ return import_cement10.Result.Ok(void 0);
647
+ });
648
+ }
649
+ };
650
+ IndexDBTestStore = class {
651
+ constructor(sthis) {
652
+ this.sthis = sthis;
653
+ this.logger = ensureLogger(sthis, "IndexDBTestStore", {});
654
+ }
655
+ async get(url, key) {
656
+ const ic = await connectIdb(url, this.sthis);
657
+ const store = getStore(ic.url, this.sthis, joinDBName).name;
658
+ this.logger.Debug().Str("key", key).Str("store", store).Msg("getting");
659
+ let bytes = await ic.db.get(store, sanitzeKey(key));
660
+ this.logger.Debug().Str("key", key).Str("store", store).Int("len", bytes.length).Msg("got");
661
+ if (typeof bytes === "string") {
662
+ bytes = this.sthis.txt.encode(bytes);
663
+ }
664
+ return bytes;
665
+ }
666
+ };
667
+ }
668
+ });
669
+
670
+ // src/index.ts
671
+ var src_exports = {};
672
+ __export(src_exports, {
673
+ CRDT: () => CRDT,
674
+ Database: () => Database,
675
+ Index: () => Index,
676
+ NotFoundError: () => NotFoundError,
677
+ PACKAGE_VERSION: () => PACKAGE_VERSION,
678
+ Result: () => import_cement.Result,
679
+ UInt8ArrayEqual: () => UInt8ArrayEqual,
680
+ blockstore: () => blockstore_exports,
681
+ bs: () => blockstore_exports,
682
+ dataDir: () => dataDir,
683
+ ensureLogger: () => ensureLogger,
684
+ ensureSuperLog: () => ensureSuperLog,
685
+ ensureSuperThis: () => ensureSuperThis,
686
+ exceptionWrapper: () => exceptionWrapper,
687
+ falsyToUndef: () => falsyToUndef,
688
+ fireproof: () => fireproof,
689
+ getKey: () => getKey,
690
+ getName: () => getName,
691
+ getStore: () => getStore,
692
+ index: () => index,
693
+ isFalsy: () => isFalsy,
694
+ isNotFoundError: () => isNotFoundError,
695
+ rt: () => runtime_exports,
696
+ runtime: () => runtime_exports,
697
+ throwFalsy: () => throwFalsy
698
+ });
699
+ module.exports = __toCommonJS(src_exports);
700
+
701
+ // src/database.ts
702
+ var import_cement14 = require("@adviser/cement");
703
+
704
+ // src/write-queue.ts
705
+ function writeQueue(worker, payload = Infinity, unbounded = false) {
706
+ const queue = [];
707
+ let isProcessing = false;
708
+ async function process() {
709
+ if (isProcessing || queue.length === 0) return;
710
+ isProcessing = true;
711
+ const tasksToProcess = queue.splice(0, payload);
712
+ const updates = tasksToProcess.map((item) => item.task);
713
+ if (unbounded) {
714
+ const promises = updates.map(async (update, index2) => {
715
+ try {
716
+ const result = await worker([update]);
717
+ tasksToProcess[index2].resolve(result);
718
+ } catch (error) {
719
+ tasksToProcess[index2].reject(error);
720
+ }
721
+ });
722
+ await Promise.all(promises);
723
+ } else {
724
+ try {
725
+ const result = await worker(updates);
726
+ tasksToProcess.forEach((task) => task.resolve(result));
727
+ } catch (error) {
728
+ tasksToProcess.forEach((task) => task.reject(error));
729
+ }
730
+ }
731
+ isProcessing = false;
732
+ void process();
733
+ }
734
+ return {
735
+ push(task) {
736
+ return new Promise((resolve, reject) => {
737
+ queue.push({ task, resolve, reject });
738
+ void process();
739
+ });
740
+ }
741
+ };
742
+ }
743
+
744
+ // src/crdt.ts
745
+ var import_cement13 = require("@adviser/cement");
746
+
747
+ // src/runtime/wait-pr-multiformats/block.ts
748
+ var block_exports = {};
749
+ __export(block_exports, {
750
+ Block: () => Block,
751
+ create: () => create,
752
+ createUnsafe: () => createUnsafe,
753
+ decode: () => decode,
754
+ encode: () => encode
755
+ });
756
+ var import_multiformats = require("multiformats");
757
+ var import_block = require("multiformats/block");
758
+ var Block = import_block.Block;
759
+ async function decode({
760
+ bytes,
761
+ codec: codec3,
762
+ hasher: hasher7
763
+ }) {
764
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
765
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
766
+ const value = await Promise.resolve(codec3.decode(bytes));
767
+ const hash = await hasher7.digest(bytes);
768
+ const cid = import_multiformats.CID.create(1, codec3.code, hash);
769
+ return new import_block.Block({ value, bytes, cid });
770
+ }
771
+ async function encode({
772
+ value,
773
+ codec: codec3,
774
+ hasher: hasher7
775
+ }) {
776
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
777
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
778
+ const bytes = await Promise.resolve(codec3.encode(value));
779
+ const hash = await hasher7.digest(bytes);
780
+ const cid = import_multiformats.CID.create(1, codec3.code, hash);
781
+ return new import_block.Block({ value, bytes, cid });
782
+ }
783
+ async function create({
784
+ bytes,
785
+ cid,
786
+ hasher: hasher7,
787
+ codec: codec3
788
+ }) {
789
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
790
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
791
+ const value = await Promise.resolve(codec3.decode(bytes));
792
+ const hash = await hasher7.digest(bytes);
793
+ if (!import_multiformats.bytes.equals(cid.multihash.bytes, hash.bytes)) {
794
+ throw new Error("CID hash does not match bytes");
795
+ }
796
+ return createUnsafe({
797
+ bytes,
798
+ cid,
799
+ value,
800
+ codec: codec3
801
+ });
802
+ }
803
+ async function createUnsafe({
804
+ bytes,
805
+ cid,
806
+ value: maybeValue,
807
+ codec: codec3
808
+ }) {
809
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
810
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
811
+ return new Block({
812
+ cid,
813
+ bytes,
814
+ value
815
+ });
816
+ }
817
+
818
+ // src/crdt-helpers.ts
819
+ var import_link = require("multiformats/link");
820
+ var import_sha25 = require("multiformats/hashes/sha2");
821
+ var codec = __toESM(require("@ipld/dag-cbor"), 1);
822
+ var import_crdt = require("@web3-storage/pail/crdt");
823
+ var import_clock2 = require("@web3-storage/pail/clock");
824
+ var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
825
+
826
+ // src/blockstore/index.ts
827
+ var blockstore_exports = {};
828
+ __export(blockstore_exports, {
829
+ BaseBlockstore: () => BaseBlockstore,
830
+ CarTransaction: () => CarTransaction,
831
+ CompactionFetcher: () => CompactionFetcher,
832
+ ConnectionBase: () => ConnectionBase,
833
+ EncryptedBlockstore: () => EncryptedBlockstore,
834
+ FragmentGateway: () => FragmentGateway,
835
+ Loader: () => Loader,
836
+ addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
837
+ ensureStart: () => ensureStart,
838
+ getGatewayFromURL: () => getGatewayFromURL,
839
+ parseCarFile: () => parseCarFile,
840
+ registerStoreProtocol: () => registerStoreProtocol,
841
+ setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
842
+ testStoreFactory: () => testStoreFactory,
843
+ toCIDBlock: () => toCIDBlock,
844
+ toStoreRuntime: () => toStoreRuntime
845
+ });
846
+
847
+ // src/blockstore/types.ts
848
+ function toCIDBlock(block) {
849
+ return block;
850
+ }
851
+
852
+ // src/blockstore/store-factory.ts
853
+ var import_cement11 = require("@adviser/cement");
854
+
855
+ // src/runtime/files.ts
856
+ var files_exports = {};
857
+ __export(files_exports, {
858
+ decodeFile: () => decodeFile,
859
+ encodeFile: () => encodeFile
860
+ });
861
+ var UnixFS = __toESM(require("@ipld/unixfs"), 1);
862
+ var raw = __toESM(require("multiformats/codecs/raw"), 1);
863
+ var import_fixed = require("@ipld/unixfs/file/chunker/fixed");
864
+ var import_balanced = require("@ipld/unixfs/file/layout/balanced");
865
+ var import_ipfs_unixfs_exporter = require("ipfs-unixfs-exporter");
866
+ var queuingStrategy = UnixFS.withCapacity();
867
+ var settings = UnixFS.configure({
868
+ fileChunkEncoder: raw,
869
+ smallFileEncoder: raw,
870
+ chunker: (0, import_fixed.withMaxChunkSize)(1024 * 1024),
871
+ fileLayout: (0, import_balanced.withWidth)(1024)
872
+ });
873
+ async function collect(collectable) {
874
+ const chunks = [];
875
+ await collectable.pipeTo(
876
+ new WritableStream({
877
+ write(chunk) {
878
+ chunks.push(chunk);
879
+ }
880
+ })
881
+ );
882
+ return chunks;
883
+ }
884
+ async function encodeFile(blob) {
885
+ const readable = createFileEncoderStream(blob);
886
+ const blocks = await collect(readable);
887
+ return { cid: blocks.at(-1).cid, blocks };
888
+ }
889
+ async function decodeFile(blocks, cid, meta) {
890
+ const entry = await (0, import_ipfs_unixfs_exporter.exporter)(cid.toString(), blocks, { length: meta.size });
891
+ const chunks = [];
892
+ for await (const chunk of entry.content()) {
893
+ chunks.push(chunk);
894
+ }
895
+ return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
896
+ }
897
+ function createFileEncoderStream(blob) {
898
+ const { readable, writable } = new TransformStream({}, queuingStrategy);
899
+ const unixfsWriter = UnixFS.createWriter({ writable, settings });
900
+ const fileBuilder = new UnixFSFileBuilder("", blob);
901
+ void (async () => {
902
+ await fileBuilder.finalize(unixfsWriter);
903
+ await unixfsWriter.close();
904
+ })();
905
+ return readable;
906
+ }
907
+ var UnixFSFileBuilder = class {
908
+ #file;
909
+ constructor(name, file) {
910
+ this.name = name;
911
+ this.#file = file;
912
+ }
913
+ async finalize(writer) {
914
+ const unixfsFileWriter = UnixFS.createFileWriter(writer);
915
+ await this.#file.stream().pipeTo(
916
+ new WritableStream({
917
+ async write(chunk) {
918
+ await unixfsFileWriter.write(chunk);
919
+ }
920
+ })
921
+ );
922
+ return await unixfsFileWriter.close();
923
+ }
924
+ };
925
+
926
+ // src/blockstore/store.ts
927
+ var import_dag_json2 = require("@ipld/dag-json");
928
+ var import_cement9 = require("@adviser/cement");
929
+
930
+ // src/types.ts
931
+ function isFalsy(value) {
932
+ return value === false && value === null && value === void 0;
933
+ }
934
+ function throwFalsy(value) {
935
+ if (isFalsy(value)) {
936
+ throw new Error("value is Falsy");
937
+ }
938
+ return value;
939
+ }
940
+ function falsyToUndef(value) {
941
+ if (isFalsy(value)) {
942
+ return void 0;
943
+ }
944
+ return value;
945
+ }
946
+
947
+ // src/blockstore/store.ts
948
+ init_utils();
949
+
950
+ // src/blockstore/loader.ts
951
+ var import_p_limit = __toESM(require("p-limit"), 1);
952
+ var import_car = require("@ipld/car");
953
+ var import_cement6 = require("@adviser/cement");
954
+
955
+ // src/blockstore/loader-helpers.ts
956
+ var import_sha2 = require("multiformats/hashes/sha2");
957
+ var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
958
+ async function parseCarFile(reader, logger) {
959
+ const roots = await reader.getRoots();
960
+ const header = await reader.get(roots[0]);
961
+ if (!header) throw logger.Error().Msg("missing header block").AsError();
962
+ const dec = await decode({ bytes: header.bytes, hasher: import_sha2.sha256, codec: dagCodec });
963
+ const fpvalue = dec.value;
964
+ if (fpvalue && !fpvalue.fp) {
965
+ throw logger.Error().Msg("missing fp").AsError();
966
+ }
967
+ return fpvalue.fp;
968
+ }
969
+
970
+ // src/blockstore/transaction.ts
971
+ var import_block3 = require("@web3-storage/pail/block");
972
+ var import_cement2 = require("@adviser/cement");
973
+ init_utils();
974
+ var CarTransaction = class extends import_block3.MemoryBlockstore {
975
+ constructor(parent, opts = { add: true, noLoader: false }) {
976
+ super();
977
+ if (opts.add) {
978
+ parent.transactions.add(this);
979
+ }
980
+ this.parent = parent;
981
+ }
982
+ async get(cid) {
983
+ return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
984
+ }
985
+ async superGet(cid) {
986
+ return super.get(cid);
987
+ }
988
+ };
989
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
990
+ const logger = ensureLogger(sthis, component, ctx);
991
+ const store = opts.store || {};
992
+ return {
993
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
994
+ applyMeta: (meta, snap) => {
995
+ return Promise.resolve();
996
+ },
997
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
998
+ compact: async (blocks) => {
999
+ return {};
1000
+ },
1001
+ autoCompact: 100,
1002
+ public: false,
1003
+ name: void 0,
1004
+ threshold: 1e3 * 1e3,
1005
+ ...opts,
1006
+ logger,
1007
+ keyBag: opts.keyBag || {},
1008
+ crypto: (0, import_cement2.toCryptoRuntime)(opts.crypto),
1009
+ store,
1010
+ storeRuntime: toStoreRuntime(store, sthis)
1011
+ };
1012
+ }
1013
+ function blockstoreFactory(sthis, opts) {
1014
+ if (opts.name) {
1015
+ return new EncryptedBlockstore(sthis, opts);
1016
+ } else {
1017
+ return new BaseBlockstore(opts);
1018
+ }
1019
+ }
1020
+ var BaseBlockstore = class {
1021
+ constructor(ebOpts = {}) {
1022
+ this.transactions = /* @__PURE__ */ new Set();
1023
+ this.sthis = ensureSuperThis(ebOpts);
1024
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1025
+ this.logger = this.ebOpts.logger;
1026
+ }
1027
+ // ready: Promise<void>;
1028
+ ready() {
1029
+ return Promise.resolve();
1030
+ }
1031
+ async close() {
1032
+ }
1033
+ async destroy() {
1034
+ }
1035
+ async compact() {
1036
+ }
1037
+ async get(cid) {
1038
+ if (!cid) throw this.logger.Error().Msg("required cid").AsError();
1039
+ for (const f of this.transactions) {
1040
+ const v = await f.superGet(cid);
1041
+ if (v) return v;
1042
+ }
1043
+ }
1044
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1045
+ async put(cid, block) {
1046
+ throw this.logger.Error().Msg("use a transaction to put").AsError();
1047
+ }
1048
+ // TransactionMeta
1049
+ async transaction(fn, _opts) {
1050
+ const t = new CarTransaction(this, _opts);
1051
+ const done = await fn(t);
1052
+ this.lastTxMeta = done;
1053
+ return { t, meta: done };
1054
+ }
1055
+ openTransaction(opts = { add: true, noLoader: false }) {
1056
+ return new CarTransaction(this, opts);
1057
+ }
1058
+ async commitTransaction(t, done, opts) {
1059
+ if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
1060
+ const cars = await this.loader?.commit(t, done, opts);
1061
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1062
+ setTimeout(() => void this.compact(), 10);
1063
+ }
1064
+ if (cars) {
1065
+ this.transactions.delete(t);
1066
+ return { meta: done, cars, t };
1067
+ }
1068
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
1069
+ }
1070
+ async *entries() {
1071
+ const seen = /* @__PURE__ */ new Set();
1072
+ for (const t of this.transactions) {
1073
+ for await (const blk of t.entries()) {
1074
+ if (seen.has(blk.cid.toString())) continue;
1075
+ seen.add(blk.cid.toString());
1076
+ yield blk;
1077
+ }
1078
+ }
1079
+ }
1080
+ };
1081
+ var EncryptedBlockstore = class extends BaseBlockstore {
1082
+ constructor(sthis, ebOpts) {
1083
+ super(ebOpts);
1084
+ this.compacting = false;
1085
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
1086
+ const { name } = ebOpts;
1087
+ if (!name) {
1088
+ throw this.logger.Error().Msg("name required").AsError();
1089
+ }
1090
+ this.name = name;
1091
+ this.loader = new Loader(this.name, ebOpts, sthis);
1092
+ }
1093
+ ready() {
1094
+ return this.loader.ready();
1095
+ }
1096
+ close() {
1097
+ return this.loader.close();
1098
+ }
1099
+ destroy() {
1100
+ return this.loader.destroy();
1101
+ }
1102
+ async get(cid) {
1103
+ const got = await super.get(cid);
1104
+ if (got) return got;
1105
+ if (!this.loader) {
1106
+ return;
1107
+ }
1108
+ return falsyToUndef(await this.loader.getBlock(cid));
1109
+ }
1110
+ async transaction(fn, opts = { noLoader: false }) {
1111
+ const { t, meta: done } = await super.transaction(fn);
1112
+ const cars = await this.loader.commit(t, done, opts);
1113
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1114
+ setTimeout(() => void this.compact(), 10);
1115
+ }
1116
+ if (cars) {
1117
+ this.transactions.delete(t);
1118
+ return { meta: done, cars, t };
1119
+ }
1120
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
1121
+ }
1122
+ async getFile(car, cid) {
1123
+ await this.ready();
1124
+ if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
1125
+ const reader = await this.loader.loadFileCar(
1126
+ car
1127
+ /*, isPublic */
1128
+ );
1129
+ const block = await reader.get(cid);
1130
+ if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
1131
+ return block.bytes;
1132
+ }
1133
+ async compact() {
1134
+ await this.ready();
1135
+ if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
1136
+ if (this.loader.carLog.length < 2) return;
1137
+ const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
1138
+ if (!compactFn || this.compacting) return;
1139
+ const blockLog = new CompactionFetcher(this);
1140
+ this.compacting = true;
1141
+ const meta = await compactFn(blockLog);
1142
+ await this.loader?.commit(blockLog.loggedBlocks, meta, {
1143
+ compact: true,
1144
+ noLoader: true
1145
+ });
1146
+ this.compacting = false;
1147
+ }
1148
+ async defaultCompact(blocks, logger) {
1149
+ if (!this.loader) {
1150
+ throw logger.Error().Msg("no loader").AsError();
1151
+ }
1152
+ if (!this.lastTxMeta) {
1153
+ throw logger.Error().Msg("no lastTxMeta").AsError();
1154
+ }
1155
+ for await (const blk of this.loader.entries(false)) {
1156
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1157
+ }
1158
+ for (const t of this.transactions) {
1159
+ for await (const blk of t.entries()) {
1160
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1161
+ }
1162
+ }
1163
+ return this.lastTxMeta;
1164
+ }
1165
+ async *entries() {
1166
+ for await (const blk of this.loader.entries()) {
1167
+ yield blk;
1168
+ }
1169
+ }
1170
+ };
1171
+ var CompactionFetcher = class {
1172
+ constructor(blocks) {
1173
+ this.blockstore = blocks;
1174
+ this.loggedBlocks = new CarTransaction(blocks);
1175
+ }
1176
+ async get(cid) {
1177
+ const block = await this.blockstore.get(cid);
1178
+ if (block) this.loggedBlocks.putSync(cid, block.bytes);
1179
+ return falsyToUndef(block);
1180
+ }
1181
+ };
1182
+
1183
+ // src/blockstore/commit-queue.ts
1184
+ var import_cement3 = require("@adviser/cement");
1185
+ var CommitQueue = class {
1186
+ constructor() {
1187
+ this.queue = [];
1188
+ this.processing = false;
1189
+ this._waitIdleItems = /* @__PURE__ */ new Set();
1190
+ }
1191
+ waitIdle() {
1192
+ if (this.queue.length === 0 && !this.processing) {
1193
+ return Promise.resolve();
1194
+ }
1195
+ const fn = new import_cement3.Future();
1196
+ this._waitIdleItems.add(fn);
1197
+ return fn.asPromise();
1198
+ }
1199
+ async enqueue(fn) {
1200
+ return new Promise((resolve, reject) => {
1201
+ const queueFn = async () => {
1202
+ try {
1203
+ resolve(await fn());
1204
+ } catch (e) {
1205
+ reject(e);
1206
+ } finally {
1207
+ this.processing = false;
1208
+ this.processNext();
1209
+ }
1210
+ };
1211
+ this.queue.push(queueFn);
1212
+ if (!this.processing) {
1213
+ this.processNext();
1214
+ }
1215
+ });
1216
+ }
1217
+ processNext() {
1218
+ if (this.queue.length > 0 && !this.processing) {
1219
+ this.processing = true;
1220
+ const queueFn = this.queue.shift();
1221
+ if (queueFn) {
1222
+ queueFn().finally(() => {
1223
+ });
1224
+ }
1225
+ }
1226
+ if (this.queue.length === 0 && !this.processing) {
1227
+ const toResolve = Array.from(this._waitIdleItems);
1228
+ this._waitIdleItems.clear();
1229
+ toResolve.map((fn) => fn.resolve());
1230
+ }
1231
+ }
1232
+ };
1233
+
1234
+ // src/runtime/key-bag.ts
1235
+ var key_bag_exports = {};
1236
+ __export(key_bag_exports, {
1237
+ KeyBag: () => KeyBag,
1238
+ getKeyBag: () => getKeyBag,
1239
+ registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
1240
+ });
1241
+ var import_cement5 = require("@adviser/cement");
1242
+ init_utils();
1243
+ var import_base582 = require("multiformats/bases/base58");
1244
+ var KeyBag = class {
1245
+ constructor(rt) {
1246
+ this.rt = rt;
1247
+ this._warnOnce = new import_cement5.ResolveOnce();
1248
+ this._seq = new import_cement5.ResolveSeq();
1249
+ this.logger = ensureLogger(rt.sthis, "KeyBag");
1250
+ this.logger.Debug().Msg("KeyBag created");
1251
+ }
1252
+ async subtleKey(key) {
1253
+ const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
1254
+ if (extractable) {
1255
+ this._warnOnce.once(
1256
+ () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
1257
+ );
1258
+ }
1259
+ return await this.rt.crypto.importKey(
1260
+ "raw",
1261
+ // raw or jwk
1262
+ import_base582.base58btc.decode(key),
1263
+ // hexStringToUint8Array(key), // raw data
1264
+ "AES-GCM",
1265
+ extractable,
1266
+ ["encrypt", "decrypt"]
1267
+ );
1268
+ }
1269
+ async ensureKeyFromUrl(url, keyFactory) {
1270
+ const storeKey = url.getParam("storekey");
1271
+ if (storeKey === "insecure") {
1272
+ return import_cement5.Result.Ok(url);
1273
+ }
1274
+ if (!storeKey) {
1275
+ const keyName = `@${keyFactory()}@`;
1276
+ const ret = await this.getNamedKey(keyName);
1277
+ if (ret.isErr()) {
1278
+ return ret;
1279
+ }
1280
+ const urb = url.build().setParam("storekey", keyName);
1281
+ return import_cement5.Result.Ok(urb.URI());
1282
+ }
1283
+ if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
1284
+ const ret = await this.getNamedKey(storeKey);
1285
+ if (ret.isErr()) {
1286
+ return ret;
1287
+ }
1288
+ }
1289
+ return import_cement5.Result.Ok(url);
1290
+ }
1291
+ async toKeyWithFingerPrint(keyStr) {
1292
+ const material = import_base582.base58btc.decode(keyStr);
1293
+ const key = await this.subtleKey(keyStr);
1294
+ const fpr = await this.rt.crypto.digestSHA256(material);
1295
+ return import_cement5.Result.Ok({
1296
+ key,
1297
+ fingerPrint: import_base582.base58btc.encode(new Uint8Array(fpr))
1298
+ });
1299
+ }
1300
+ async setNamedKey(name, key) {
1301
+ return this._seq.add(() => this._setNamedKey(name, key));
1302
+ }
1303
+ // avoid deadlock
1304
+ async _setNamedKey(name, key) {
1305
+ const item = {
1306
+ name,
1307
+ key
1308
+ };
1309
+ const bag = await this.rt.getBag();
1310
+ this.logger.Debug().Str("name", name).Msg("setNamedKey");
1311
+ await bag.set(name, item);
1312
+ return await this.toKeyWithFingerPrint(item.key);
1313
+ }
1314
+ async getNamedExtractableKey(name, failIfNotFound = false) {
1315
+ const ret = await this.getNamedKey(name, failIfNotFound);
1316
+ if (ret.isErr()) {
1317
+ return ret;
1318
+ }
1319
+ const named = ret.Ok();
1320
+ return import_cement5.Result.Ok({
1321
+ ...named,
1322
+ extract: async () => {
1323
+ const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
1324
+ return {
1325
+ key: ext,
1326
+ keyStr: import_base582.base58btc.encode(ext)
1327
+ };
1328
+ }
1329
+ });
1330
+ }
1331
+ async getNamedKey(name, failIfNotFound = false) {
1332
+ const id = this.rt.sthis.nextId(4).str;
1333
+ return this._seq.add(async () => {
1334
+ const bag = await this.rt.getBag();
1335
+ const named = await bag.get(name);
1336
+ if (named) {
1337
+ const fpr = await this.toKeyWithFingerPrint(named.key);
1338
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
1339
+ return fpr;
1340
+ }
1341
+ if (failIfNotFound) {
1342
+ this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
1343
+ return import_cement5.Result.Err(new Error(`Key not found: ${name}`));
1344
+ }
1345
+ const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
1346
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
1347
+ return ret;
1348
+ });
1349
+ }
1350
+ };
1351
+ var keyBagProviderFactories = new Map(
1352
+ [
1353
+ {
1354
+ protocol: "file:",
1355
+ factory: async (url, sthis) => {
1356
+ const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
1357
+ return new KeyBagProviderFile2(url, sthis);
1358
+ }
1359
+ },
1360
+ {
1361
+ protocol: "indexdb:",
1362
+ factory: async (url, sthis) => {
1363
+ const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
1364
+ return new KeyBagProviderIndexDB2(url, sthis);
1365
+ }
1366
+ }
1367
+ ].map((i) => [i.protocol, i])
1368
+ );
1369
+ function registerKeyBagProviderFactory(item) {
1370
+ const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
1371
+ keyBagProviderFactories.set(protocol, {
1372
+ ...item,
1373
+ protocol
1374
+ });
1375
+ }
1376
+ function defaultKeyBagOpts(sthis, kbo) {
1377
+ if (kbo.keyRuntime) {
1378
+ return kbo.keyRuntime;
1379
+ }
1380
+ const logger = ensureLogger(sthis, "KeyBag");
1381
+ let url;
1382
+ if (kbo.url) {
1383
+ url = import_cement5.URI.from(kbo.url);
1384
+ logger.Debug().Url(url).Msg("from opts");
1385
+ } else {
1386
+ let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
1387
+ if ((0, import_cement5.runtimeFn)().isBrowser) {
1388
+ url = import_cement5.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
1389
+ } else {
1390
+ if (!bagFnameOrUrl) {
1391
+ const home = sthis.env.get("HOME");
1392
+ bagFnameOrUrl = `${home}/.fireproof/keybag`;
1393
+ url = import_cement5.URI.from(`file://${bagFnameOrUrl}`);
1394
+ } else {
1395
+ url = import_cement5.URI.from(bagFnameOrUrl);
1396
+ }
1397
+ }
1398
+ logger.Debug().Url(url).Msg("from env");
1399
+ }
1400
+ const kitem = keyBagProviderFactories.get(url.protocol);
1401
+ if (!kitem) {
1402
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
1403
+ }
1404
+ const getBag = async () => kitem.factory(url, sthis);
1405
+ if (url.hasParam("masterkey")) {
1406
+ throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
1407
+ }
1408
+ return {
1409
+ url,
1410
+ crypto: kbo.crypto || (0, import_cement5.toCryptoRuntime)({}),
1411
+ sthis,
1412
+ logger,
1413
+ keyLength: kbo.keyLength || 16,
1414
+ getBag,
1415
+ id: () => {
1416
+ return url.toString();
1417
+ }
1418
+ };
1419
+ }
1420
+ var _keyBags = new import_cement5.KeyedResolvOnce();
1421
+ async function getKeyBag(sthis, kbo = {}) {
1422
+ await sthis.start();
1423
+ const rt = defaultKeyBagOpts(sthis, kbo);
1424
+ return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
1425
+ }
1426
+
1427
+ // src/blockstore/commitor.ts
1428
+ var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
1429
+ var import_sha22 = require("multiformats/hashes/sha2");
1430
+ var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
1431
+ async function encodeCarFile(roots, t, codec3) {
1432
+ let size = 0;
1433
+ const headerSize = CBW.headerLength({ roots });
1434
+ size += headerSize;
1435
+ for (const { cid, bytes } of t.entries()) {
1436
+ size += CBW.blockLength({ cid, bytes });
1437
+ }
1438
+ const buffer = new Uint8Array(size);
1439
+ const writer = CBW.createWriter(buffer, { headerSize });
1440
+ for (const r of roots) {
1441
+ writer.addRoot(r);
1442
+ }
1443
+ for (const { cid, bytes } of t.entries()) {
1444
+ writer.write({ cid, bytes });
1445
+ }
1446
+ writer.close();
1447
+ return await encode({ value: writer.bytes, hasher: import_sha22.sha256, codec: codec3 });
1448
+ }
1449
+ async function createCarFile(encoder, cid, t) {
1450
+ return encodeCarFile([cid], t, encoder);
1451
+ }
1452
+ async function commitFiles(fileStore, walStore, t, done) {
1453
+ const { files: roots } = makeFileCarHeader(done);
1454
+ const cids = [];
1455
+ const codec3 = (await fileStore.keyedCrypto()).codec();
1456
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
1457
+ for (const car of cars) {
1458
+ const { cid, bytes } = car;
1459
+ await fileStore.save({ cid, bytes });
1460
+ await walStore.enqueueFile(
1461
+ cid
1462
+ /*, !!opts.public*/
1463
+ );
1464
+ cids.push(cid);
1465
+ }
1466
+ return cids;
1467
+ }
1468
+ function makeFileCarHeader(result) {
1469
+ const files = [];
1470
+ for (const [, meta] of Object.entries(result.files || {})) {
1471
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1472
+ files.push(meta.cid);
1473
+ }
1474
+ }
1475
+ return { ...result, files };
1476
+ }
1477
+ async function prepareCarFilesFiles(encoder, roots, t) {
1478
+ return [await encodeCarFile(roots, t, encoder)];
1479
+ }
1480
+ function makeCarHeader(meta, cars, compact = false) {
1481
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1482
+ return { ...coreHeader, meta };
1483
+ }
1484
+ async function encodeCarHeader(fp) {
1485
+ return await encode({
1486
+ value: { fp },
1487
+ hasher: import_sha22.sha256,
1488
+ codec: dagCodec2
1489
+ });
1490
+ }
1491
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
1492
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
1493
+ const rootBlock = await encodeCarHeader(fp);
1494
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
1495
+ const cids = [];
1496
+ for (const car of cars) {
1497
+ const { cid, bytes } = car;
1498
+ await params.carStore.save({ cid, bytes });
1499
+ cids.push(cid);
1500
+ }
1501
+ const newDbMeta = { cars: cids };
1502
+ await params.WALStore.enqueue(newDbMeta, opts);
1503
+ await params.metaStore.save(newDbMeta);
1504
+ return { cgrp: cids, header: fp };
1505
+ }
1506
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
1507
+ const carFiles = [];
1508
+ threshold = threshold || 128e3 * 8;
1509
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1510
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1511
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
1512
+ let cidRootBlock = rootBlock;
1513
+ for (const { cid, bytes } of t.entries()) {
1514
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
1515
+ if (newsize >= threshold) {
1516
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1517
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1518
+ clonedt.putSync(cid, bytes);
1519
+ cidRootBlock = { cid, bytes };
1520
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
1521
+ } else {
1522
+ clonedt.putSync(cid, bytes);
1523
+ }
1524
+ }
1525
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1526
+ return carFiles;
1527
+ }
1528
+
1529
+ // src/blockstore/loader.ts
1530
+ var import_sha23 = require("multiformats/hashes/sha2");
1531
+
1532
+ // src/blockstore/task-manager.ts
1533
+ init_utils();
1534
+ var TaskManager = class {
1535
+ constructor(sthis, callback) {
1536
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
1537
+ this.queue = [];
1538
+ this.isProcessing = false;
1539
+ this.logger = ensureLogger(sthis, "TaskManager");
1540
+ this.callback = callback;
1541
+ }
1542
+ async handleEvent(cid, parents, dbMeta) {
1543
+ for (const parent of parents) {
1544
+ this.eventsWeHandled.add(parent.toString());
1545
+ }
1546
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
1547
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
1548
+ void this.processQueue();
1549
+ }
1550
+ async processQueue() {
1551
+ if (this.isProcessing) return;
1552
+ this.isProcessing = true;
1553
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1554
+ const first = filteredQueue[0];
1555
+ if (!first) {
1556
+ return;
1557
+ }
1558
+ try {
1559
+ await this.callback(first.dbMeta);
1560
+ this.eventsWeHandled.add(first.cid);
1561
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1562
+ } catch (err) {
1563
+ if (first.retries++ > 3) {
1564
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
1565
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
1566
+ }
1567
+ await new Promise((resolve) => setTimeout(resolve, 50));
1568
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
1569
+ } finally {
1570
+ this.isProcessing = false;
1571
+ if (this.queue.length > 0) {
1572
+ void this.processQueue();
1573
+ }
1574
+ }
1575
+ }
1576
+ };
1577
+
1578
+ // src/blockstore/loader.ts
1579
+ function carLogIncludesGroup(list, cids) {
1580
+ return list.some((arr) => {
1581
+ return arr.toString() === cids.toString();
1582
+ });
1583
+ }
1584
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
1585
+ const byString = /* @__PURE__ */ new Map();
1586
+ for (const cid of list) {
1587
+ if (remove.has(cid.toString())) continue;
1588
+ byString.set(cid.toString(), cid);
1589
+ }
1590
+ return [...byString.values()];
1591
+ }
1592
+ var Loader = class {
1593
+ constructor(name, ebOpts, sthis) {
1594
+ this.commitQueue = new CommitQueue();
1595
+ this.isCompacting = false;
1596
+ this.carReaders = /* @__PURE__ */ new Map();
1597
+ this.seenCompacted = /* @__PURE__ */ new Set();
1598
+ this.processedCars = /* @__PURE__ */ new Set();
1599
+ this.carLog = [];
1600
+ this.getBlockCache = /* @__PURE__ */ new Map();
1601
+ this.seenMeta = /* @__PURE__ */ new Set();
1602
+ this.writeLimit = (0, import_p_limit.default)(1);
1603
+ this.onceReady = new import_cement6.ResolveOnce();
1604
+ this.name = name;
1605
+ this.sthis = sthis;
1606
+ this.ebOpts = defaultedBlockstoreRuntime(
1607
+ sthis,
1608
+ {
1609
+ ...ebOpts,
1610
+ name
1611
+ },
1612
+ "Loader"
1613
+ );
1614
+ this.logger = this.ebOpts.logger;
1615
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
1616
+ await this.handleDbMetasFromStore([dbMeta]);
1617
+ });
1618
+ }
1619
+ // readonly id = uuidv4();
1620
+ async keyBag() {
1621
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
1622
+ }
1623
+ async carStore() {
1624
+ return this.ebOpts.storeRuntime.makeDataStore(this);
1625
+ }
1626
+ async fileStore() {
1627
+ return this.ebOpts.storeRuntime.makeDataStore(this);
1628
+ }
1629
+ async WALStore() {
1630
+ return this.ebOpts.storeRuntime.makeWALStore(this);
1631
+ }
1632
+ async metaStore() {
1633
+ return this.ebOpts.storeRuntime.makeMetaStore(this);
1634
+ }
1635
+ async ready() {
1636
+ return this.onceReady.once(async () => {
1637
+ const metas = await (await this.metaStore()).load();
1638
+ if (this.ebOpts.meta) {
1639
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
1640
+ } else if (metas) {
1641
+ await this.handleDbMetasFromStore(metas);
1642
+ }
1643
+ });
1644
+ }
1645
+ async close() {
1646
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1647
+ await Promise.all(toClose.map((store) => store.close()));
1648
+ }
1649
+ async destroy() {
1650
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1651
+ await Promise.all(toDestroy.map((store) => store.destroy()));
1652
+ }
1653
+ // async snapToCar(carCid: AnyLink | string) {
1654
+ // await this.ready
1655
+ // if (typeof carCid === 'string') {
1656
+ // carCid = CID.parse(carCid)
1657
+ // }
1658
+ // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
1659
+ // this.carLog = [carCid, ...carHeader.cars]
1660
+ // await this.getMoreReaders(carHeader.cars)
1661
+ // await this._applyCarHeader(carHeader, true)
1662
+ // }
1663
+ async handleDbMetasFromStore(metas) {
1664
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
1665
+ for (const meta of metas) {
1666
+ await this.writeLimit(async () => {
1667
+ await this.mergeDbMetaIntoClock(meta);
1668
+ });
1669
+ }
1670
+ }
1671
+ async mergeDbMetaIntoClock(meta) {
1672
+ if (this.isCompacting) {
1673
+ throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1674
+ }
1675
+ if (this.seenMeta.has(meta.cars.toString())) return;
1676
+ this.seenMeta.add(meta.cars.toString());
1677
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
1678
+ return;
1679
+ }
1680
+ const carHeader = await this.loadCarHeaderFromMeta(meta);
1681
+ carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1682
+ await this.getMoreReaders(carHeader.cars.flat());
1683
+ this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1684
+ await this.ebOpts.applyMeta?.(carHeader.meta);
1685
+ }
1686
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1687
+ // const { key } = meta;
1688
+ // if (key) {
1689
+ // await this.setKey(key);
1690
+ // }
1691
+ // }
1692
+ async loadCarHeaderFromMeta({ cars: cids }) {
1693
+ const reader = await this.loadCar(cids[0]);
1694
+ return await parseCarFile(reader, this.logger);
1695
+ }
1696
+ // async _getKey(): Promise<string | undefined> {
1697
+ // if (this.key) return this.key;
1698
+ // // generate a random key
1699
+ // if (!this.ebOpts.public) {
1700
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1701
+ // }
1702
+ // return this.key || undefined;
1703
+ // }
1704
+ async commitFiles(t, done) {
1705
+ await this.ready();
1706
+ const fstore = await this.fileStore();
1707
+ const wstore = await this.WALStore();
1708
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1709
+ }
1710
+ async loadFileCar(cid) {
1711
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1712
+ }
1713
+ async commit(t, done, opts = { noLoader: false, compact: false }) {
1714
+ await this.ready();
1715
+ const fstore = await this.fileStore();
1716
+ const params = {
1717
+ encoder: (await fstore.keyedCrypto()).codec(),
1718
+ carLog: this.carLog,
1719
+ carStore: fstore,
1720
+ WALStore: await this.WALStore(),
1721
+ metaStore: await this.metaStore(),
1722
+ threshold: this.ebOpts.threshold
1723
+ };
1724
+ return this.commitQueue.enqueue(async () => {
1725
+ await this.cacheTransaction(t);
1726
+ const ret = await commit(params, t, done, opts);
1727
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1728
+ return ret.cgrp;
1729
+ });
1730
+ }
1731
+ async updateCarLog(cids, fp, compact) {
1732
+ if (compact) {
1733
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
1734
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1735
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1736
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1737
+ } else {
1738
+ this.carLog.unshift(cids);
1739
+ }
1740
+ }
1741
+ async cacheTransaction(t) {
1742
+ for await (const block of t.entries()) {
1743
+ const sBlock = block.cid.toString();
1744
+ if (!this.getBlockCache.has(sBlock)) {
1745
+ this.getBlockCache.set(sBlock, block);
1746
+ }
1747
+ }
1748
+ }
1749
+ async cacheCarReader(carCidStr, reader) {
1750
+ if (this.processedCars.has(carCidStr)) return;
1751
+ this.processedCars.add(carCidStr);
1752
+ for await (const block of reader.blocks()) {
1753
+ const sBlock = block.cid.toString();
1754
+ if (!this.getBlockCache.has(sBlock)) {
1755
+ this.getBlockCache.set(sBlock, block);
1756
+ }
1757
+ }
1758
+ }
1759
+ async removeCidsForCompact(cid) {
1760
+ const carHeader = await this.loadCarHeaderFromMeta({
1761
+ cars: [cid]
1762
+ });
1763
+ for (const cids of carHeader.compact) {
1764
+ for (const cid2 of cids) {
1765
+ await (await this.carStore()).remove(cid2);
1766
+ }
1767
+ }
1768
+ }
1769
+ // async flushCars() {
1770
+ // await this.ready
1771
+ // // for each cid in car log, make a dbMeta
1772
+ // for (const cid of this.carLog) {
1773
+ // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1774
+ // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1775
+ // }
1776
+ // }
1777
+ async *entries(cache2 = true) {
1778
+ await this.ready();
1779
+ if (cache2) {
1780
+ for (const [, block] of this.getBlockCache) {
1781
+ yield block;
1782
+ }
1783
+ } else {
1784
+ for (const [, block] of this.getBlockCache) {
1785
+ yield block;
1786
+ }
1787
+ for (const cids of this.carLog) {
1788
+ for (const cid of cids) {
1789
+ const reader = await this.loadCar(cid);
1790
+ if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1791
+ for await (const block of reader.blocks()) {
1792
+ const sCid = block.cid.toString();
1793
+ if (!this.getBlockCache.has(sCid)) {
1794
+ yield block;
1795
+ }
1796
+ }
1797
+ }
1798
+ }
1799
+ }
1800
+ }
1801
+ async getBlock(cid) {
1802
+ await this.ready();
1803
+ const sCid = cid.toString();
1804
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1805
+ const getCarCid = async (carCid) => {
1806
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1807
+ const reader = await this.loadCar(carCid);
1808
+ if (!reader) {
1809
+ throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1810
+ }
1811
+ await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1812
+ return;
1813
+ });
1814
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1815
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1816
+ };
1817
+ const getCompactCarCids = async (carCid) => {
1818
+ const reader = await this.loadCar(carCid);
1819
+ if (!reader) {
1820
+ throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1821
+ }
1822
+ const header = await parseCarFile(reader, this.logger);
1823
+ const compacts = header.compact;
1824
+ let got2;
1825
+ const batchSize2 = 5;
1826
+ for (let i = 0; i < compacts.length; i += batchSize2) {
1827
+ const promises = [];
1828
+ for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1829
+ for (const cid2 of compacts[j]) {
1830
+ promises.push(getCarCid(cid2));
1831
+ }
1832
+ }
1833
+ try {
1834
+ got2 = await Promise.any(promises);
1835
+ } catch {
1836
+ }
1837
+ if (got2) break;
1838
+ }
1839
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1840
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1841
+ };
1842
+ let got;
1843
+ const batchSize = 5;
1844
+ for (let i = 0; i < this.carLog.length; i += batchSize) {
1845
+ const batch = this.carLog.slice(i, i + batchSize);
1846
+ const promises = batch.flatMap((slice) => slice.map(getCarCid));
1847
+ try {
1848
+ got = await Promise.any(promises);
1849
+ } catch {
1850
+ }
1851
+ if (got) break;
1852
+ }
1853
+ if (!got) {
1854
+ try {
1855
+ got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1856
+ } catch {
1857
+ }
1858
+ }
1859
+ return got;
1860
+ }
1861
+ async loadCar(cid) {
1862
+ if (!this.carStore) {
1863
+ throw this.logger.Error().Msg("car store not initialized").AsError();
1864
+ }
1865
+ const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1866
+ return loaded;
1867
+ }
1868
+ async makeDecoderAndCarReader(cid, local, remote) {
1869
+ const cidsString = cid.toString();
1870
+ let loadedCar = void 0;
1871
+ let activeStore = local;
1872
+ try {
1873
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1874
+ loadedCar = await local.load(cid);
1875
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1876
+ } catch (e) {
1877
+ if (remote) {
1878
+ const remoteCar = await remote.load(cid);
1879
+ if (remoteCar) {
1880
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1881
+ await local.save(remoteCar);
1882
+ loadedCar = remoteCar;
1883
+ activeStore = remote;
1884
+ }
1885
+ } else {
1886
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1887
+ }
1888
+ }
1889
+ if (!loadedCar) {
1890
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
1891
+ }
1892
+ const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha23.sha256, codec: (await activeStore.keyedCrypto()).codec() });
1893
+ const rawReader = await import_car.CarReader.fromBytes(bytes.value);
1894
+ const readerP = Promise.resolve(rawReader);
1895
+ const cachedReaderP = readerP.then(async (reader) => {
1896
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
1897
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1898
+ return;
1899
+ });
1900
+ return reader;
1901
+ });
1902
+ this.carReaders.set(cidsString, cachedReaderP);
1903
+ return readerP;
1904
+ }
1905
+ //What if instead it returns an Array of CarHeader
1906
+ async storesLoadCar(cid, local, remote) {
1907
+ const cidsString = cid.toString();
1908
+ let dacr = this.carReaders.get(cidsString);
1909
+ if (!dacr) {
1910
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
1911
+ this.carReaders.set(cidsString, dacr);
1912
+ }
1913
+ return dacr;
1914
+ }
1915
+ async getMoreReaders(cids) {
1916
+ const limit = (0, import_p_limit.default)(5);
1917
+ const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1918
+ await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1919
+ }
1920
+ };
1921
+
1922
+ // src/runtime/keyed-crypto.ts
1923
+ var keyed_crypto_exports = {};
1924
+ __export(keyed_crypto_exports, {
1925
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1926
+ keyedCryptoFactory: () => keyedCryptoFactory
1927
+ });
1928
+ init_utils();
1929
+ var import_base583 = require("multiformats/bases/base58");
1930
+ var import_sha24 = require("multiformats/hashes/sha2");
1931
+ var CBOR = __toESM(require("cborg"), 1);
1932
+ var generateIV = {
1933
+ random: {
1934
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1935
+ calc: async (ko, crypto, data) => {
1936
+ return crypto.randomBytes(ko.ivLength);
1937
+ },
1938
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1939
+ verify: async (ko, crypto, iv, data) => {
1940
+ return true;
1941
+ }
1942
+ },
1943
+ hash: {
1944
+ calc: async (ko, crypto, data) => {
1945
+ const hash = await import_sha24.sha256.digest(data);
1946
+ const hashBytes = new Uint8Array(hash.bytes);
1947
+ const hashArray = new Uint8Array(ko.ivLength);
1948
+ for (let i = 0; i < hashBytes.length; i++) {
1949
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
1950
+ }
1951
+ return hashArray;
1952
+ },
1953
+ verify: async function(ko, crypto, iv, data) {
1954
+ return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1955
+ }
1956
+ }
1957
+ };
1958
+ function getGenerateIVFn(url, opts) {
1959
+ const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
1960
+ return generateIV[ivhash] || generateIV["hash"];
1961
+ }
1962
+ var BlockIvKeyIdCodec = class {
1963
+ constructor(ko, iv, opts) {
1964
+ this.code = 3147065;
1965
+ this.name = "Fireproof@encrypted-block:aes-gcm";
1966
+ this.ko = ko;
1967
+ this.iv = iv;
1968
+ this.opts = opts || {};
1969
+ }
1970
+ async encode(data) {
1971
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1972
+ const { iv } = this.ko.algo(calcIv);
1973
+ const fprt = await this.ko.fingerPrint();
1974
+ const keyId = import_base583.base58btc.decode(fprt);
1975
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1976
+ return CBOR.encode({
1977
+ iv,
1978
+ keyId,
1979
+ data: await this.ko._encrypt({ iv, bytes: data })
1980
+ });
1981
+ }
1982
+ async decode(abytes) {
1983
+ let bytes;
1984
+ if (abytes instanceof Uint8Array) {
1985
+ bytes = abytes;
1986
+ } else {
1987
+ bytes = new Uint8Array(abytes);
1988
+ }
1989
+ const { iv, keyId, data } = CBOR.decode(bytes);
1990
+ const fprt = await this.ko.fingerPrint();
1991
+ this.ko.logger.Debug().Str("fp", import_base583.base58btc.encode(keyId)).Msg("decode");
1992
+ if (import_base583.base58btc.encode(keyId) !== fprt) {
1993
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base583.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
1994
+ }
1995
+ const result = await this.ko._decrypt({ iv, bytes: data });
1996
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1997
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1998
+ }
1999
+ return result;
2000
+ }
2001
+ };
2002
+ var keyedCrypto = class {
2003
+ constructor(url, key, cyopt, sthis) {
2004
+ this.ivLength = 12;
2005
+ this.isEncrypting = true;
2006
+ this.logger = ensureLogger(sthis, "keyedCrypto");
2007
+ this.crypto = cyopt;
2008
+ this.key = key;
2009
+ this.url = url;
2010
+ }
2011
+ fingerPrint() {
2012
+ return Promise.resolve(this.key.fingerPrint);
2013
+ }
2014
+ codec(iv, opts) {
2015
+ return new BlockIvKeyIdCodec(this, iv, opts);
2016
+ }
2017
+ algo(iv) {
2018
+ return {
2019
+ name: "AES-GCM",
2020
+ iv: iv || this.crypto.randomBytes(this.ivLength),
2021
+ tagLength: 128
2022
+ };
2023
+ }
2024
+ async _decrypt(data) {
2025
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
2026
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
2027
+ }
2028
+ async _encrypt(data) {
2029
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
2030
+ const a = this.algo(data.iv);
2031
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
2032
+ }
2033
+ };
2034
+ var nullCodec = class {
2035
+ constructor() {
2036
+ this.code = 0;
2037
+ this.name = "Fireproof@unencrypted-block";
2038
+ }
2039
+ encode(data) {
2040
+ return data;
2041
+ }
2042
+ decode(data) {
2043
+ return data;
2044
+ }
2045
+ };
2046
+ var noCrypto = class {
2047
+ constructor(url, cyrt, sthis) {
2048
+ this.ivLength = 0;
2049
+ this.code = 0;
2050
+ this.name = "Fireproof@unencrypted-block";
2051
+ this.isEncrypting = false;
2052
+ this._fingerPrint = "noCrypto:" + Math.random();
2053
+ this.logger = ensureLogger(sthis, "noCrypto");
2054
+ this.crypto = cyrt;
2055
+ this.url = url;
2056
+ }
2057
+ fingerPrint() {
2058
+ return Promise.resolve(this._fingerPrint);
2059
+ }
2060
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2061
+ codec(iv) {
2062
+ return new nullCodec();
2063
+ }
2064
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2065
+ algo(iv) {
2066
+ return {
2067
+ name: "noCrypto",
2068
+ iv: new Uint8Array(),
2069
+ tagLength: 0
2070
+ };
2071
+ }
2072
+ _decrypt() {
2073
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
2074
+ }
2075
+ _encrypt() {
2076
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
2077
+ }
2078
+ };
2079
+ async function keyedCryptoFactory(url, kb, sthis) {
2080
+ const storekey = url.getParam("storekey");
2081
+ if (storekey && storekey !== "insecure") {
2082
+ let rkey = await kb.getNamedKey(storekey, true);
2083
+ if (rkey.isErr()) {
2084
+ try {
2085
+ rkey = await kb.toKeyWithFingerPrint(storekey);
2086
+ } catch (e) {
2087
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
2088
+ }
2089
+ }
2090
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
2091
+ }
2092
+ return new noCrypto(url, kb.rt.crypto, sthis);
2093
+ }
2094
+
2095
+ // src/blockstore/fragment-gateway.ts
2096
+ var import_cement7 = require("@adviser/cement");
2097
+ var import_base584 = require("multiformats/bases/base58");
2098
+ var import_cborg = require("cborg");
2099
+ init_utils();
2100
+ function getFragSize(url) {
2101
+ const fragSize = url.getParam("fragSize");
2102
+ let ret = 0;
2103
+ if (fragSize) {
2104
+ ret = parseInt(fragSize);
2105
+ }
2106
+ if (isNaN(ret) || ret <= 0) {
2107
+ ret = 0;
2108
+ }
2109
+ return ret;
2110
+ }
2111
+ async function getFrags(url, innerGW, headerSize, logger) {
2112
+ const fragSize = getFragSize(url);
2113
+ if (!fragSize) {
2114
+ const res = await innerGW.get(url);
2115
+ if (res.isErr()) {
2116
+ return [res];
2117
+ }
2118
+ const data = res.unwrap();
2119
+ return [
2120
+ import_cement7.Result.Ok({
2121
+ fid: new Uint8Array(0),
2122
+ ofs: 0,
2123
+ len: data.length,
2124
+ data
2125
+ })
2126
+ ];
2127
+ }
2128
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
2129
+ if (firstRaw.isErr()) {
2130
+ return [firstRaw];
2131
+ }
2132
+ const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
2133
+ const blockSize = firstFragment.data.length;
2134
+ const ops = [Promise.resolve(import_cement7.Result.Ok(firstFragment))];
2135
+ const fidStr = import_base584.base58btc.encode(firstFragment.fid);
2136
+ const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
2137
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
2138
+ ops.push(
2139
+ (async (furl, ofs2) => {
2140
+ const raw2 = await innerGW.get(furl);
2141
+ if (raw2.isErr()) {
2142
+ return raw2;
2143
+ }
2144
+ const fragment = (0, import_cborg.decode)(raw2.unwrap());
2145
+ if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
2146
+ return import_cement7.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
2147
+ }
2148
+ if (fragment.ofs !== ofs2) {
2149
+ return import_cement7.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
2150
+ }
2151
+ return import_cement7.Result.Ok(fragment);
2152
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
2153
+ );
2154
+ }
2155
+ return Promise.all(ops);
2156
+ }
2157
+ var FragmentGateway = class {
2158
+ constructor(sthis, innerGW) {
2159
+ this.fidLength = 4;
2160
+ this.headerSize = 32;
2161
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
2162
+ this.logger = this.sthis.logger;
2163
+ this.innerGW = innerGW;
2164
+ }
2165
+ slicer(url, body) {
2166
+ const fragSize = getFragSize(url);
2167
+ if (!fragSize) {
2168
+ return [this.innerGW.put(url, body)];
2169
+ }
2170
+ const blocksize = fragSize - this.headerSize;
2171
+ if (blocksize <= 0) {
2172
+ throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
2173
+ }
2174
+ const ops = [];
2175
+ const fid = this.sthis.nextId(this.fidLength);
2176
+ const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
2177
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
2178
+ const block = (0, import_cborg.encode)({
2179
+ fid: fid.bin,
2180
+ ofs,
2181
+ len: body.length,
2182
+ data: body.slice(ofs, ofs + blocksize)
2183
+ });
2184
+ if (block.length > fragSize) {
2185
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
2186
+ }
2187
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
2188
+ }
2189
+ return ops;
2190
+ }
2191
+ buildUrl(baseUrl, key) {
2192
+ return this.innerGW.buildUrl(baseUrl, key);
2193
+ }
2194
+ async destroy(iurl) {
2195
+ return this.innerGW.destroy(iurl);
2196
+ }
2197
+ async start(url) {
2198
+ this.headerSize = (0, import_cborg.encode)({
2199
+ fid: this.sthis.nextId(this.fidLength).bin,
2200
+ ofs: 1024 * 1024,
2201
+ // 32bit
2202
+ len: 16 * 1024 * 1024,
2203
+ // 32bit
2204
+ data: new Uint8Array(1024)
2205
+ }).length - 1024;
2206
+ return this.innerGW.start(url);
2207
+ }
2208
+ async close(url) {
2209
+ return this.innerGW.close(url);
2210
+ }
2211
+ async put(url, body) {
2212
+ await Promise.all(this.slicer(url, body));
2213
+ return import_cement7.Result.Ok(void 0);
2214
+ }
2215
+ async get(url) {
2216
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
2217
+ let buffer = void 0;
2218
+ for (const rfrag of rfrags) {
2219
+ if (rfrag.isErr()) {
2220
+ return import_cement7.Result.Err(rfrag.Err());
2221
+ }
2222
+ const frag = rfrag.Ok();
2223
+ buffer = buffer || new Uint8Array(frag.len);
2224
+ buffer.set(frag.data, frag.ofs);
2225
+ }
2226
+ return import_cement7.Result.Ok(buffer || new Uint8Array(0));
2227
+ }
2228
+ async subscribe(url, callback) {
2229
+ if (this.innerGW.subscribe) {
2230
+ return this.innerGW.subscribe(url, callback);
2231
+ } else {
2232
+ return import_cement7.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
2233
+ }
2234
+ }
2235
+ async delete(url) {
2236
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
2237
+ for (const rfrag of rfrags) {
2238
+ if (rfrag.isErr()) {
2239
+ return import_cement7.Result.Err(rfrag.Err());
2240
+ }
2241
+ const frag = rfrag.Ok();
2242
+ const fidStr = import_base584.base58btc.encode(frag.fid);
2243
+ const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
2244
+ await this.innerGW.delete(fragUrl);
2245
+ }
2246
+ return import_cement7.Result.Ok(void 0);
2247
+ }
2248
+ };
2249
+
2250
+ // src/blockstore/meta-key-helper.ts
2251
+ var import_dag_json = require("@ipld/dag-json");
2252
+ var import_clock = require("@web3-storage/pail/clock");
2253
+ var import_multiformats2 = require("multiformats");
2254
+ var import_base64 = require("multiformats/bases/base64");
2255
+ var import_cement8 = require("@adviser/cement");
2256
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
2257
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
2258
+ if (!crdtEntries.length) {
2259
+ sthis.logger.Debug().Any("byteHeads", byteHeads).Msg("No CRDT entries found");
2260
+ return [];
2261
+ }
2262
+ if (!crdtEntries.map) {
2263
+ sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
2264
+ return [];
2265
+ }
2266
+ return Promise.all(
2267
+ crdtEntries.map(async (crdtEntry) => {
2268
+ const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
2269
+ const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
2270
+ return {
2271
+ eventCid: eventBlock.cid,
2272
+ parents: crdtEntry.parents,
2273
+ dbMeta
2274
+ };
2275
+ })
2276
+ );
2277
+ }
2278
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
2279
+ try {
2280
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
2281
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
2282
+ if (keyInfo.length) {
2283
+ const dbMeta = keyInfo[0].dbMeta;
2284
+ if (dbMeta.key) {
2285
+ const kb = await getKeyBag(sthis);
2286
+ const keyName = getStoreKeyName(uri);
2287
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
2288
+ if (res.isErr()) {
2289
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
2290
+ throw res.Err();
2291
+ }
2292
+ }
2293
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
2294
+ return import_cement8.Result.Ok(dbMeta);
2295
+ }
2296
+ sthis.logger.Debug().Any("data", data).Msg("No crypto in gateway meta payload");
2297
+ return import_cement8.Result.Ok(void 0);
2298
+ } catch (error) {
2299
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
2300
+ return import_cement8.Result.Err(error);
2301
+ }
2302
+ }
2303
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
2304
+ try {
2305
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
2306
+ const keyName = getStoreKeyName(uri);
2307
+ const kb = await getKeyBag(sthis);
2308
+ const res = await kb.getNamedExtractableKey(keyName, true);
2309
+ if (res.isErr()) {
2310
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
2311
+ throw res.Err();
2312
+ }
2313
+ const keyData = await res.Ok().extract();
2314
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
2315
+ const { dbMeta, parents } = dbMetas[0];
2316
+ const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
2317
+ dbMeta.key = keyData.keyStr;
2318
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
2319
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
2320
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
2321
+ return import_cement8.Result.Ok(encoded);
2322
+ } catch (error) {
2323
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
2324
+ return import_cement8.Result.Err(error);
2325
+ }
2326
+ }
2327
+ function getStoreKeyName(url) {
2328
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
2329
+ const idx = url.getParam("index");
2330
+ if (idx) {
2331
+ storeKeyName.push(idx);
2332
+ }
2333
+ storeKeyName.push("data");
2334
+ return `@${storeKeyName.join(":")}@`;
2335
+ }
2336
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
2337
+ const event = await import_clock.EventBlock.create(
2338
+ {
2339
+ dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
2340
+ },
2341
+ parents
2342
+ );
2343
+ return event;
2344
+ }
2345
+ async function encodeEventsWithParents(sthis, events, parents) {
2346
+ const crdtEntries = events.map((event) => {
2347
+ const base64String = import_base64.base64pad.encode(event.bytes);
2348
+ return {
2349
+ cid: event.cid.toString(),
2350
+ data: base64String,
2351
+ parents: parents.map((p) => p.toString())
2352
+ };
2353
+ });
2354
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
2355
+ }
2356
+
2357
+ // src/blockstore/store.ts
2358
+ var import_p_retry = __toESM(require("p-retry"), 1);
2359
+ var import_p_map = __toESM(require("p-map"), 1);
2360
+ function guardVersion(url) {
2361
+ if (!url.hasParam("version")) {
2362
+ return import_cement9.Result.Err(`missing version: ${url.toString()}`);
2363
+ }
2364
+ return import_cement9.Result.Ok(url);
2365
+ }
2366
+ var BaseStoreImpl = class {
2367
+ constructor(name, url, opts, sthis, logger) {
2368
+ this._onStarted = [];
2369
+ this._onClosed = [];
2370
+ this.name = name;
2371
+ this._url = url;
2372
+ this.keybag = opts.keybag;
2373
+ this.sthis = sthis;
2374
+ this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
2375
+ this.gateway = new FragmentGateway(this.sthis, opts.gateway);
2376
+ this.loader = opts.loader;
2377
+ }
2378
+ url() {
2379
+ return this._url;
2380
+ }
2381
+ onStarted(fn) {
2382
+ this._onStarted.push(fn);
2383
+ }
2384
+ onClosed(fn) {
2385
+ this._onClosed.push(fn);
2386
+ }
2387
+ async ready() {
2388
+ return;
2389
+ }
2390
+ async keyedCrypto() {
2391
+ return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
2392
+ }
2393
+ async start() {
2394
+ this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
2395
+ this._url = this._url.build().setParam("store", this.storeType).URI();
2396
+ const res = await this.gateway.start(this._url);
2397
+ if (res.isErr()) {
2398
+ this.logger.Error().Result("gw-start", res).Msg("started-gateway");
2399
+ return res;
2400
+ }
2401
+ this._url = res.Ok();
2402
+ const kb = await this.keybag();
2403
+ const skRes = await kb.ensureKeyFromUrl(this._url, () => {
2404
+ const idx = this._url.getParam("index");
2405
+ const storeKeyName = [this.name];
2406
+ if (idx) {
2407
+ storeKeyName.push(idx);
2408
+ }
2409
+ storeKeyName.push(this.storeType);
2410
+ return storeKeyName.join(":");
2411
+ });
2412
+ if (skRes.isErr()) {
2413
+ return skRes;
2414
+ }
2415
+ this._url = skRes.Ok();
2416
+ const version = guardVersion(this._url);
2417
+ if (version.isErr()) {
2418
+ this.logger.Error().Result("version", version).Msg("guardVersion");
2419
+ await this.close();
2420
+ return version;
2421
+ }
2422
+ if (this.ready) {
2423
+ const fn = this.ready.bind(this);
2424
+ const ready = await (0, import_cement9.exception2Result)(fn);
2425
+ if (ready.isErr()) {
2426
+ await this.close();
2427
+ return ready;
2428
+ }
2429
+ }
2430
+ this._onStarted.forEach((fn) => fn());
2431
+ this.logger.Debug().Msg("started");
2432
+ return version;
2433
+ }
2434
+ };
2435
+ var MetaStoreImpl = class extends BaseStoreImpl {
2436
+ // remote: boolean;
2437
+ constructor(sthis, name, url, opts) {
2438
+ super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
2439
+ this.storeType = "meta";
2440
+ this.subscribers = /* @__PURE__ */ new Map();
2441
+ this.parents = [];
2442
+ if (
2443
+ /*this.remote && */
2444
+ opts.gateway.subscribe
2445
+ ) {
2446
+ this.onStarted(async () => {
2447
+ this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
2448
+ opts.gateway.subscribe?.(this.url(), async (message) => {
2449
+ this.logger.Debug().Msg("Received message from gateway");
2450
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
2451
+ await Promise.all(
2452
+ dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
2453
+ );
2454
+ this.updateParentsFromDbMetas(dbMetas);
2455
+ });
2456
+ });
2457
+ }
2458
+ }
2459
+ updateParentsFromDbMetas(dbMetas) {
2460
+ const cids = dbMetas.map((m) => m.eventCid);
2461
+ const dbMetaParents = dbMetas.flatMap((m) => m.parents);
2462
+ const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
2463
+ const dbMetaParentsSet = new Set(dbMetaParents.map((p) => p.toString()));
2464
+ this.parents = Array.from(uniqueParentsMap.values()).filter((p) => !dbMetaParentsSet.has(p.toString()));
2465
+ }
2466
+ async handleByteHeads(byteHeads) {
2467
+ return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
2468
+ }
2469
+ async load() {
2470
+ const branch = "main";
2471
+ const url = await this.gateway.buildUrl(this.url(), branch);
2472
+ if (url.isErr()) {
2473
+ throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
2474
+ }
2475
+ const bytes = await this.gateway.get(url.Ok());
2476
+ if (bytes.isErr()) {
2477
+ if (isNotFoundError(bytes)) {
2478
+ return void 0;
2479
+ }
2480
+ throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
2481
+ }
2482
+ const dbMetas = await this.handleByteHeads(bytes.Ok());
2483
+ await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
2484
+ this.updateParentsFromDbMetas(dbMetas);
2485
+ return dbMetas.map((m) => m.dbMeta);
2486
+ }
2487
+ async save(meta, branch) {
2488
+ branch = branch || "main";
2489
+ this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
2490
+ const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
2491
+ const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
2492
+ const url = await this.gateway.buildUrl(this.url(), branch);
2493
+ if (url.isErr()) {
2494
+ throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
2495
+ }
2496
+ this.parents = [event.cid];
2497
+ const res = await this.gateway.put(url.Ok(), bytes);
2498
+ if (res.isErr()) {
2499
+ throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
2500
+ }
2501
+ return res;
2502
+ }
2503
+ async close() {
2504
+ await this.gateway.close(this.url());
2505
+ this._onClosed.forEach((fn) => fn());
2506
+ return import_cement9.Result.Ok(void 0);
2507
+ }
2508
+ async destroy() {
2509
+ return this.gateway.destroy(this.url());
2510
+ }
2511
+ };
2512
+ var DataStoreImpl = class extends BaseStoreImpl {
2513
+ // readonly tag: string = "car-base";
2514
+ constructor(sthis, name, url, opts) {
2515
+ super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
2516
+ this.storeType = "data";
2517
+ }
2518
+ async load(cid) {
2519
+ this.logger.Debug().Any("cid", cid).Msg("loading");
2520
+ const url = await this.gateway.buildUrl(this.url(), cid.toString());
2521
+ if (url.isErr()) {
2522
+ throw this.logger.Error().Err(url.Err()).Str("cid", cid.toString()).Msg("got error from gateway.buildUrl").AsError();
2523
+ }
2524
+ const res = await this.gateway.get(url.Ok());
2525
+ if (res.isErr()) {
2526
+ throw res.Err();
2527
+ }
2528
+ return { cid, bytes: res.Ok() };
2529
+ }
2530
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2531
+ async save(car, opts) {
2532
+ this.logger.Debug().Any("cid", car.cid.toString()).Msg("saving");
2533
+ const url = await this.gateway.buildUrl(this.url(), car.cid.toString());
2534
+ if (url.isErr()) {
2535
+ throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError();
2536
+ }
2537
+ const res = await this.gateway.put(url.Ok(), car.bytes);
2538
+ if (res.isErr()) {
2539
+ throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
2540
+ }
2541
+ return res.Ok();
2542
+ }
2543
+ async remove(cid) {
2544
+ const url = await this.gateway.buildUrl(this.url(), cid.toString());
2545
+ if (url.isErr()) {
2546
+ return url;
2547
+ }
2548
+ return this.gateway.delete(url.Ok());
2549
+ }
2550
+ async close() {
2551
+ await this.gateway.close(this.url());
2552
+ this._onClosed.forEach((fn) => fn());
2553
+ return import_cement9.Result.Ok(void 0);
2554
+ }
2555
+ destroy() {
2556
+ return this.gateway.destroy(this.url());
2557
+ }
2558
+ };
2559
+ var WALStoreImpl = class extends BaseStoreImpl {
2560
+ constructor(loader, url, opts) {
2561
+ super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
2562
+ this.storeType = "wal";
2563
+ this._ready = new import_cement9.ResolveOnce();
2564
+ this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
2565
+ this.processing = void 0;
2566
+ this.processQueue = new CommitQueue();
2567
+ this.loader = loader;
2568
+ }
2569
+ async ready() {
2570
+ return this._ready.once(async () => {
2571
+ const walState = await this.load().catch((e) => {
2572
+ this.logger.Error().Any("error", e).Msg("error loading wal");
2573
+ return void 0;
2574
+ });
2575
+ if (!walState) {
2576
+ this.walState.operations = [];
2577
+ this.walState.fileOperations = [];
2578
+ } else {
2579
+ this.walState.operations = walState.operations || [];
2580
+ this.walState.fileOperations = walState.fileOperations || [];
2581
+ }
2582
+ });
2583
+ }
2584
+ async enqueue(dbMeta, opts) {
2585
+ await this.ready();
2586
+ if (opts.compact) {
2587
+ this.walState.operations = [];
2588
+ this.walState.noLoaderOps = [dbMeta];
2589
+ } else if (opts.noLoader) {
2590
+ this.walState.noLoaderOps.push(dbMeta);
2591
+ } else {
2592
+ this.walState.operations.push(dbMeta);
2593
+ }
2594
+ await this.save(this.walState);
2595
+ if (!opts.noLoader) {
2596
+ void this.process();
2597
+ }
2598
+ }
2599
+ async enqueueFile(fileCid, publicFile = false) {
2600
+ await this.ready();
2601
+ this.walState.fileOperations.push({ cid: fileCid, public: publicFile });
2602
+ }
2603
+ async process() {
2604
+ await this.ready();
2605
+ if (!this.loader.remoteCarStore) return;
2606
+ await this.processQueue.enqueue(async () => {
2607
+ try {
2608
+ await this._doProcess();
2609
+ } catch (e) {
2610
+ this.logger.Error().Any("error", e).Msg("error processing wal");
2611
+ }
2612
+ if (this.walState.operations.length || this.walState.fileOperations.length || this.walState.noLoaderOps.length) {
2613
+ setTimeout(() => void this.process(), 0);
2614
+ }
2615
+ });
2616
+ }
2617
+ async _doProcess() {
2618
+ if (!this.loader.remoteCarStore) return;
2619
+ const operations = [...this.walState.operations];
2620
+ const noLoaderOps = [...this.walState.noLoaderOps];
2621
+ const fileOperations = [...this.walState.fileOperations];
2622
+ if (operations.length + noLoaderOps.length + fileOperations.length === 0) return;
2623
+ const concurrencyLimit = 3;
2624
+ const retryableUpload = (fn, description) => (0, import_p_retry.default)(fn, {
2625
+ retries: 5,
2626
+ onFailedAttempt: (error) => {
2627
+ this.logger.Warn().Msg(`Attempt ${error.attemptNumber} failed for ${description}. There are ${error.retriesLeft} retries left.`);
2628
+ }
2629
+ });
2630
+ try {
2631
+ await (0, import_p_map.default)(
2632
+ noLoaderOps,
2633
+ async (dbMeta) => {
2634
+ await retryableUpload(async () => {
2635
+ for (const cid of dbMeta.cars) {
2636
+ const car = await (await this.loader.carStore()).load(cid);
2637
+ if (!car) {
2638
+ if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
2639
+ throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
2640
+ }
2641
+ } else {
2642
+ await throwFalsy(this.loader.remoteCarStore).save(car);
2643
+ }
2644
+ }
2645
+ this.walState.noLoaderOps = this.walState.noLoaderOps.filter((op) => op !== dbMeta);
2646
+ }, `noLoaderOp with dbMeta.cars=${dbMeta.cars.toString()}`);
2647
+ },
2648
+ { concurrency: concurrencyLimit }
2649
+ );
2650
+ await (0, import_p_map.default)(
2651
+ operations,
2652
+ async (dbMeta) => {
2653
+ await retryableUpload(async () => {
2654
+ for (const cid of dbMeta.cars) {
2655
+ const car = await (await this.loader.carStore()).load(cid);
2656
+ if (!car) {
2657
+ if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
2658
+ throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
2659
+ }
2660
+ } else {
2661
+ await throwFalsy(this.loader.remoteCarStore).save(car);
2662
+ }
2663
+ }
2664
+ this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
2665
+ }, `operation with dbMeta.cars=${dbMeta.cars.toString()}`);
2666
+ },
2667
+ { concurrency: concurrencyLimit }
2668
+ );
2669
+ await (0, import_p_map.default)(
2670
+ fileOperations,
2671
+ async ({ cid: fileCid, public: publicFile }) => {
2672
+ await retryableUpload(async () => {
2673
+ const fileBlock = await (await this.loader.fileStore()).load(fileCid);
2674
+ if (!fileBlock) {
2675
+ throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
2676
+ }
2677
+ await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
2678
+ this.walState.fileOperations = this.walState.fileOperations.filter((op) => op.cid !== fileCid);
2679
+ }, `fileOperation with cid=${fileCid.toString()}`);
2680
+ },
2681
+ { concurrency: concurrencyLimit }
2682
+ );
2683
+ if (operations.length) {
2684
+ const lastOp = operations[operations.length - 1];
2685
+ await retryableUpload(async () => {
2686
+ await this.loader.remoteMetaStore?.save(lastOp);
2687
+ }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
2688
+ }
2689
+ } catch (error) {
2690
+ this.logger.Error().Any("error", error).Msg("Processing failed");
2691
+ return;
2692
+ } finally {
2693
+ await this.save(this.walState);
2694
+ }
2695
+ }
2696
+ async load() {
2697
+ this.logger.Debug().Msg("loading");
2698
+ const filepath = await this.gateway.buildUrl(this.url(), "main");
2699
+ if (filepath.isErr()) {
2700
+ throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
2701
+ }
2702
+ const bytes = await this.gateway.get(filepath.Ok());
2703
+ if (bytes.isErr()) {
2704
+ if (isNotFoundError(bytes)) {
2705
+ return void 0;
2706
+ }
2707
+ throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
2708
+ }
2709
+ try {
2710
+ return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
2711
+ } catch (e) {
2712
+ throw this.logger.Error().Err(e).Msg("error parse").AsError();
2713
+ }
2714
+ }
2715
+ async save(state) {
2716
+ const filepath = await this.gateway.buildUrl(this.url(), "main");
2717
+ if (filepath.isErr()) {
2718
+ throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
2719
+ }
2720
+ let encoded;
2721
+ try {
2722
+ encoded = (0, import_dag_json2.format)(state);
2723
+ } catch (e) {
2724
+ throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
2725
+ }
2726
+ const res = await this.gateway.put(filepath.Ok(), this.sthis.txt.encode(encoded));
2727
+ if (res.isErr()) {
2728
+ throw this.logger.Error().Err(res.Err()).Str("filePath", filepath.Ok().toString()).Msg("error saving").AsError();
2729
+ }
2730
+ }
2731
+ async close() {
2732
+ await this.gateway.close(this.url());
2733
+ this._onClosed.forEach((fn) => fn());
2734
+ return import_cement9.Result.Ok(void 0);
2735
+ }
2736
+ destroy() {
2737
+ return this.gateway.destroy(this.url());
2738
+ }
2739
+ };
2740
+
2741
+ // src/blockstore/store-factory.ts
2742
+ init_utils();
2743
+ function ensureIsIndex(url, isIndex) {
2744
+ if (isIndex) {
2745
+ return url.build().setParam("index", isIndex).URI();
2746
+ }
2747
+ return url.build().delParam("index").URI();
2748
+ }
2749
+ function ensureName(name, url) {
2750
+ if (!url.hasParam("name")) {
2751
+ return url.build().setParam("name", name).URI();
2752
+ }
2753
+ return url;
2754
+ }
2755
+ var storeFactory = /* @__PURE__ */ new Map();
2756
+ function buildURL(optURL, loader) {
2757
+ const storeOpts = loader.ebOpts.store;
2758
+ const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
2759
+ let obuUrl;
2760
+ if (obuItem && obuItem.overrideBaseURL) {
2761
+ obuUrl = import_cement11.URI.from(obuItem.overrideBaseURL);
2762
+ }
2763
+ const ret = ensureIsIndex(
2764
+ import_cement11.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2765
+ storeOpts.isIndex
2766
+ );
2767
+ return ret;
2768
+ }
2769
+ var onceGateway = new import_cement11.KeyedResolvOnce();
2770
+ async function getGatewayFromURL(url, sthis) {
2771
+ return onceGateway.get(url.toString()).once(async () => {
2772
+ const item = storeFactory.get(url.protocol);
2773
+ if (item) {
2774
+ const ret = {
2775
+ gateway: await item.gateway(sthis),
2776
+ test: await item.test(sthis)
2777
+ };
2778
+ const res = await ret.gateway.start(url);
2779
+ if (res.isErr()) {
2780
+ sthis.logger.Error().Result("start", res).Msg("start failed");
2781
+ return void 0;
2782
+ }
2783
+ return ret;
2784
+ }
2785
+ sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2786
+ return void 0;
2787
+ });
2788
+ }
2789
+ function registerStoreProtocol(item) {
2790
+ let protocol = item.protocol;
2791
+ if (!protocol.endsWith(":")) {
2792
+ protocol += ":";
2793
+ }
2794
+ if (storeFactory.has(protocol)) {
2795
+ if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
2796
+ throw new Error(`we need a logger here`);
2797
+ return () => {
2798
+ };
2799
+ }
2800
+ }
2801
+ if (item.overrideBaseURL) {
2802
+ Array.from(storeFactory.values()).forEach((items) => {
2803
+ items.overrideBaseURL = void 0;
2804
+ });
2805
+ }
2806
+ storeFactory.set(protocol, item);
2807
+ return () => {
2808
+ storeFactory.delete(protocol);
2809
+ };
2810
+ }
2811
+ var onceDataStoreFactory = new import_cement11.KeyedResolvOnce();
2812
+ async function dataStoreFactory(loader) {
2813
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
2814
+ const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
2815
+ return onceDataStoreFactory.get(url.toString()).once(async () => {
2816
+ const gateway = await getGatewayFromURL(url, sthis);
2817
+ if (!gateway) {
2818
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2819
+ }
2820
+ const store = new DataStoreImpl(sthis, loader.name, url, {
2821
+ gateway: gateway.gateway,
2822
+ keybag: () => getKeyBag(loader.sthis, {
2823
+ ...loader.ebOpts.keyBag
2824
+ })
2825
+ });
2826
+ return store;
2827
+ });
2828
+ }
2829
+ var onceMetaStoreFactory = new import_cement11.KeyedResolvOnce();
2830
+ async function metaStoreFactory(loader) {
2831
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
2832
+ const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
2833
+ return onceMetaStoreFactory.get(url.toString()).once(async () => {
2834
+ sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
2835
+ const gateway = await getGatewayFromURL(url, sthis);
2836
+ if (!gateway) {
2837
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2838
+ }
2839
+ const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
2840
+ gateway: gateway.gateway,
2841
+ keybag: () => getKeyBag(loader.sthis, {
2842
+ ...loader.ebOpts.keyBag
2843
+ })
2844
+ });
2845
+ return store;
2846
+ });
2847
+ }
2848
+ var onceRemoteWalFactory = new import_cement11.KeyedResolvOnce();
2849
+ async function remoteWalFactory(loader) {
2850
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
2851
+ const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
2852
+ return onceRemoteWalFactory.get(url.toString()).once(async () => {
2853
+ const gateway = await getGatewayFromURL(url, sthis);
2854
+ if (!gateway) {
2855
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2856
+ }
2857
+ sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
2858
+ const store = new WALStoreImpl(loader, url, {
2859
+ gateway: gateway.gateway,
2860
+ keybag: () => getKeyBag(loader.sthis, {
2861
+ ...loader.ebOpts.keyBag
2862
+ })
2863
+ });
2864
+ return store;
2865
+ });
2866
+ }
2867
+ async function testStoreFactory(url, sthis) {
2868
+ sthis = ensureSuperLog(sthis, "testStoreFactory");
2869
+ const gateway = await getGatewayFromURL(url, sthis);
2870
+ if (!gateway) {
2871
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2872
+ }
2873
+ return gateway.test;
2874
+ }
2875
+ async function ensureStart(store, logger) {
2876
+ const ret = await store.start();
2877
+ if (ret.isErr()) {
2878
+ throw logger.Error().Result("start", ret).Msg("start failed").AsError();
2879
+ }
2880
+ logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2881
+ return store;
2882
+ }
2883
+ function toStoreRuntime(opts, sthis) {
2884
+ const logger = ensureLogger(sthis, "toStoreRuntime", {});
2885
+ return {
2886
+ makeMetaStore: async (loader) => {
2887
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
2888
+ return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
2889
+ },
2890
+ makeDataStore: async (loader) => {
2891
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
2892
+ return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
2893
+ },
2894
+ makeWALStore: async (loader) => {
2895
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
2896
+ return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
2897
+ },
2898
+ encodeFile: opts.encodeFile || encodeFile,
2899
+ decodeFile: opts.decodeFile || decodeFile
2900
+ };
2901
+ }
2902
+ registerStoreProtocol({
2903
+ protocol: "file:",
2904
+ gateway: async (sthis) => {
2905
+ const { FileGateway } = await Promise.resolve().then(() => __toESM(require_bundle_not_impl(), 1));
2906
+ return new FileGateway(sthis);
2907
+ },
2908
+ test: async (sthis) => {
2909
+ const { FileTestStore } = await Promise.resolve().then(() => __toESM(require_bundle_not_impl(), 1));
2910
+ return new FileTestStore(sthis);
2911
+ }
2912
+ });
2913
+ registerStoreProtocol({
2914
+ protocol: "indexdb:",
2915
+ gateway: async (sthis) => {
2916
+ const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway_skip_esm(), gateway_skip_esm_exports));
2917
+ return new IndexDBGateway2(sthis);
2918
+ },
2919
+ test: async (sthis) => {
2920
+ const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway_skip_esm(), gateway_skip_esm_exports));
2921
+ return new IndexDBTestStore2(sthis);
2922
+ }
2923
+ });
2924
+
2925
+ // src/blockstore/store-remote.ts
2926
+ async function RemoteDataStore(sthis, name, url, opts) {
2927
+ const ds = new DataStoreImpl(sthis, name, url, opts);
2928
+ await ds.start();
2929
+ return ds;
2930
+ }
2931
+ async function RemoteMetaStore(sthis, name, url, opts) {
2932
+ const ms = new MetaStoreImpl(
2933
+ sthis,
2934
+ name,
2935
+ url,
2936
+ opts
2937
+ /* , true*/
2938
+ );
2939
+ await ms.start();
2940
+ return ms;
2941
+ }
2942
+
2943
+ // src/blockstore/connection-base.ts
2944
+ var ConnectionBase = class {
2945
+ constructor(url, logger) {
2946
+ this.loaded = Promise.resolve();
2947
+ this.logger = logger;
2948
+ this.url = url;
2949
+ }
2950
+ async refresh() {
2951
+ await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
2952
+ await (await throwFalsy(this.loader).WALStore()).process();
2953
+ }
2954
+ async connect_X({ loader }) {
2955
+ if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
2956
+ await this.connectMeta_X({ loader });
2957
+ await this.connectStorage_X({ loader });
2958
+ }
2959
+ async connectMeta_X({ loader }) {
2960
+ if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
2961
+ this.loader = loader;
2962
+ await this.onConnect();
2963
+ const metaUrl = this.url.build().defParam("store", "meta").URI();
2964
+ const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
2965
+ if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2966
+ const dbName = metaUrl.getParam("name");
2967
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
2968
+ const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
2969
+ gateway: gateway.gateway,
2970
+ keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
2971
+ loader
2972
+ });
2973
+ this.loader.remoteMetaStore = remote;
2974
+ this.loaded = this.loader.ready().then(async () => {
2975
+ remote.load().then(async () => {
2976
+ (await throwFalsy(this.loader).WALStore()).process();
2977
+ });
2978
+ });
2979
+ }
2980
+ async connectStorage_X({ loader }) {
2981
+ if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
2982
+ this.loader = loader;
2983
+ const dataUrl = this.url.build().defParam("store", "data").URI();
2984
+ const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
2985
+ if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2986
+ const name = dataUrl.getParam("name");
2987
+ if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
2988
+ loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
2989
+ gateway: gateway.gateway,
2990
+ keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
2991
+ });
2992
+ loader.remoteFileStore = loader.remoteCarStore;
2993
+ }
2994
+ // move this stuff to connect
2995
+ // async getDashboardURL(compact = true) {
2996
+ // const baseUrl = 'https://dashboard.fireproof.storage/'
2997
+ // if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
2998
+ // // if (compact) {
2999
+ // // await this.compact()
3000
+ // // }
3001
+ // const currents = await this.loader?.metaStore?.load()
3002
+ // if (!currents) throw new Error("Can't sync empty database: save data first")
3003
+ // if (currents.length > 1)
3004
+ // throw new Error("Can't sync database with split heads: make an update first")
3005
+ // const current = currents[0]
3006
+ // const params = {
3007
+ // car: current.car.toString()
3008
+ // }
3009
+ // if (current.key) {
3010
+ // // @ts-ignore
3011
+ // params.key = current.key.toString()
3012
+ // }
3013
+ // // @ts-ignore
3014
+ // if (this.name) {
3015
+ // // @ts-ignore
3016
+ // params.name = this.name
3017
+ // }
3018
+ // const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
3019
+ // console.log('Import to dashboard: ' + url.toString())
3020
+ // return url
3021
+ // }
3022
+ // openDashboard() {
3023
+ // void this.getDashboardURL().then(url => {
3024
+ // if (url) window.open(url.toString(), '_blank')
3025
+ // })
3026
+ // }
3027
+ };
3028
+
3029
+ // src/crdt-helpers.ts
3030
+ function time(tag) {
3031
+ }
3032
+ function timeEnd(tag) {
3033
+ }
3034
+ function toString(key, logger) {
3035
+ switch (typeof key) {
3036
+ case "string":
3037
+ case "number":
3038
+ return key.toString();
3039
+ default:
3040
+ throw logger.Error().Msg("Invalid key type").AsError();
3041
+ }
3042
+ }
3043
+ async function applyBulkUpdateToCrdt(store, tblocks, head, updates, logger) {
3044
+ let result = null;
3045
+ if (updates.length > 1) {
3046
+ const batch = await Batch.create(tblocks, head);
3047
+ for (const update of updates) {
3048
+ const link = await writeDocContent(store, tblocks, update, logger);
3049
+ await batch.put(toString(update.id, logger), link);
3050
+ }
3051
+ result = await batch.commit();
3052
+ } else if (updates.length === 1) {
3053
+ const link = await writeDocContent(store, tblocks, updates[0], logger);
3054
+ result = await (0, import_crdt.put)(tblocks, head, toString(updates[0].id, logger), link);
3055
+ }
3056
+ if (!result) throw logger.Error().Uint64("updates.len", updates.length).Msg("Missing result").AsError();
3057
+ if (result.event) {
3058
+ for (const { cid, bytes } of [
3059
+ ...result.additions,
3060
+ // ...result.removals,
3061
+ result.event
3062
+ ]) {
3063
+ tblocks.putSync(cid, bytes);
3064
+ }
3065
+ }
3066
+ return { head: result.head };
3067
+ }
3068
+ async function writeDocContent(store, blocks, update, logger) {
3069
+ let value;
3070
+ if (update.del) {
3071
+ value = { del: true };
3072
+ } else {
3073
+ if (!update.value) throw logger.Error().Msg("Missing value").AsError();
3074
+ await processFiles(store, blocks, update.value, logger);
3075
+ value = { doc: update.value };
3076
+ }
3077
+ const block = await encode({ value, hasher: import_sha25.sha256, codec });
3078
+ blocks.putSync(block.cid, block.bytes);
3079
+ return block.cid;
3080
+ }
3081
+ async function processFiles(store, blocks, doc, logger) {
3082
+ if (doc._files) {
3083
+ await processFileset(logger, store, blocks, doc._files);
3084
+ }
3085
+ if (doc._publicFiles) {
3086
+ await processFileset(
3087
+ logger,
3088
+ store,
3089
+ blocks,
3090
+ doc._publicFiles
3091
+ /*, true*/
3092
+ );
3093
+ }
3094
+ }
3095
+ async function processFileset(logger, store, blocks, files) {
3096
+ const dbBlockstore = blocks.parent;
3097
+ if (!dbBlockstore.loader) throw logger.Error().Msg("Missing loader, database name is required").AsError();
3098
+ const t = new CarTransaction(dbBlockstore);
3099
+ const didPut = [];
3100
+ for (const filename in files) {
3101
+ if (File === files[filename].constructor) {
3102
+ const file = files[filename];
3103
+ const { cid, blocks: fileBlocks } = await store.encodeFile(file);
3104
+ didPut.push(filename);
3105
+ for (const block of fileBlocks) {
3106
+ t.putSync(block.cid, block.bytes);
3107
+ }
3108
+ files[filename] = { cid, type: file.type, size: file.size };
3109
+ } else {
3110
+ const { cid, type, size, car } = files[filename];
3111
+ if (cid && type && size && car) {
3112
+ files[filename] = { cid, type, size, car };
3113
+ }
3114
+ }
3115
+ }
3116
+ if (didPut.length) {
3117
+ const car = await dbBlockstore.loader.commitFiles(
3118
+ t,
3119
+ { files }
3120
+ );
3121
+ if (car) {
3122
+ for (const name of didPut) {
3123
+ files[name] = { car, ...files[name] };
3124
+ }
3125
+ }
3126
+ }
3127
+ }
3128
+ async function getValueFromCrdt(blocks, head, key, logger) {
3129
+ if (!head.length) throw logger.Debug().Msg("Getting from an empty database").AsError();
3130
+ const link = await (0, import_crdt.get)(blocks, head, key);
3131
+ if (!link) throw logger.Error().Str("key", key).Msg(`Missing key`).AsError();
3132
+ return await getValueFromLink(blocks, link, logger);
3133
+ }
3134
+ function readFiles(blocks, { doc }) {
3135
+ if (!doc) return;
3136
+ if (doc._files) {
3137
+ readFileset(blocks, doc._files);
3138
+ }
3139
+ if (doc._publicFiles) {
3140
+ readFileset(blocks, doc._publicFiles, true);
3141
+ }
3142
+ }
3143
+ function readFileset(blocks, files, isPublic = false) {
3144
+ for (const filename in files) {
3145
+ const fileMeta = files[filename];
3146
+ if (fileMeta.cid) {
3147
+ if (isPublic) {
3148
+ fileMeta.url = `https://${fileMeta.cid.toString()}.ipfs.w3s.link/`;
3149
+ }
3150
+ if (fileMeta.car) {
3151
+ fileMeta.file = async () => await blocks.ebOpts.storeRuntime.decodeFile(
3152
+ {
3153
+ get: async (cid) => {
3154
+ return await blocks.getFile(throwFalsy(fileMeta.car), cid);
3155
+ }
3156
+ },
3157
+ fileMeta.cid,
3158
+ fileMeta
3159
+ );
3160
+ }
3161
+ }
3162
+ files[filename] = fileMeta;
3163
+ }
3164
+ }
3165
+ async function getValueFromLink(blocks, link, logger) {
3166
+ const block = await blocks.get(link);
3167
+ if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
3168
+ const { value } = await decode({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
3169
+ const cvalue = {
3170
+ ...value,
3171
+ cid: link
3172
+ };
3173
+ readFiles(blocks, cvalue);
3174
+ return cvalue;
3175
+ }
3176
+ var DirtyEventFetcher = class extends import_clock2.EventFetcher {
3177
+ constructor(logger, blocks) {
3178
+ super(blocks);
3179
+ this.logger = logger;
3180
+ }
3181
+ async get(link) {
3182
+ try {
3183
+ return super.get(link);
3184
+ } catch (e) {
3185
+ this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
3186
+ return { value: void 0 };
3187
+ }
3188
+ }
3189
+ };
3190
+ async function clockChangesSince(blocks, head, since, opts, logger) {
3191
+ const eventsFetcher = opts.dirty ? new DirtyEventFetcher(logger, blocks) : new import_clock2.EventFetcher(blocks);
3192
+ const keys = /* @__PURE__ */ new Set();
3193
+ const updates = await gatherUpdates(
3194
+ blocks,
3195
+ eventsFetcher,
3196
+ head,
3197
+ since,
3198
+ [],
3199
+ keys,
3200
+ /* @__PURE__ */ new Set(),
3201
+ opts.limit || Infinity,
3202
+ logger
3203
+ );
3204
+ return { result: updates.reverse(), head };
3205
+ }
3206
+ async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], keys, didLinks, limit, logger) {
3207
+ if (limit <= 0) return updates;
3208
+ const sHead = head.map((l) => l.toString());
3209
+ for (const link of since) {
3210
+ if (sHead.includes(link.toString())) {
3211
+ return updates;
3212
+ }
3213
+ }
3214
+ for (const link of head) {
3215
+ if (didLinks.has(link.toString())) continue;
3216
+ didLinks.add(link.toString());
3217
+ const { value: event } = await eventsFetcher.get(link);
3218
+ if (!event) continue;
3219
+ const { type } = event.data;
3220
+ let ops = [];
3221
+ if (type === "batch") {
3222
+ ops = event.data.ops;
3223
+ } else if (type === "put") {
3224
+ ops = [event.data];
3225
+ }
3226
+ for (let i = ops.length - 1; i >= 0; i--) {
3227
+ const { key, value } = ops[i];
3228
+ if (!keys.has(key)) {
3229
+ const docValue = await getValueFromLink(blocks, value, logger);
3230
+ updates.push({ id: key, value: docValue.doc, del: docValue.del, clock: link });
3231
+ limit--;
3232
+ keys.add(key);
3233
+ }
3234
+ }
3235
+ if (event.parents) {
3236
+ updates = await gatherUpdates(blocks, eventsFetcher, event.parents, since, updates, keys, didLinks, limit, logger);
3237
+ }
3238
+ }
3239
+ return updates;
3240
+ }
3241
+ async function* getAllEntries(blocks, head, logger) {
3242
+ for await (const [key, link] of (0, import_crdt.entries)(blocks, head)) {
3243
+ const docValue = await getValueFromLink(blocks, link, logger);
3244
+ yield { id: key, value: docValue.doc, del: docValue.del };
3245
+ }
3246
+ }
3247
+ async function* clockVis(blocks, head) {
3248
+ for await (const line of (0, import_clock2.vis)(blocks, head)) {
3249
+ yield line;
3250
+ }
3251
+ }
3252
+ var isCompacting = false;
3253
+ async function doCompact(blockLog, head, logger) {
3254
+ if (isCompacting) {
3255
+ return;
3256
+ }
3257
+ isCompacting = true;
3258
+ time("compact head");
3259
+ for (const cid of head) {
3260
+ const bl = await blockLog.get(cid);
3261
+ if (!bl) throw logger.Error().Ref("cid", cid).Msg("Missing head block").AsError();
3262
+ }
3263
+ timeEnd("compact head");
3264
+ time("compact all entries");
3265
+ for await (const _entry of getAllEntries(blockLog, head, logger)) {
3266
+ continue;
3267
+ }
3268
+ timeEnd("compact all entries");
3269
+ time("compact clock vis");
3270
+ for await (const _line of (0, import_clock2.vis)(blockLog, head)) {
3271
+ }
3272
+ timeEnd("compact clock vis");
3273
+ time("compact root");
3274
+ const result = await (0, import_crdt.root)(blockLog, head);
3275
+ timeEnd("compact root");
3276
+ time("compact root blocks");
3277
+ for (const { cid, bytes } of [...result.additions, ...result.removals]) {
3278
+ blockLog.loggedBlocks.putSync(cid, bytes);
3279
+ }
3280
+ timeEnd("compact root blocks");
3281
+ time("compact changes");
3282
+ await clockChangesSince(blockLog, head, [], {}, logger);
3283
+ timeEnd("compact changes");
3284
+ isCompacting = false;
3285
+ }
3286
+ async function getBlock(blocks, cidString) {
3287
+ const block = await blocks.get((0, import_link.parse)(cidString));
3288
+ if (!block) throw new Error(`Missing block ${cidString}`);
3289
+ const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
3290
+ return new Block({ cid, value, bytes: block.bytes });
3291
+ }
3292
+
3293
+ // src/indexer-helpers.ts
3294
+ var import_sha26 = require("multiformats/hashes/sha2");
3295
+ var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
3296
+ var import_charwise = __toESM(require("charwise"), 1);
3297
+ var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
3298
+ var import_utils12 = require("prolly-trees/utils");
3299
+ var import_cache = require("prolly-trees/cache");
3300
+ var IndexTree = class {
3301
+ };
3302
+ function refCompare(aRef, bRef) {
3303
+ if (Number.isNaN(aRef)) return -1;
3304
+ if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
3305
+ if (aRef === Infinity) return 1;
3306
+ return (0, import_utils12.simpleCompare)(aRef, bRef);
3307
+ }
3308
+ function compare(a, b) {
3309
+ const [aKey, aRef] = a;
3310
+ const [bKey, bRef] = b;
3311
+ const comp = (0, import_utils12.simpleCompare)(aKey, bKey);
3312
+ if (comp !== 0) return comp;
3313
+ return refCompare(aRef, bRef);
3314
+ }
3315
+ var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils12.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
3316
+ var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils12.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils12.simpleCompare };
3317
+ function indexEntriesForChanges(changes, mapFn) {
3318
+ const indexEntries = [];
3319
+ changes.forEach(({ id: key, value, del }) => {
3320
+ if (del || !value) return;
3321
+ let mapCalled = false;
3322
+ const mapReturn = mapFn({ ...value, _id: key }, (k, v) => {
3323
+ mapCalled = true;
3324
+ if (typeof k === "undefined") return;
3325
+ indexEntries.push({
3326
+ key: [import_charwise.default.encode(k), key],
3327
+ value: v || null
3328
+ });
3329
+ });
3330
+ if (!mapCalled && mapReturn) {
3331
+ indexEntries.push({
3332
+ key: [import_charwise.default.encode(mapReturn), key],
3333
+ value: null
3334
+ });
3335
+ }
3336
+ });
3337
+ return indexEntries;
3338
+ }
3339
+ function makeProllyGetBlock(blocks) {
3340
+ return async (address) => {
3341
+ const block = await blocks.get(address);
3342
+ if (!block) throw new Error(`Missing block ${address.toString()}`);
3343
+ const { cid, bytes } = block;
3344
+ return create({ cid, bytes, hasher: import_sha26.sha256, codec: codec2 });
3345
+ };
3346
+ }
3347
+ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
3348
+ if (!indexEntries.length) return inIndex;
3349
+ if (!inIndex.root) {
3350
+ if (!inIndex.cid) {
3351
+ let returnRootBlock = void 0;
3352
+ let returnNode = void 0;
3353
+ for await (const node of await DbIndex.create({
3354
+ get: makeProllyGetBlock(tblocks),
3355
+ list: indexEntries,
3356
+ ...opts
3357
+ })) {
3358
+ const block = await node.block;
3359
+ await tblocks.put(block.cid, block.bytes);
3360
+ returnRootBlock = block;
3361
+ returnNode = node;
3362
+ }
3363
+ if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
3364
+ return { root: returnNode, cid: returnRootBlock.cid };
3365
+ } else {
3366
+ inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
3367
+ }
3368
+ }
3369
+ const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
3370
+ if (root3) {
3371
+ for await (const block of newBlocks) {
3372
+ await tblocks.put(block.cid, block.bytes);
3373
+ }
3374
+ return { root: root3, cid: (await root3.block).cid };
3375
+ } else {
3376
+ return { root: void 0, cid: void 0 };
3377
+ }
3378
+ }
3379
+ async function loadIndex(tblocks, cid, opts) {
3380
+ return await DbIndex.load({ cid, get: makeProllyGetBlock(tblocks), ...opts });
3381
+ }
3382
+ async function applyQuery(crdt, resp, query) {
3383
+ if (query.descending) {
3384
+ resp.result = resp.result.reverse();
3385
+ }
3386
+ if (query.limit) {
3387
+ resp.result = resp.result.slice(0, query.limit);
3388
+ }
3389
+ if (query.includeDocs) {
3390
+ resp.result = await Promise.all(
3391
+ resp.result.map(async (row) => {
3392
+ const val = await crdt.get(row.id);
3393
+ const doc = val ? { ...val.doc, _id: row.id } : void 0;
3394
+ return { ...row, doc };
3395
+ })
3396
+ );
3397
+ }
3398
+ return {
3399
+ rows: resp.result.map(({ key, ...row }) => {
3400
+ return {
3401
+ key: import_charwise.default.decode(key),
3402
+ ...row
3403
+ };
3404
+ })
3405
+ };
3406
+ }
3407
+ function encodeRange(range) {
3408
+ return [import_charwise.default.encode(range[0]), import_charwise.default.encode(range[1])];
3409
+ }
3410
+ function encodeKey(key) {
3411
+ return import_charwise.default.encode(key);
3412
+ }
3413
+
3414
+ // src/indexer.ts
3415
+ init_utils();
3416
+ function index(sthis, { _crdt }, name, mapFn, meta) {
3417
+ if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3418
+ if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
3419
+ if (_crdt.indexers.has(name)) {
3420
+ const idx = _crdt.indexers.get(name);
3421
+ idx.applyMapFn(name, mapFn, meta);
3422
+ } else {
3423
+ const idx = new Index(sthis, _crdt, name, mapFn, meta);
3424
+ _crdt.indexers.set(name, idx);
3425
+ }
3426
+ return _crdt.indexers.get(name);
3427
+ }
3428
+ var Index = class {
3429
+ constructor(sthis, crdt, name, mapFn, meta) {
3430
+ this.mapFnString = "";
3431
+ this.byKey = new IndexTree();
3432
+ this.byId = new IndexTree();
3433
+ this.includeDocsDefault = false;
3434
+ this.logger = ensureLogger(sthis, "Index");
3435
+ this.blockstore = crdt.indexBlockstore;
3436
+ this.crdt = crdt;
3437
+ this.applyMapFn(name, mapFn, meta);
3438
+ this.name = name;
3439
+ if (!(this.mapFnString || this.initError)) throw this.logger.Error().Msg("missing mapFnString").AsError();
3440
+ }
3441
+ ready() {
3442
+ return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
3443
+ });
3444
+ }
3445
+ close() {
3446
+ return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
3447
+ });
3448
+ }
3449
+ destroy() {
3450
+ return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
3451
+ });
3452
+ }
3453
+ applyMapFn(name, mapFn, meta) {
3454
+ if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
3455
+ if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
3456
+ this.name = name;
3457
+ try {
3458
+ if (meta) {
3459
+ if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
3460
+ throw this.logger.Error().Msg("cannot apply different head meta").AsError();
3461
+ }
3462
+ if (this.mapFnString) {
3463
+ if (this.mapFnString !== meta.map) {
3464
+ this.logger.Warn().Msg(`cannot apply different mapFn meta: old mapFnString ${this.mapFnString} new mapFnString ${meta.map}`);
3465
+ } else {
3466
+ this.byId.cid = meta.byId;
3467
+ this.byKey.cid = meta.byKey;
3468
+ this.indexHead = meta.head;
3469
+ }
3470
+ } else {
3471
+ this.mapFnString = meta.map;
3472
+ this.byId.cid = meta.byId;
3473
+ this.byKey.cid = meta.byKey;
3474
+ this.indexHead = meta.head;
3475
+ }
3476
+ } else {
3477
+ if (this.mapFn) {
3478
+ if (mapFn) {
3479
+ if (this.mapFn.toString() !== mapFn.toString()) {
3480
+ throw this.logger.Error().Msg("cannot apply different mapFn app2").AsError();
3481
+ }
3482
+ }
3483
+ } else {
3484
+ if (!mapFn) {
3485
+ mapFn = (doc) => doc[name] ?? void 0;
3486
+ }
3487
+ if (this.mapFnString) {
3488
+ if (this.mapFnString !== mapFn.toString()) {
3489
+ throw this.logger.Error().Str("mapFnString", this.mapFnString).Str("mapFn", mapFn.toString()).Msg("cannot apply different mapFn app").AsError();
3490
+ }
3491
+ } else {
3492
+ this.mapFnString = mapFn.toString();
3493
+ }
3494
+ this.mapFn = mapFn;
3495
+ }
3496
+ }
3497
+ const matches = /=>\s*(.*)/.test(this.mapFnString);
3498
+ this.includeDocsDefault = matches;
3499
+ } catch (e) {
3500
+ this.initError = e;
3501
+ }
3502
+ }
3503
+ async query(opts = {}) {
3504
+ await this.ready();
3505
+ await this._updateIndex();
3506
+ await this._hydrateIndex();
3507
+ if (!this.byKey.root) {
3508
+ return await applyQuery(this.crdt, { result: [] }, opts);
3509
+ }
3510
+ if (this.includeDocsDefault && opts.includeDocs === void 0) opts.includeDocs = true;
3511
+ if (opts.range) {
3512
+ const eRange = encodeRange(opts.range);
3513
+ return await applyQuery(this.crdt, await throwFalsy(this.byKey.root).range(eRange[0], eRange[1]), opts);
3514
+ }
3515
+ if (opts.key) {
3516
+ const encodedKey = encodeKey(opts.key);
3517
+ return await applyQuery(this.crdt, await throwFalsy(this.byKey.root).get(encodedKey), opts);
3518
+ }
3519
+ if (Array.isArray(opts.keys)) {
3520
+ const results = await Promise.all(
3521
+ opts.keys.map(async (key) => {
3522
+ const encodedKey = encodeKey(key);
3523
+ return (await applyQuery(this.crdt, await throwFalsy(this.byKey.root).get(encodedKey), opts)).rows;
3524
+ })
3525
+ );
3526
+ return { rows: results.flat() };
3527
+ }
3528
+ if (opts.prefix) {
3529
+ if (!Array.isArray(opts.prefix)) opts.prefix = [opts.prefix];
3530
+ const start = [...opts.prefix, NaN];
3531
+ const end = [...opts.prefix, Infinity];
3532
+ const encodedR = encodeRange([start, end]);
3533
+ return await applyQuery(this.crdt, await this.byKey.root.range(...encodedR), opts);
3534
+ }
3535
+ const all = await this.byKey.root.getAllEntries();
3536
+ return await applyQuery(
3537
+ this.crdt,
3538
+ {
3539
+ // @ts-expect-error getAllEntries returns a different type than range
3540
+ result: all.result.map(({ key: [k, id], value }) => ({
3541
+ key: k,
3542
+ id,
3543
+ value
3544
+ }))
3545
+ },
3546
+ opts
3547
+ );
3548
+ }
3549
+ _resetIndex() {
3550
+ this.byId = new IndexTree();
3551
+ this.byKey = new IndexTree();
3552
+ this.indexHead = void 0;
3553
+ }
3554
+ async _hydrateIndex() {
3555
+ if (this.byId.root && this.byKey.root) return;
3556
+ if (!this.byId.cid || !this.byKey.cid) return;
3557
+ this.byId.root = await loadIndex(this.blockstore, this.byId.cid, byIdOpts);
3558
+ this.byKey.root = await loadIndex(this.blockstore, this.byKey.cid, byKeyOpts);
3559
+ }
3560
+ async _updateIndex() {
3561
+ await this.ready();
3562
+ if (this.initError) throw this.initError;
3563
+ if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
3564
+ let result, head;
3565
+ if (!this.indexHead || this.indexHead.length === 0) {
3566
+ ({ result, head } = await this.crdt.allDocs());
3567
+ } else {
3568
+ ({ result, head } = await this.crdt.changes(this.indexHead));
3569
+ }
3570
+ if (result.length === 0) {
3571
+ this.indexHead = head;
3572
+ }
3573
+ let staleKeyIndexEntries = [];
3574
+ let removeIdIndexEntries = [];
3575
+ if (this.byId.root) {
3576
+ const removeIds = result.map(({ id: key }) => key);
3577
+ const { result: oldChangeEntries } = await this.byId.root.getMany(removeIds);
3578
+ staleKeyIndexEntries = oldChangeEntries.map((key) => ({ key, del: true }));
3579
+ removeIdIndexEntries = oldChangeEntries.map((key) => ({ key: key[1], del: true }));
3580
+ }
3581
+ const indexEntries = indexEntriesForChanges(result, this.mapFn);
3582
+ const byIdIndexEntries = indexEntries.map(({ key }) => ({
3583
+ key: key[1],
3584
+ value: key
3585
+ }));
3586
+ const indexerMeta = { indexes: /* @__PURE__ */ new Map() };
3587
+ for (const [name, indexer] of this.crdt.indexers) {
3588
+ if (indexer.indexHead) {
3589
+ indexerMeta.indexes?.set(name, {
3590
+ byId: indexer.byId.cid,
3591
+ byKey: indexer.byKey.cid,
3592
+ head: indexer.indexHead,
3593
+ map: indexer.mapFnString,
3594
+ name: indexer.name
3595
+ });
3596
+ }
3597
+ }
3598
+ if (result.length === 0) {
3599
+ return indexerMeta;
3600
+ }
3601
+ const { meta } = await this.blockstore.transaction(async (tblocks) => {
3602
+ this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
3603
+ this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
3604
+ this.indexHead = head;
3605
+ if (this.byId.cid && this.byKey.cid) {
3606
+ const idxMeta = {
3607
+ byId: this.byId.cid,
3608
+ byKey: this.byKey.cid,
3609
+ head,
3610
+ map: this.mapFnString,
3611
+ name: this.name
3612
+ };
3613
+ indexerMeta.indexes?.set(this.name, idxMeta);
3614
+ }
3615
+ return indexerMeta;
3616
+ });
3617
+ return meta;
3618
+ }
3619
+ };
3620
+
3621
+ // src/crdt-clock.ts
3622
+ var import_clock3 = require("@web3-storage/pail/clock");
3623
+ var import_crdt2 = require("@web3-storage/pail/crdt");
3624
+ var import_cement12 = require("@adviser/cement");
3625
+
3626
+ // src/apply-head-queue.ts
3627
+ function applyHeadQueue(worker, logger) {
3628
+ const queue = [];
3629
+ let isProcessing = false;
3630
+ async function* process() {
3631
+ if (isProcessing || queue.length === 0) return;
3632
+ isProcessing = true;
3633
+ const allUpdates = [];
3634
+ try {
3635
+ while (queue.length > 0) {
3636
+ queue.sort((a, b) => b.updates ? 1 : -1);
3637
+ const task = queue.shift();
3638
+ if (!task) continue;
3639
+ await worker(task.newHead, task.prevHead, task.updates !== void 0).catch((e) => {
3640
+ throw logger.Error().Err(e).Msg("int_applyHead worker error").AsError();
3641
+ });
3642
+ if (task.updates) {
3643
+ allUpdates.push(...task.updates);
3644
+ }
3645
+ if (!queue.some((t) => t.updates) || task.updates) {
3646
+ const allTasksHaveUpdates = queue.every((task2) => task2.updates !== null);
3647
+ yield { updates: allUpdates, all: allTasksHaveUpdates };
3648
+ allUpdates.length = 0;
3649
+ }
3650
+ }
3651
+ } finally {
3652
+ isProcessing = false;
3653
+ const generator = process();
3654
+ let result = await generator.next();
3655
+ while (!result.done) {
3656
+ result = await generator.next();
3657
+ }
3658
+ }
3659
+ }
3660
+ return {
3661
+ push(task) {
3662
+ queue.push(task);
3663
+ return process();
3664
+ },
3665
+ size() {
3666
+ return queue.length;
3667
+ }
3668
+ };
3669
+ }
3670
+
3671
+ // src/crdt-clock.ts
3672
+ init_utils();
3673
+ var CRDTClock = class {
3674
+ constructor(blockstore) {
3675
+ // todo: track local and remote clocks independently, merge on read
3676
+ // that way we can drop the whole remote if we need to
3677
+ // should go with making sure the local clock only references locally available blockstore on write
3678
+ this.head = [];
3679
+ this.zoomers = /* @__PURE__ */ new Set();
3680
+ this.watchers = /* @__PURE__ */ new Set();
3681
+ this.emptyWatchers = /* @__PURE__ */ new Set();
3682
+ this._ready = new import_cement12.ResolveOnce();
3683
+ this.blockstore = blockstore;
3684
+ this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
3685
+ this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
3686
+ }
3687
+ async ready() {
3688
+ return this._ready.once(async () => {
3689
+ await this.blockstore.ready();
3690
+ });
3691
+ }
3692
+ async close() {
3693
+ await this.blockstore.close();
3694
+ }
3695
+ setHead(head) {
3696
+ this.head = head;
3697
+ }
3698
+ async applyHead(newHead, prevHead, updates) {
3699
+ for await (const { updates: updatesAcc, all } of this.applyHeadQueue.push({
3700
+ newHead,
3701
+ prevHead,
3702
+ updates
3703
+ })) {
3704
+ return this.processUpdates(updatesAcc, all, prevHead);
3705
+ }
3706
+ }
3707
+ async processUpdates(updatesAcc, all, prevHead) {
3708
+ let internalUpdates = updatesAcc;
3709
+ if (this.watchers.size && !all) {
3710
+ const changes = await clockChangesSince(throwFalsy(this.blockstore), this.head, prevHead, {}, this.logger);
3711
+ internalUpdates = changes.result;
3712
+ }
3713
+ this.zoomers.forEach((fn) => fn());
3714
+ this.notifyWatchers(internalUpdates || []);
3715
+ }
3716
+ notifyWatchers(updates) {
3717
+ this.emptyWatchers.forEach((fn) => fn());
3718
+ this.watchers.forEach((fn) => fn(updates || []));
3719
+ }
3720
+ onTick(fn) {
3721
+ this.watchers.add(fn);
3722
+ }
3723
+ onTock(fn) {
3724
+ this.emptyWatchers.add(fn);
3725
+ }
3726
+ onZoom(fn) {
3727
+ this.zoomers.add(fn);
3728
+ }
3729
+ async int_applyHead(newHead, prevHead, localUpdates) {
3730
+ const noLoader = !localUpdates;
3731
+ const ogHead = sortClockHead(this.head);
3732
+ newHead = sortClockHead(newHead);
3733
+ if (compareClockHeads(ogHead, newHead)) {
3734
+ return;
3735
+ }
3736
+ const ogPrev = sortClockHead(prevHead);
3737
+ if (compareClockHeads(ogHead, ogPrev)) {
3738
+ this.setHead(newHead);
3739
+ return;
3740
+ }
3741
+ if (!this.blockstore) {
3742
+ throw this.logger.Error().Msg("missing blockstore").AsError();
3743
+ }
3744
+ await validateBlocks(this.logger, newHead, this.blockstore);
3745
+ if (!this.transaction) {
3746
+ this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
3747
+ }
3748
+ const tblocks = this.transaction;
3749
+ const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3750
+ const result = await (0, import_crdt2.root)(tblocks, advancedHead);
3751
+ for (const { cid, bytes } of [
3752
+ ...result.additions
3753
+ // ...result.removals
3754
+ ]) {
3755
+ tblocks.putSync(cid, bytes);
3756
+ }
3757
+ if (!noLoader) {
3758
+ await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
3759
+ this.transaction = void 0;
3760
+ }
3761
+ this.setHead(advancedHead);
3762
+ }
3763
+ };
3764
+ function sortClockHead(clockHead) {
3765
+ return clockHead.sort((a, b) => a.toString().localeCompare(b.toString()));
3766
+ }
3767
+ async function validateBlocks(logger, newHead, blockstore) {
3768
+ if (!blockstore) throw logger.Error().Msg("missing blockstore");
3769
+ newHead.map(async (cid) => {
3770
+ const got = await blockstore.get(cid);
3771
+ if (!got) {
3772
+ throw logger.Error().Str("cid", cid.toString()).Msg("int_applyHead missing block").AsError();
3773
+ }
3774
+ });
3775
+ }
3776
+ function compareClockHeads(head1, head2) {
3777
+ return head1.toString() === head2.toString();
3778
+ }
3779
+ async function advanceBlocks(logger, newHead, tblocks, head) {
3780
+ for (const cid of newHead) {
3781
+ try {
3782
+ head = await (0, import_clock3.advance)(tblocks, head, cid);
3783
+ } catch (e) {
3784
+ logger.Debug().Err(e).Msg("failed to advance head");
3785
+ continue;
3786
+ }
3787
+ }
3788
+ return head;
3789
+ }
3790
+
3791
+ // src/crdt.ts
3792
+ init_utils();
3793
+ var CRDT = class {
3794
+ constructor(sthis, name, opts = {}) {
3795
+ this.indexers = /* @__PURE__ */ new Map();
3796
+ this.onceReady = new import_cement13.ResolveOnce();
3797
+ this.sthis = sthis;
3798
+ this.name = name;
3799
+ this.logger = ensureLogger(sthis, "CRDT");
3800
+ this.opts = opts;
3801
+ this.blockstore = blockstoreFactory(sthis, {
3802
+ name,
3803
+ applyMeta: async (meta) => {
3804
+ const crdtMeta = meta;
3805
+ if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
3806
+ await this.clock.applyHead(crdtMeta.head, []);
3807
+ },
3808
+ compact: async (blocks) => {
3809
+ await doCompact(blocks, this.clock.head, this.logger);
3810
+ return { head: this.clock.head };
3811
+ },
3812
+ autoCompact: this.opts.autoCompact || 100,
3813
+ store: { ...this.opts.store, isIndex: void 0 },
3814
+ public: this.opts.public,
3815
+ meta: this.opts.meta,
3816
+ threshold: this.opts.threshold
3817
+ });
3818
+ this.indexBlockstore = blockstoreFactory(sthis, {
3819
+ name,
3820
+ applyMeta: async (meta) => {
3821
+ const idxCarMeta = meta;
3822
+ if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
3823
+ for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
3824
+ index(this.sthis, { _crdt: this }, name2, void 0, idx);
3825
+ }
3826
+ },
3827
+ store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
3828
+ public: this.opts.public
3829
+ });
3830
+ this.clock = new CRDTClock(this.blockstore);
3831
+ this.clock.onZoom(() => {
3832
+ for (const idx of this.indexers.values()) {
3833
+ idx._resetIndex();
3834
+ }
3835
+ });
3836
+ }
3837
+ async bulk(updates) {
3838
+ await this.ready();
3839
+ const prevHead = [...this.clock.head];
3840
+ const done = await this.blockstore.transaction(async (blocks) => {
3841
+ const { head } = await applyBulkUpdateToCrdt(
3842
+ this.blockstore.ebOpts.storeRuntime,
3843
+ blocks,
3844
+ this.clock.head,
3845
+ updates,
3846
+ this.logger
3847
+ );
3848
+ updates = updates.map((dupdate) => {
3849
+ readFiles(this.blockstore, { doc: dupdate.value });
3850
+ return dupdate;
3851
+ });
3852
+ return { head };
3853
+ });
3854
+ await this.clock.applyHead(done.meta.head, prevHead, updates);
3855
+ return done.meta;
3856
+ }
3857
+ async ready() {
3858
+ return this.onceReady.once(async () => {
3859
+ try {
3860
+ await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
3861
+ } catch (e) {
3862
+ throw this.logger.Error().Err(e).Msg(`CRDT is not ready`).AsError();
3863
+ }
3864
+ });
3865
+ }
3866
+ async close() {
3867
+ await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
3868
+ }
3869
+ async destroy() {
3870
+ await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
3871
+ }
3872
+ // if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
3873
+ async allDocs() {
3874
+ await this.ready();
3875
+ const result = [];
3876
+ for await (const entry of getAllEntries(this.blockstore, this.clock.head, this.logger)) {
3877
+ result.push(entry);
3878
+ }
3879
+ return { result, head: this.clock.head };
3880
+ }
3881
+ async vis() {
3882
+ await this.ready();
3883
+ const txt = [];
3884
+ for await (const line of clockVis(this.blockstore, this.clock.head)) {
3885
+ txt.push(line);
3886
+ }
3887
+ return txt.join("\n");
3888
+ }
3889
+ async getBlock(cidString) {
3890
+ await this.ready();
3891
+ return await getBlock(this.blockstore, cidString);
3892
+ }
3893
+ async get(key) {
3894
+ await this.ready();
3895
+ const result = await getValueFromCrdt(this.blockstore, this.clock.head, key, this.logger);
3896
+ if (result.del) return void 0;
3897
+ return result;
3898
+ }
3899
+ async changes(since = [], opts = {}) {
3900
+ await this.ready();
3901
+ return await clockChangesSince(this.blockstore, this.clock.head, since, opts, this.logger);
3902
+ }
3903
+ async compact() {
3904
+ const blocks = this.blockstore;
3905
+ return await blocks.compact();
3906
+ }
3907
+ };
3908
+
3909
+ // src/database.ts
3910
+ init_utils();
3911
+ var Database = class {
3912
+ constructor(name, opts) {
3913
+ this.opts = {};
3914
+ this._listening = false;
3915
+ this._listeners = /* @__PURE__ */ new Set();
3916
+ this._noupdate_listeners = /* @__PURE__ */ new Set();
3917
+ this._ready = new import_cement14.ResolveOnce();
3918
+ this.name = name;
3919
+ this.opts = opts || this.opts;
3920
+ this.sthis = ensureSuperThis(this.opts);
3921
+ this.logger = ensureLogger(this.sthis, "Database");
3922
+ this._crdt = new CRDT(this.sthis, name, this.opts);
3923
+ this.blockstore = this._crdt.blockstore;
3924
+ this._writeQueue = writeQueue(async (updates) => {
3925
+ return await this._crdt.bulk(updates);
3926
+ });
3927
+ this._crdt.clock.onTock(() => {
3928
+ this._no_update_notify();
3929
+ });
3930
+ }
3931
+ static {
3932
+ this.databases = /* @__PURE__ */ new Map();
3933
+ }
3934
+ async close() {
3935
+ await this.ready();
3936
+ await this._crdt.close();
3937
+ await this.blockstore.close();
3938
+ }
3939
+ async destroy() {
3940
+ await this.ready();
3941
+ await this._crdt.destroy();
3942
+ await this.blockstore.destroy();
3943
+ }
3944
+ async ready() {
3945
+ return this._ready.once(async () => {
3946
+ await this.sthis.start();
3947
+ await this._crdt.ready();
3948
+ await this.blockstore.ready();
3949
+ });
3950
+ }
3951
+ async get(id) {
3952
+ if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
3953
+ await this.ready();
3954
+ this.logger.Debug().Str("id", id).Msg("get");
3955
+ const got = await this._crdt.get(id).catch((e) => {
3956
+ throw new NotFoundError(`Not found: ${id} - ${e.message}`);
3957
+ });
3958
+ if (!got) throw new NotFoundError(`Not found: ${id}`);
3959
+ const { doc } = got;
3960
+ return { ...doc, _id: id };
3961
+ }
3962
+ async put(doc) {
3963
+ await this.ready();
3964
+ this.logger.Debug().Str("id", doc._id).Msg("put");
3965
+ const { _id, ...value } = doc;
3966
+ const docId = _id || this.sthis.timeOrderedNextId().str;
3967
+ const result = await this._writeQueue.push({
3968
+ id: docId,
3969
+ value: {
3970
+ ...value,
3971
+ _id: docId
3972
+ }
3973
+ });
3974
+ return { id: docId, clock: result?.head, name: this.name };
3975
+ }
3976
+ async del(id) {
3977
+ await this.ready();
3978
+ this.logger.Debug().Str("id", id).Msg("del");
3979
+ const result = await this._writeQueue.push({ id, del: true });
3980
+ return { id, clock: result?.head, name: this.name };
3981
+ }
3982
+ async changes(since = [], opts = {}) {
3983
+ await this.ready();
3984
+ this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
3985
+ const { result, head } = await this._crdt.changes(since, opts);
3986
+ const rows = result.map(({ id: key, value, del, clock }) => ({
3987
+ key,
3988
+ value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
3989
+ clock
3990
+ }));
3991
+ return { rows, clock: head, name: this.name };
3992
+ }
3993
+ async allDocs(opts = {}) {
3994
+ await this.ready();
3995
+ this.logger.Debug().Msg("allDocs");
3996
+ const { result, head } = await this._crdt.allDocs();
3997
+ const rows = result.map(({ id: key, value, del }) => ({
3998
+ key,
3999
+ value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
4000
+ }));
4001
+ return { rows, clock: head, name: this.name };
4002
+ }
4003
+ async allDocuments() {
4004
+ return this.allDocs();
4005
+ }
4006
+ subscribe(listener, updates) {
4007
+ this.logger.Debug().Bool("updates", updates).Msg("subscribe");
4008
+ if (updates) {
4009
+ if (!this._listening) {
4010
+ this._listening = true;
4011
+ this._crdt.clock.onTick((updates2) => {
4012
+ void this._notify(updates2);
4013
+ });
4014
+ }
4015
+ this._listeners.add(listener);
4016
+ return () => {
4017
+ this._listeners.delete(listener);
4018
+ };
4019
+ } else {
4020
+ this._noupdate_listeners.add(listener);
4021
+ return () => {
4022
+ this._noupdate_listeners.delete(listener);
4023
+ };
4024
+ }
4025
+ }
4026
+ // todo if we add this onto dbs in fireproof.ts then we can make index.ts a separate package
4027
+ async query(field, opts = {}) {
4028
+ await this.ready();
4029
+ this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
4030
+ const _crdt = this._crdt;
4031
+ const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
4032
+ return await idx.query(opts);
4033
+ }
4034
+ async compact() {
4035
+ await this.ready();
4036
+ await this._crdt.compact();
4037
+ }
4038
+ async _notify(updates) {
4039
+ await this.ready();
4040
+ if (this._listeners.size) {
4041
+ const docs = updates.map(({ id, value }) => ({ ...value, _id: id }));
4042
+ for (const listener of this._listeners) {
4043
+ await (async () => await listener(docs))().catch((e) => {
4044
+ this.logger.Error().Err(e).Msg("subscriber error");
4045
+ });
4046
+ }
4047
+ }
4048
+ }
4049
+ async _no_update_notify() {
4050
+ await this.ready();
4051
+ if (this._noupdate_listeners.size) {
4052
+ for (const listener of this._noupdate_listeners) {
4053
+ await (async () => await listener([]))().catch((e) => {
4054
+ this.logger.Error().Err(e).Msg("subscriber error");
4055
+ });
4056
+ }
4057
+ }
4058
+ }
4059
+ };
4060
+ function toSortedArray(set) {
4061
+ if (!set) return [];
4062
+ return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
4063
+ }
4064
+ function fireproof(name, opts) {
4065
+ const key = JSON.stringify(
4066
+ toSortedArray({
4067
+ name,
4068
+ stores: toSortedArray(opts?.store?.stores)
4069
+ })
4070
+ );
4071
+ let db = Database.databases.get(key);
4072
+ if (!db) {
4073
+ db = new Database(name, opts);
4074
+ Database.databases.set(key, db);
4075
+ }
4076
+ return db;
4077
+ }
4078
+ function makeName(fnString) {
4079
+ const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
4080
+ let found = null;
4081
+ const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
4082
+ if (matches.length === 0) {
4083
+ found = /=>\s*{?\s*([^{}]+)\s*}?/.exec(fnString);
4084
+ if (found && found[1].includes("return")) {
4085
+ found = null;
4086
+ }
4087
+ }
4088
+ if (!found) {
4089
+ return fnString;
4090
+ } else {
4091
+ return found[1];
4092
+ }
4093
+ }
4094
+
4095
+ // src/runtime/index.ts
4096
+ var runtime_exports = {};
4097
+ __export(runtime_exports, {
4098
+ FILESTORE_VERSION: () => FILESTORE_VERSION,
4099
+ INDEXDB_VERSION: () => INDEXDB_VERSION,
4100
+ files: () => files_exports,
4101
+ getFileName: () => getFileName,
4102
+ getFileSystem: () => getFileSystem,
4103
+ getPath: () => getPath,
4104
+ kb: () => key_bag_exports,
4105
+ kc: () => keyed_crypto_exports,
4106
+ mf: () => wait_pr_multiformats_exports,
4107
+ runtimeFn: () => import_cement15.runtimeFn,
4108
+ toArrayBuffer: () => toArrayBuffer
4109
+ });
4110
+ init_utils2();
4111
+
4112
+ // src/runtime/wait-pr-multiformats/index.ts
4113
+ var wait_pr_multiformats_exports = {};
4114
+ __export(wait_pr_multiformats_exports, {
4115
+ block: () => block_exports,
4116
+ codec: () => codec_interface_exports
4117
+ });
4118
+
4119
+ // src/runtime/wait-pr-multiformats/codec-interface.ts
4120
+ var codec_interface_exports = {};
4121
+
4122
+ // src/runtime/index.ts
4123
+ var import_cement15 = require("@adviser/cement");
4124
+
4125
+ // src/runtime/gateways/file/version.ts
4126
+ var FILESTORE_VERSION = "v0.19-file";
4127
+
4128
+ // src/runtime/index.ts
4129
+ init_version();
4130
+
4131
+ // src/index.ts
4132
+ init_utils();
4133
+
4134
+ // src/version.ts
4135
+ var PACKAGE_VERSION = Object.keys({
4136
+ "0.19.112-dev-web": "xxxx"
4137
+ })[0];
4138
+ //# sourceMappingURL=index.cjs.map