@fireproof/core 0.19.99 → 0.19.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
  2. package/chunk-3EB3ENHT.js.map +1 -0
  3. package/chunk-HQ7D3PEU.js +61 -0
  4. package/chunk-HQ7D3PEU.js.map +1 -0
  5. package/chunk-PZ5AY32C.js +10 -0
  6. package/deno-filesystem-Q2IJ7YDR.js +57 -0
  7. package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
  8. package/deno.json +3 -1
  9. package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
  10. package/gateway-GK5QZ6KP.js.map +1 -0
  11. package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
  12. package/gateway-TQTGDRCN.js.map +1 -0
  13. package/index.cjs +2232 -1781
  14. package/index.cjs.map +1 -1
  15. package/index.d.cts +261 -117
  16. package/index.d.ts +261 -117
  17. package/index.global.js +12776 -11829
  18. package/index.global.js.map +1 -1
  19. package/index.js +1936 -1579
  20. package/index.js.map +1 -1
  21. package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
  22. package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
  23. package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
  24. package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
  25. package/key-bag-memory-LWE6ARPX.js +29 -0
  26. package/key-bag-memory-LWE6ARPX.js.map +1 -0
  27. package/metafile-cjs.json +1 -1
  28. package/metafile-esm.json +1 -1
  29. package/metafile-iife.json +1 -1
  30. package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
  31. package/node-filesystem-CFRXFSO7.js.map +1 -0
  32. package/package.json +4 -2
  33. package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
  34. package/tests/blockstore/keyed-crypto.test.ts +63 -227
  35. package/tests/blockstore/loader.test.ts +19 -11
  36. package/tests/blockstore/store.test.ts +23 -19
  37. package/tests/blockstore/transaction.test.ts +12 -12
  38. package/tests/fireproof/all-gateway.test.ts +201 -193
  39. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
  40. package/tests/fireproof/crdt.test.ts +67 -16
  41. package/tests/fireproof/database.test.ts +183 -21
  42. package/tests/fireproof/fireproof.test.ts +83 -74
  43. package/tests/fireproof/hello.test.ts +18 -14
  44. package/tests/fireproof/indexer.test.ts +53 -43
  45. package/tests/fireproof/utils.test.ts +18 -6
  46. package/tests/gateway/file/loader-config.test.ts +303 -0
  47. package/tests/gateway/indexdb/loader-config.test.ts +75 -0
  48. package/tests/helpers.ts +27 -9
  49. package/tests/react/useFireproof.test.tsx +1 -1
  50. package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
  51. package/utils-STA2C35G.js.map +1 -0
  52. package/chunk-OFGPKRCM.js.map +0 -1
  53. package/chunk-WS3YRPIA.js +0 -75
  54. package/chunk-WS3YRPIA.js.map +0 -1
  55. package/gateway-5FCWPX5W.js.map +0 -1
  56. package/gateway-H7UD6TNB.js.map +0 -1
  57. package/mem-filesystem-YPPJV7Q2.js +0 -41
  58. package/mem-filesystem-YPPJV7Q2.js.map +0 -1
  59. package/node-filesystem-INX4ZTHE.js.map +0 -1
  60. package/tests/fireproof/config.test.ts +0 -172
  61. /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
  62. /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.js CHANGED
@@ -1,3 +1,6 @@
1
+ import {
2
+ INDEXDB_VERSION
3
+ } from "./chunk-PB4BKL4O.js";
1
4
  import {
2
5
  FILESTORE_VERSION
3
6
  } from "./chunk-7EWIAXTM.js";
@@ -6,28 +9,30 @@ import {
6
9
  getFileSystem,
7
10
  getPath,
8
11
  toArrayBuffer
9
- } from "./chunk-WS3YRPIA.js";
10
- import {
11
- INDEXDB_VERSION
12
- } from "./chunk-PB4BKL4O.js";
12
+ } from "./chunk-HQ7D3PEU.js";
13
13
  import {
14
14
  NotFoundError,
15
+ PARAM,
15
16
  Result,
16
17
  UInt8ArrayEqual,
17
- __export,
18
- dataDir,
19
18
  ensureLogger,
20
19
  ensureSuperLog,
21
20
  ensureSuperThis,
22
21
  exceptionWrapper,
22
+ falsyToUndef,
23
23
  getKey,
24
24
  getName,
25
25
  getStore,
26
- isNotFoundError
27
- } from "./chunk-OFGPKRCM.js";
26
+ isFalsy,
27
+ isNotFoundError,
28
+ throwFalsy
29
+ } from "./chunk-3EB3ENHT.js";
30
+ import {
31
+ __export
32
+ } from "./chunk-PZ5AY32C.js";
28
33
 
29
34
  // src/database.ts
30
- import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
35
+ import { BuildURI as BuildURI2, KeyedResolvOnce as KeyedResolvOnce3, ResolveOnce as ResolveOnce6, URI as URI8 } from "@adviser/cement";
31
36
 
32
37
  // src/write-queue.ts
33
38
  function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -72,85 +77,6 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
72
77
  // src/crdt.ts
73
78
  import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
74
79
 
75
- // src/runtime/wait-pr-multiformats/block.ts
76
- var block_exports = {};
77
- __export(block_exports, {
78
- Block: () => Block,
79
- create: () => create,
80
- createUnsafe: () => createUnsafe,
81
- decode: () => decode,
82
- encode: () => encode
83
- });
84
- import { bytes as binary, CID } from "multiformats";
85
- import { Block as mfBlock } from "multiformats/block";
86
- var Block = mfBlock;
87
- async function decode({
88
- bytes,
89
- codec: codec3,
90
- hasher: hasher7
91
- }) {
92
- if (bytes == null) throw new Error('Missing required argument "bytes"');
93
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
94
- const value = await Promise.resolve(codec3.decode(bytes));
95
- const hash = await hasher7.digest(bytes);
96
- const cid = CID.create(1, codec3.code, hash);
97
- return new mfBlock({ value, bytes, cid });
98
- }
99
- async function encode({
100
- value,
101
- codec: codec3,
102
- hasher: hasher7
103
- }) {
104
- if (typeof value === "undefined") throw new Error('Missing required argument "value"');
105
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
106
- const bytes = await Promise.resolve(codec3.encode(value));
107
- const hash = await hasher7.digest(bytes);
108
- const cid = CID.create(1, codec3.code, hash);
109
- return new mfBlock({ value, bytes, cid });
110
- }
111
- async function create({
112
- bytes,
113
- cid,
114
- hasher: hasher7,
115
- codec: codec3
116
- }) {
117
- if (bytes == null) throw new Error('Missing required argument "bytes"');
118
- if (hasher7 == null) throw new Error('Missing required argument "hasher"');
119
- const value = await Promise.resolve(codec3.decode(bytes));
120
- const hash = await hasher7.digest(bytes);
121
- if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
122
- throw new Error("CID hash does not match bytes");
123
- }
124
- return createUnsafe({
125
- bytes,
126
- cid,
127
- value,
128
- codec: codec3
129
- });
130
- }
131
- async function createUnsafe({
132
- bytes,
133
- cid,
134
- value: maybeValue,
135
- codec: codec3
136
- }) {
137
- const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
138
- if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
139
- return new Block({
140
- cid,
141
- bytes,
142
- value
143
- });
144
- }
145
-
146
- // src/crdt-helpers.ts
147
- import { parse as parse3 } from "multiformats/link";
148
- import { sha256 as hasher5 } from "multiformats/hashes/sha2";
149
- import * as codec from "@ipld/dag-cbor";
150
- import { put, get, entries, root } from "@web3-storage/pail/crdt";
151
- import { EventFetcher, vis } from "@web3-storage/pail/clock";
152
- import * as Batch from "@web3-storage/pail/crdt/batch";
153
-
154
80
  // src/blockstore/index.ts
155
81
  var blockstore_exports = {};
156
82
  __export(blockstore_exports, {
@@ -162,8 +88,11 @@ __export(blockstore_exports, {
162
88
  FragmentGateway: () => FragmentGateway,
163
89
  Loader: () => Loader,
164
90
  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
165
- ensureStart: () => ensureStart,
166
- getGatewayFromURL: () => getGatewayFromURL,
91
+ ensureStoreEnDeFile: () => ensureStoreEnDeFile,
92
+ fileGatewayFactoryItem: () => fileGatewayFactoryItem,
93
+ getDefaultURI: () => getDefaultURI,
94
+ getGatewayFactoryItem: () => getGatewayFactoryItem,
95
+ getStartedGateway: () => getStartedGateway,
167
96
  parseCarFile: () => parseCarFile,
168
97
  registerStoreProtocol: () => registerStoreProtocol,
169
98
  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
@@ -178,7 +107,7 @@ function toCIDBlock(block) {
178
107
  }
179
108
 
180
109
  // src/blockstore/store-factory.ts
181
- import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
110
+ import { KeyedResolvOnce as KeyedResolvOnce2, Result as Result7 } from "@adviser/cement";
182
111
 
183
112
  // src/runtime/files.ts
184
113
  var files_exports = {};
@@ -252,329 +181,437 @@ var UnixFSFileBuilder = class {
252
181
  };
253
182
 
254
183
  // src/blockstore/store.ts
255
- import pLimit2 from "p-limit";
256
184
  import { format as format2, parse as parse2 } from "@ipld/dag-json";
257
- import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result5 } from "@adviser/cement";
258
-
259
- // src/types.ts
260
- function isFalsy(value) {
261
- return value === false && value === null && value === void 0;
262
- }
263
- function throwFalsy(value) {
264
- if (isFalsy(value)) {
265
- throw new Error("value is Falsy");
266
- }
267
- return value;
268
- }
269
- function falsyToUndef(value) {
270
- if (isFalsy(value)) {
271
- return void 0;
272
- }
273
- return value;
274
- }
275
-
276
- // src/blockstore/loader.ts
277
- import pLimit from "p-limit";
278
- import { CarReader } from "@ipld/car";
279
- import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
185
+ import { ResolveOnce as ResolveOnce3, Result as Result5, exception2Result } from "@adviser/cement";
280
186
 
281
- // src/blockstore/loader-helpers.ts
282
- import { sha256 as hasher } from "multiformats/hashes/sha2";
283
- import * as dagCodec from "@ipld/dag-cbor";
284
- async function parseCarFile(reader, logger) {
285
- const roots = await reader.getRoots();
286
- const header = await reader.get(roots[0]);
287
- if (!header) throw logger.Error().Msg("missing header block").AsError();
288
- const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
289
- const fpvalue = dec.value;
290
- if (fpvalue && !fpvalue.fp) {
291
- throw logger.Error().Msg("missing fp").AsError();
187
+ // src/blockstore/commit-queue.ts
188
+ import { Future } from "@adviser/cement";
189
+ var CommitQueue = class {
190
+ constructor() {
191
+ this.queue = [];
192
+ this.processing = false;
193
+ this._waitIdleItems = /* @__PURE__ */ new Set();
292
194
  }
293
- return fpvalue.fp;
294
- }
295
-
296
- // src/blockstore/transaction.ts
297
- import { MemoryBlockstore } from "@web3-storage/pail/block";
298
- import { toCryptoRuntime } from "@adviser/cement";
299
- var CarTransaction = class extends MemoryBlockstore {
300
- constructor(parent, opts = { add: true, noLoader: false }) {
301
- super();
302
- if (opts.add) {
303
- parent.transactions.add(this);
195
+ waitIdle() {
196
+ if (this.queue.length === 0 && !this.processing) {
197
+ return Promise.resolve();
304
198
  }
305
- this.parent = parent;
199
+ const fn = new Future();
200
+ this._waitIdleItems.add(fn);
201
+ return fn.asPromise();
306
202
  }
307
- async get(cid) {
308
- return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
203
+ async enqueue(fn) {
204
+ return new Promise((resolve, reject) => {
205
+ const queueFn = async () => {
206
+ try {
207
+ resolve(await fn());
208
+ } catch (e) {
209
+ reject(e);
210
+ } finally {
211
+ this.processing = false;
212
+ this.processNext();
213
+ }
214
+ };
215
+ this.queue.push(queueFn);
216
+ if (!this.processing) {
217
+ this.processNext();
218
+ }
219
+ });
309
220
  }
310
- async superGet(cid) {
311
- return super.get(cid);
221
+ processNext() {
222
+ if (this.queue.length > 0 && !this.processing) {
223
+ this.processing = true;
224
+ const queueFn = this.queue.shift();
225
+ if (queueFn) {
226
+ queueFn().finally(() => {
227
+ });
228
+ }
229
+ }
230
+ if (this.queue.length === 0 && !this.processing) {
231
+ const toResolve = Array.from(this._waitIdleItems);
232
+ this._waitIdleItems.clear();
233
+ toResolve.map((fn) => fn.resolve());
234
+ }
312
235
  }
313
236
  };
314
- function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
315
- const logger = ensureLogger(sthis, component, ctx);
316
- const store = opts.store || {};
317
- return {
237
+
238
+ // src/runtime/keyed-crypto.ts
239
+ var keyed_crypto_exports = {};
240
+ __export(keyed_crypto_exports, {
241
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
242
+ keyedCryptoFactory: () => keyedCryptoFactory
243
+ });
244
+ import { base58btc } from "multiformats/bases/base58";
245
+ import { sha256 as hasher } from "multiformats/hashes/sha2";
246
+ import * as CBOR from "cborg";
247
+ var generateIV = {
248
+ random: {
318
249
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
319
- applyMeta: (meta, snap) => {
320
- return Promise.resolve();
250
+ calc: async (ko, crypto, data) => {
251
+ return crypto.randomBytes(ko.ivLength);
321
252
  },
322
253
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
323
- compact: async (blocks) => {
324
- return {};
254
+ verify: async (ko, crypto, iv, data) => {
255
+ return true;
256
+ }
257
+ },
258
+ hash: {
259
+ calc: async (ko, crypto, data) => {
260
+ const hash = await hasher.digest(data);
261
+ const hashBytes = new Uint8Array(hash.bytes);
262
+ const hashArray = new Uint8Array(ko.ivLength);
263
+ for (let i = 0; i < hashBytes.length; i++) {
264
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
265
+ }
266
+ return hashArray;
325
267
  },
326
- autoCompact: 100,
327
- public: false,
328
- name: void 0,
329
- threshold: 1e3 * 1e3,
330
- ...opts,
331
- logger,
332
- keyBag: opts.keyBag || {},
333
- crypto: toCryptoRuntime(opts.crypto),
334
- store,
335
- storeRuntime: toStoreRuntime(store, sthis)
336
- };
337
- }
338
- function blockstoreFactory(sthis, opts) {
339
- if (opts.name) {
340
- return new EncryptedBlockstore(sthis, opts);
341
- } else {
342
- return new BaseBlockstore(opts);
268
+ verify: async function(ko, crypto, iv, data) {
269
+ return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
270
+ }
343
271
  }
272
+ };
273
+ function getGenerateIVFn(url, opts) {
274
+ const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
275
+ return generateIV[ivhash] || generateIV["hash"];
344
276
  }
345
- var BaseBlockstore = class {
346
- constructor(ebOpts = {}) {
347
- this.transactions = /* @__PURE__ */ new Set();
348
- this.sthis = ensureSuperThis(ebOpts);
349
- this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
350
- this.logger = this.ebOpts.logger;
351
- }
352
- // ready: Promise<void>;
353
- ready() {
354
- return Promise.resolve();
355
- }
356
- async close() {
277
+ var BlockIvKeyIdCodec = class {
278
+ constructor(ko, iv, opts) {
279
+ this.code = 3147065;
280
+ this.name = "Fireproof@encrypted-block:aes-gcm";
281
+ this.ko = ko;
282
+ this.iv = iv;
283
+ this.opts = opts || {};
357
284
  }
358
- async destroy() {
285
+ async encode(data) {
286
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
287
+ const { iv } = this.ko.algo(calcIv);
288
+ const fprt = await this.ko.fingerPrint();
289
+ const keyId = base58btc.decode(fprt);
290
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
291
+ return CBOR.encode({
292
+ iv,
293
+ keyId,
294
+ data: await this.ko._encrypt({ iv, bytes: data })
295
+ });
359
296
  }
360
- async get(cid) {
361
- if (!cid) throw this.logger.Error().Msg("required cid").AsError();
362
- for (const f of this.transactions) {
363
- const v = await f.superGet(cid);
364
- if (v) return v;
297
+ async decode(abytes) {
298
+ let bytes;
299
+ if (abytes instanceof Uint8Array) {
300
+ bytes = abytes;
301
+ } else {
302
+ bytes = new Uint8Array(abytes);
365
303
  }
366
- }
367
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
368
- async put(cid, block) {
369
- throw this.logger.Error().Msg("use a transaction to put").AsError();
370
- }
371
- // TransactionMeta
372
- async transaction(fn, _opts) {
373
- const t = new CarTransaction(this, _opts);
374
- const done = await fn(t);
375
- this.lastTxMeta = done;
376
- return { t, meta: done };
377
- }
378
- async *entries() {
379
- const seen = /* @__PURE__ */ new Set();
380
- for (const t of this.transactions) {
381
- for await (const blk of t.entries()) {
382
- if (seen.has(blk.cid.toString())) continue;
383
- seen.add(blk.cid.toString());
384
- yield blk;
385
- }
304
+ const { iv, keyId, data } = CBOR.decode(bytes);
305
+ const fprt = await this.ko.fingerPrint();
306
+ this.ko.logger.Debug().Str("fp", base58btc.encode(keyId)).Msg("decode");
307
+ if (base58btc.encode(keyId) !== fprt) {
308
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
386
309
  }
387
- }
388
- };
389
- var EncryptedBlockstore = class extends BaseBlockstore {
390
- constructor(sthis, ebOpts) {
391
- super(ebOpts);
392
- this.compacting = false;
393
- this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
394
- const { name } = ebOpts;
395
- if (!name) {
396
- throw this.logger.Error().Msg("name required").AsError();
310
+ const result = await this.ko._decrypt({ iv, bytes: data });
311
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
312
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
397
313
  }
398
- this.name = name;
399
- this.loader = new Loader(this.name, ebOpts, sthis);
314
+ return result;
400
315
  }
401
- ready() {
402
- return this.loader.ready();
316
+ };
317
+ var keyedCrypto = class {
318
+ constructor(url, key, cyopt, sthis) {
319
+ this.ivLength = 12;
320
+ this.isEncrypting = true;
321
+ this.logger = ensureLogger(sthis, "keyedCrypto");
322
+ this.crypto = cyopt;
323
+ this.key = key;
324
+ this.url = url;
403
325
  }
404
- close() {
405
- return this.loader.close();
326
+ fingerPrint() {
327
+ return Promise.resolve(this.key.fingerPrint);
406
328
  }
407
- destroy() {
408
- return this.loader.destroy();
329
+ codec(iv, opts) {
330
+ return new BlockIvKeyIdCodec(this, iv, opts);
409
331
  }
410
- async get(cid) {
411
- const got = await super.get(cid);
412
- if (got) return got;
413
- if (!this.loader) {
414
- return;
415
- }
416
- return falsyToUndef(await this.loader.getBlock(cid));
332
+ algo(iv) {
333
+ return {
334
+ name: "AES-GCM",
335
+ iv: iv || this.crypto.randomBytes(this.ivLength),
336
+ tagLength: 128
337
+ };
417
338
  }
418
- async transaction(fn, opts = { noLoader: false }) {
419
- const { t, meta: done } = await super.transaction(fn);
420
- const cars = await this.loader.commit(t, done, opts);
421
- if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
422
- setTimeout(() => void this.compact(), 10);
423
- }
424
- if (cars) {
425
- this.transactions.delete(t);
426
- return { meta: done, cars, t };
427
- }
428
- throw this.logger.Error().Msg("failed to commit car files").AsError();
339
+ async _decrypt(data) {
340
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
341
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
429
342
  }
430
- async getFile(car, cid) {
431
- await this.ready();
432
- if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
433
- const reader = await this.loader.loadFileCar(
434
- car
435
- /*, isPublic */
436
- );
437
- const block = await reader.get(cid);
438
- if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
439
- return block.bytes;
343
+ async _encrypt(data) {
344
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
345
+ const a = this.algo(data.iv);
346
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
440
347
  }
441
- async compact() {
442
- await this.ready();
443
- if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
444
- if (this.loader.carLog.length < 2) return;
445
- const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
446
- if (!compactFn || this.compacting) return;
447
- const blockLog = new CompactionFetcher(this);
448
- this.compacting = true;
449
- const meta = await compactFn(blockLog);
450
- await this.loader?.commit(blockLog.loggedBlocks, meta, {
451
- compact: true,
452
- noLoader: true
453
- });
454
- this.compacting = false;
348
+ };
349
+ var nullCodec = class {
350
+ constructor() {
351
+ this.code = 0;
352
+ this.name = "Fireproof@unencrypted-block";
455
353
  }
456
- async defaultCompact(blocks, logger) {
457
- if (!this.loader) {
458
- throw logger.Error().Msg("no loader").AsError();
459
- }
460
- if (!this.lastTxMeta) {
461
- throw logger.Error().Msg("no lastTxMeta").AsError();
462
- }
463
- for await (const blk of this.loader.entries(false)) {
464
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
465
- }
466
- for (const t of this.transactions) {
467
- for await (const blk of t.entries()) {
468
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
469
- }
470
- }
471
- return this.lastTxMeta;
354
+ encode(data) {
355
+ return data;
472
356
  }
473
- async *entries() {
474
- for await (const blk of this.loader.entries()) {
475
- yield blk;
476
- }
357
+ decode(data) {
358
+ return data;
477
359
  }
478
360
  };
479
- var CompactionFetcher = class {
480
- constructor(blocks) {
481
- this.blockstore = blocks;
482
- this.loggedBlocks = new CarTransaction(blocks);
361
+ var noCrypto = class {
362
+ constructor(url, cyrt, sthis) {
363
+ this.ivLength = 0;
364
+ this.code = 0;
365
+ this.name = "Fireproof@unencrypted-block";
366
+ this.isEncrypting = false;
367
+ this._fingerPrint = "noCrypto:" + Math.random();
368
+ this.logger = ensureLogger(sthis, "noCrypto");
369
+ this.crypto = cyrt;
370
+ this.url = url;
483
371
  }
484
- async get(cid) {
485
- const block = await this.blockstore.get(cid);
486
- if (block) this.loggedBlocks.putSync(cid, block.bytes);
487
- return falsyToUndef(block);
372
+ fingerPrint() {
373
+ return Promise.resolve(this._fingerPrint);
488
374
  }
489
- };
490
-
491
- // src/blockstore/commit-queue.ts
492
- import { Future } from "@adviser/cement";
493
- var CommitQueue = class {
494
- constructor() {
495
- this.queue = [];
496
- this.processing = false;
497
- this._waitIdleItems = /* @__PURE__ */ new Set();
375
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
376
+ codec(iv) {
377
+ return new nullCodec();
498
378
  }
499
- waitIdle() {
500
- if (this.queue.length === 0 && !this.processing) {
501
- return Promise.resolve();
502
- }
503
- const fn = new Future();
504
- this._waitIdleItems.add(fn);
505
- return fn.asPromise();
379
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
380
+ algo(iv) {
381
+ return {
382
+ name: "noCrypto",
383
+ iv: new Uint8Array(),
384
+ tagLength: 0
385
+ };
506
386
  }
507
- async enqueue(fn) {
508
- return new Promise((resolve, reject) => {
509
- const queueFn = async () => {
510
- try {
511
- resolve(await fn());
512
- } catch (e) {
513
- reject(e);
514
- } finally {
515
- this.processing = false;
516
- this.processNext();
517
- }
518
- };
519
- this.queue.push(queueFn);
520
- if (!this.processing) {
521
- this.processNext();
522
- }
523
- });
387
+ _decrypt() {
388
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
524
389
  }
525
- processNext() {
526
- if (this.queue.length > 0 && !this.processing) {
527
- this.processing = true;
528
- const queueFn = this.queue.shift();
529
- if (queueFn) {
530
- queueFn().finally(() => {
531
- });
390
+ _encrypt() {
391
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
392
+ }
393
+ };
394
+ async function keyedCryptoFactory(url, kb, sthis) {
395
+ const storekey = url.getParam("storekey" /* STORE_KEY */);
396
+ if (storekey && storekey !== "insecure") {
397
+ let rkey = await kb.getNamedKey(storekey, true);
398
+ if (rkey.isErr()) {
399
+ try {
400
+ rkey = await kb.toKeyWithFingerPrint(storekey);
401
+ } catch (e) {
402
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
532
403
  }
533
404
  }
534
- if (this.queue.length === 0 && !this.processing) {
535
- const toResolve = Array.from(this._waitIdleItems);
536
- this._waitIdleItems.clear();
537
- toResolve.map((fn) => fn.resolve());
538
- }
405
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
539
406
  }
540
- };
407
+ return new noCrypto(url, kb.rt.crypto, sthis);
408
+ }
541
409
 
542
- // src/runtime/key-bag.ts
543
- var key_bag_exports = {};
544
- __export(key_bag_exports, {
545
- KeyBag: () => KeyBag,
546
- getKeyBag: () => getKeyBag,
547
- registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
548
- });
549
- import {
550
- KeyedResolvOnce,
551
- ResolveOnce,
552
- ResolveSeq,
553
- Result as Result2,
554
- runtimeFn,
555
- toCryptoRuntime as toCryptoRuntime2,
556
- URI
557
- } from "@adviser/cement";
558
- import { base58btc } from "multiformats/bases/base58";
559
- var KeyBag = class {
560
- constructor(rt) {
561
- this.rt = rt;
562
- this._warnOnce = new ResolveOnce();
563
- this._seq = new ResolveSeq();
564
- this.logger = ensureLogger(rt.sthis, "KeyBag");
565
- this.logger.Debug().Msg("KeyBag created");
410
+ // src/blockstore/fragment-gateway.ts
411
+ import { Result as Result2 } from "@adviser/cement";
412
+ import { base58btc as base58btc2 } from "multiformats/bases/base58";
413
+ import { encode as encode2, decode as decode2 } from "cborg";
414
+ function getFragSize(url) {
415
+ const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
416
+ let ret = 0;
417
+ if (fragSize) {
418
+ ret = parseInt(fragSize);
566
419
  }
567
- async subtleKey(key) {
568
- const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
569
- if (extractable) {
570
- this._warnOnce.once(
571
- () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
420
+ if (isNaN(ret) || ret <= 0) {
421
+ ret = 0;
422
+ }
423
+ return ret;
424
+ }
425
+ async function getFrags(url, innerGW, headerSize, logger) {
426
+ const fragSize = getFragSize(url);
427
+ if (!fragSize) {
428
+ const res = await innerGW.get(url);
429
+ if (res.isErr()) {
430
+ return [res];
431
+ }
432
+ const data = res.unwrap();
433
+ return [
434
+ Result2.Ok({
435
+ fid: new Uint8Array(0),
436
+ ofs: 0,
437
+ len: data.length,
438
+ data
439
+ })
440
+ ];
441
+ }
442
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
443
+ if (firstRaw.isErr()) {
444
+ return [firstRaw];
445
+ }
446
+ const firstFragment = decode2(firstRaw.unwrap());
447
+ const blockSize = firstFragment.data.length;
448
+ const ops = [Promise.resolve(Result2.Ok(firstFragment))];
449
+ const fidStr = base58btc2.encode(firstFragment.fid);
450
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
451
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
452
+ ops.push(
453
+ (async (furl, ofs2) => {
454
+ const raw2 = await innerGW.get(furl);
455
+ if (raw2.isErr()) {
456
+ return raw2;
457
+ }
458
+ const fragment = decode2(raw2.unwrap());
459
+ if (base58btc2.encode(fragment.fid) !== fidStr) {
460
+ return Result2.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
461
+ }
462
+ if (fragment.ofs !== ofs2) {
463
+ return Result2.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
464
+ }
465
+ return Result2.Ok(fragment);
466
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
467
+ );
468
+ }
469
+ return Promise.all(ops);
470
+ }
471
+ var FragmentGateway = class {
472
+ constructor(sthis, innerGW) {
473
+ this.fidLength = 4;
474
+ this.headerSize = 32;
475
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
476
+ this.logger = this.sthis.logger;
477
+ this.innerGW = innerGW;
478
+ }
479
+ slicer(url, body) {
480
+ const fragSize = getFragSize(url);
481
+ if (!fragSize) {
482
+ return [this.innerGW.put(url, body)];
483
+ }
484
+ const blocksize = fragSize - this.headerSize;
485
+ if (blocksize <= 0) {
486
+ throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
487
+ }
488
+ const ops = [];
489
+ const fid = this.sthis.nextId(this.fidLength);
490
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
491
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
492
+ const block = encode2({
493
+ fid: fid.bin,
494
+ ofs,
495
+ len: body.length,
496
+ data: body.slice(ofs, ofs + blocksize)
497
+ });
498
+ if (block.length > fragSize) {
499
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
500
+ }
501
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
502
+ }
503
+ return ops;
504
+ }
505
+ buildUrl(baseUrl, key) {
506
+ return this.innerGW.buildUrl(baseUrl, key);
507
+ }
508
+ async destroy(iurl) {
509
+ return this.innerGW.destroy(iurl);
510
+ }
511
+ async start(url) {
512
+ this.headerSize = encode2({
513
+ fid: this.sthis.nextId(this.fidLength).bin,
514
+ ofs: 1024 * 1024,
515
+ // 32bit
516
+ len: 16 * 1024 * 1024,
517
+ // 32bit
518
+ data: new Uint8Array(1024)
519
+ }).length - 1024;
520
+ return this.innerGW.start(url);
521
+ }
522
+ async close(url) {
523
+ return this.innerGW.close(url);
524
+ }
525
+ async put(url, body) {
526
+ await Promise.all(this.slicer(url, body));
527
+ return Result2.Ok(void 0);
528
+ }
529
+ async get(url) {
530
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
531
+ let buffer = void 0;
532
+ for (const rfrag of rfrags) {
533
+ if (rfrag.isErr()) {
534
+ return Result2.Err(rfrag.Err());
535
+ }
536
+ const frag = rfrag.Ok();
537
+ buffer = buffer || new Uint8Array(frag.len);
538
+ buffer.set(frag.data, frag.ofs);
539
+ }
540
+ return Result2.Ok(buffer || new Uint8Array(0));
541
+ }
542
+ async subscribe(url, callback) {
543
+ if (this.innerGW.subscribe) {
544
+ return this.innerGW.subscribe(url, callback);
545
+ } else {
546
+ return Result2.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
547
+ }
548
+ }
549
+ async delete(url) {
550
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
551
+ for (const rfrag of rfrags) {
552
+ if (rfrag.isErr()) {
553
+ return Result2.Err(rfrag.Err());
554
+ }
555
+ const frag = rfrag.Ok();
556
+ let fragUrl;
557
+ if (rfrags.length > 1) {
558
+ const fidStr = base58btc2.encode(frag.fid);
559
+ fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
560
+ } else {
561
+ fragUrl = url;
562
+ }
563
+ await this.innerGW.delete(fragUrl);
564
+ }
565
+ return Result2.Ok(void 0);
566
+ }
567
+ };
568
+
569
+ // src/blockstore/meta-key-helper.ts
570
+ import { format, parse } from "@ipld/dag-json";
571
+ import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
572
+ import { CID } from "multiformats";
573
+ import { base64pad } from "multiformats/bases/base64";
574
+ import { Result as Result4 } from "@adviser/cement";
575
+
576
+ // src/runtime/key-bag.ts
577
+ var key_bag_exports = {};
578
+ __export(key_bag_exports, {
579
+ KeyBag: () => KeyBag,
580
+ defaultKeyBagOpts: () => defaultKeyBagOpts,
581
+ getKeyBag: () => getKeyBag,
582
+ registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
583
+ });
584
+ import {
585
+ KeyedResolvOnce,
586
+ ResolveOnce,
587
+ ResolveSeq,
588
+ Result as Result3,
589
+ runtimeFn,
590
+ toCryptoRuntime,
591
+ URI as URI2
592
+ } from "@adviser/cement";
593
+ import { base58btc as base58btc3 } from "multiformats/bases/base58";
594
+ var KeyBag = class {
595
+ constructor(rt) {
596
+ this.rt = rt;
597
+ this._warnOnce = new ResolveOnce();
598
+ this._seq = new ResolveSeq();
599
+ this.logger = ensureLogger(rt.sthis, "KeyBag", {
600
+ id: rt.id()
601
+ });
602
+ this.logger.Debug().Msg("KeyBag created");
603
+ }
604
+ async subtleKey(key) {
605
+ const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
606
+ if (extractable) {
607
+ this._warnOnce.once(
608
+ () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
572
609
  );
573
610
  }
574
611
  return await this.rt.crypto.importKey(
575
612
  "raw",
576
613
  // raw or jwk
577
- base58btc.decode(key),
614
+ base58btc3.decode(key),
578
615
  // hexStringToUint8Array(key), // raw data
579
616
  "AES-GCM",
580
617
  extractable,
@@ -582,9 +619,9 @@ var KeyBag = class {
582
619
  );
583
620
  }
584
621
  async ensureKeyFromUrl(url, keyFactory) {
585
- const storeKey = url.getParam("storekey");
622
+ const storeKey = url.getParam("storekey" /* STORE_KEY */);
586
623
  if (storeKey === "insecure") {
587
- return Result2.Ok(url);
624
+ return Result3.Ok(url);
588
625
  }
589
626
  if (!storeKey) {
590
627
  const keyName = `@${keyFactory()}@`;
@@ -592,8 +629,8 @@ var KeyBag = class {
592
629
  if (ret.isErr()) {
593
630
  return ret;
594
631
  }
595
- const urb = url.build().setParam("storekey", keyName);
596
- return Result2.Ok(urb.URI());
632
+ const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
633
+ return Result3.Ok(urb.URI());
597
634
  }
598
635
  if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
599
636
  const ret = await this.getNamedKey(storeKey);
@@ -601,15 +638,15 @@ var KeyBag = class {
601
638
  return ret;
602
639
  }
603
640
  }
604
- return Result2.Ok(url);
641
+ return Result3.Ok(url);
605
642
  }
606
643
  async toKeyWithFingerPrint(keyStr) {
607
- const material = base58btc.decode(keyStr);
644
+ const material = base58btc3.decode(keyStr);
608
645
  const key = await this.subtleKey(keyStr);
609
646
  const fpr = await this.rt.crypto.digestSHA256(material);
610
- return Result2.Ok({
647
+ return Result3.Ok({
611
648
  key,
612
- fingerPrint: base58btc.encode(new Uint8Array(fpr))
649
+ fingerPrint: base58btc3.encode(new Uint8Array(fpr))
613
650
  });
614
651
  }
615
652
  async setNamedKey(name, key) {
@@ -632,13 +669,13 @@ var KeyBag = class {
632
669
  return ret;
633
670
  }
634
671
  const named = ret.Ok();
635
- return Result2.Ok({
672
+ return Result3.Ok({
636
673
  ...named,
637
674
  extract: async () => {
638
675
  const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
639
676
  return {
640
677
  key: ext,
641
- keyStr: base58btc.encode(ext)
678
+ keyStr: base58btc3.encode(ext)
642
679
  };
643
680
  }
644
681
  });
@@ -655,9 +692,9 @@ var KeyBag = class {
655
692
  }
656
693
  if (failIfNotFound) {
657
694
  this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
658
- return Result2.Err(new Error(`Key not found: ${name}`));
695
+ return Result3.Err(new Error(`Key not found: ${name}`));
659
696
  }
660
- const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
697
+ const ret = await this._setNamedKey(name, base58btc3.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
661
698
  this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
662
699
  return ret;
663
700
  });
@@ -668,14 +705,14 @@ var keyBagProviderFactories = new Map(
668
705
  {
669
706
  protocol: "file:",
670
707
  factory: async (url, sthis) => {
671
- const { KeyBagProviderFile } = await import("./key-bag-file-WADZBHYG.js");
708
+ const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
672
709
  return new KeyBagProviderFile(url, sthis);
673
710
  }
674
711
  },
675
712
  {
676
713
  protocol: "indexdb:",
677
714
  factory: async (url, sthis) => {
678
- const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-PGVAI3FJ.js");
715
+ const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
679
716
  return new KeyBagProviderIndexDB(url, sthis);
680
717
  }
681
718
  }
@@ -689,44 +726,63 @@ function registerKeyBagProviderFactory(item) {
689
726
  });
690
727
  }
691
728
  function defaultKeyBagOpts(sthis, kbo) {
729
+ kbo = kbo || {};
692
730
  if (kbo.keyRuntime) {
693
731
  return kbo.keyRuntime;
694
732
  }
695
733
  const logger = ensureLogger(sthis, "KeyBag");
696
734
  let url;
697
735
  if (kbo.url) {
698
- url = URI.from(kbo.url);
736
+ url = URI2.from(kbo.url);
699
737
  logger.Debug().Url(url).Msg("from opts");
700
738
  } else {
701
739
  let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
702
740
  if (runtimeFn().isBrowser) {
703
- url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
741
+ url = URI2.from(bagFnameOrUrl || "indexdb://fp-keybag");
704
742
  } else {
705
743
  if (!bagFnameOrUrl) {
706
744
  const home = sthis.env.get("HOME");
707
745
  bagFnameOrUrl = `${home}/.fireproof/keybag`;
708
- url = URI.from(`file://${bagFnameOrUrl}`);
746
+ url = URI2.from(`file://${bagFnameOrUrl}`);
709
747
  } else {
710
- url = URI.from(bagFnameOrUrl);
748
+ url = URI2.from(bagFnameOrUrl);
711
749
  }
712
750
  }
713
751
  logger.Debug().Url(url).Msg("from env");
714
752
  }
715
- const kitem = keyBagProviderFactories.get(url.protocol);
716
- if (!kitem) {
717
- throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
753
+ let keyProviderFactory;
754
+ switch (url.protocol) {
755
+ case "file:":
756
+ keyProviderFactory = async () => {
757
+ const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
758
+ return new KeyBagProviderFile(url, sthis);
759
+ };
760
+ break;
761
+ case "indexdb:":
762
+ keyProviderFactory = async () => {
763
+ const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
764
+ return new KeyBagProviderIndexDB(url, sthis);
765
+ };
766
+ break;
767
+ case "memory:":
768
+ keyProviderFactory = async () => {
769
+ const { KeyBagProviderMemory } = await import("./key-bag-memory-LWE6ARPX.js");
770
+ return new KeyBagProviderMemory(url, sthis);
771
+ };
772
+ break;
773
+ default:
774
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
718
775
  }
719
- const getBag = async () => kitem.factory(url, sthis);
720
776
  if (url.hasParam("masterkey")) {
721
777
  throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
722
778
  }
723
779
  return {
724
780
  url,
725
- crypto: kbo.crypto || toCryptoRuntime2({}),
781
+ crypto: kbo.crypto || toCryptoRuntime({}),
726
782
  sthis,
727
783
  logger,
728
784
  keyLength: kbo.keyLength || 16,
729
- getBag,
785
+ getBag: keyProviderFactory,
730
786
  id: () => {
731
787
  return url.toString();
732
788
  }
@@ -739,932 +795,938 @@ async function getKeyBag(sthis, kbo = {}) {
739
795
  return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
740
796
  }
741
797
 
742
- // src/blockstore/commitor.ts
743
- import * as CBW from "@ipld/car/buffer-writer";
744
- import { sha256 as hasher2 } from "multiformats/hashes/sha2";
745
- import * as dagCodec2 from "@ipld/dag-cbor";
746
- async function encodeCarFile(roots, t, codec3) {
747
- let size = 0;
748
- const headerSize = CBW.headerLength({ roots });
749
- size += headerSize;
750
- for (const { cid, bytes } of t.entries()) {
751
- size += CBW.blockLength({ cid, bytes });
752
- }
753
- const buffer = new Uint8Array(size);
754
- const writer = CBW.createWriter(buffer, { headerSize });
755
- for (const r of roots) {
756
- writer.addRoot(r);
757
- }
758
- for (const { cid, bytes } of t.entries()) {
759
- writer.write({ cid, bytes });
798
+ // src/blockstore/meta-key-helper.ts
799
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
800
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
801
+ if (!crdtEntries.length) {
802
+ sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
803
+ return [];
760
804
  }
761
- writer.close();
762
- return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
763
- }
764
- async function createCarFile(encoder, cid, t) {
765
- return encodeCarFile([cid], t, encoder);
766
- }
767
- async function commitFiles(fileStore, walStore, t, done) {
768
- const { files: roots } = makeFileCarHeader(done);
769
- const cids = [];
770
- const codec3 = (await fileStore.keyedCrypto()).codec();
771
- const cars = await prepareCarFilesFiles(codec3, roots, t);
772
- for (const car of cars) {
773
- const { cid, bytes } = car;
774
- await fileStore.save({ cid, bytes });
775
- await walStore.enqueueFile(
776
- cid
777
- /*, !!opts.public*/
778
- );
779
- cids.push(cid);
805
+ if (!crdtEntries.map) {
806
+ sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
807
+ return [];
780
808
  }
781
- return cids;
809
+ return Promise.all(
810
+ crdtEntries.map(async (crdtEntry) => {
811
+ const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
812
+ const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
813
+ return {
814
+ eventCid: eventBlock.cid,
815
+ parents: crdtEntry.parents,
816
+ dbMeta
817
+ };
818
+ })
819
+ );
782
820
  }
783
- function makeFileCarHeader(result) {
784
- const files = [];
785
- for (const [, meta] of Object.entries(result.files || {})) {
786
- if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
787
- files.push(meta.cid);
821
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
822
+ try {
823
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
824
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
825
+ if (keyInfo.length) {
826
+ const dbMeta = keyInfo[0].dbMeta;
827
+ if (dbMeta.key) {
828
+ const kb = await getKeyBag(sthis);
829
+ const keyName = getStoreKeyName(uri);
830
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
831
+ if (res.isErr()) {
832
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
833
+ throw res.Err();
834
+ }
835
+ }
836
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
837
+ return Result4.Ok(dbMeta);
788
838
  }
839
+ sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
840
+ return Result4.Ok(void 0);
841
+ } catch (error) {
842
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
843
+ return Result4.Err(error);
789
844
  }
790
- return { ...result, files };
791
845
  }
792
- async function prepareCarFilesFiles(encoder, roots, t) {
793
- return [await encodeCarFile(roots, t, encoder)];
794
- }
795
- function makeCarHeader(meta, cars, compact = false) {
796
- const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
797
- return { ...coreHeader, meta };
798
- }
799
- async function encodeCarHeader(fp) {
800
- return await encode({
801
- value: { fp },
802
- hasher: hasher2,
803
- codec: dagCodec2
804
- });
805
- }
806
- async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
807
- const fp = makeCarHeader(done, params.carLog, !!opts.compact);
808
- const rootBlock = await encodeCarHeader(fp);
809
- const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
810
- const cids = [];
811
- for (const car of cars) {
812
- const { cid, bytes } = car;
813
- await params.carStore.save({ cid, bytes });
814
- cids.push(cid);
846
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
847
+ try {
848
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
849
+ const keyName = getStoreKeyName(uri);
850
+ const kb = await getKeyBag(sthis);
851
+ const res = await kb.getNamedExtractableKey(keyName, true);
852
+ if (res.isErr()) {
853
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
854
+ throw res.Err();
855
+ }
856
+ const keyData = await res.Ok().extract();
857
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
858
+ const { dbMeta, parents } = dbMetas[0];
859
+ const parentLinks = parents.map((p) => CID.parse(p));
860
+ dbMeta.key = keyData.keyStr;
861
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
862
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
863
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
864
+ return Result4.Ok(encoded);
865
+ } catch (error) {
866
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
867
+ return Result4.Err(error);
815
868
  }
816
- const newDbMeta = { cars: cids };
817
- await params.WALStore.enqueue(newDbMeta, opts);
818
- await params.metaStore.save(newDbMeta);
819
- return { cgrp: cids, header: fp };
820
869
  }
821
- async function prepareCarFiles(encoder, threshold, rootBlock, t) {
822
- const carFiles = [];
823
- threshold = threshold || 128e3 * 8;
824
- let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
825
- clonedt.putSync(rootBlock.cid, rootBlock.bytes);
826
- let newsize = CBW.blockLength(toCIDBlock(rootBlock));
827
- let cidRootBlock = rootBlock;
828
- for (const { cid, bytes } of t.entries()) {
829
- newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
830
- if (newsize >= threshold) {
831
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
832
- clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
833
- clonedt.putSync(cid, bytes);
834
- cidRootBlock = { cid, bytes };
835
- newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
836
- } else {
837
- clonedt.putSync(cid, bytes);
838
- }
870
+ function getStoreKeyName(url) {
871
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
872
+ const idx = url.getParam("index");
873
+ if (idx) {
874
+ storeKeyName.push(idx);
839
875
  }
840
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
841
- return carFiles;
876
+ storeKeyName.push("data");
877
+ return `@${storeKeyName.join(":")}@`;
878
+ }
879
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
880
+ const event = await EventBlock.create(
881
+ {
882
+ dbMeta: sthis.txt.encode(format(dbMeta))
883
+ },
884
+ parents
885
+ );
886
+ return event;
887
+ }
888
+ async function encodeEventsWithParents(sthis, events, parents) {
889
+ const crdtEntries = events.map((event) => {
890
+ const base64String = base64pad.encode(event.bytes);
891
+ return {
892
+ cid: event.cid.toString(),
893
+ data: base64String,
894
+ parents: parents.map((p) => p.toString())
895
+ };
896
+ });
897
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
842
898
  }
843
899
 
844
- // src/blockstore/loader.ts
845
- import { sha256 as hasher3 } from "multiformats/hashes/sha2";
846
-
847
- // src/blockstore/task-manager.ts
848
- var TaskManager = class {
849
- constructor(sthis, callback) {
850
- this.eventsWeHandled = /* @__PURE__ */ new Set();
851
- this.queue = [];
852
- this.isProcessing = false;
853
- this.logger = ensureLogger(sthis, "TaskManager");
854
- this.callback = callback;
855
- }
856
- async handleEvent(cid, parents, dbMeta) {
857
- for (const parent of parents) {
858
- this.eventsWeHandled.add(parent.toString());
859
- }
860
- this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
861
- this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
862
- void this.processQueue();
863
- }
864
- async processQueue() {
865
- if (this.isProcessing) return;
866
- this.isProcessing = true;
867
- const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
868
- const first = filteredQueue[0];
869
- if (!first) {
870
- return;
871
- }
872
- try {
873
- await this.callback(first.dbMeta);
874
- this.eventsWeHandled.add(first.cid);
875
- this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
876
- } catch (err) {
877
- if (first.retries++ > 3) {
878
- this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
879
- this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
880
- }
881
- await new Promise((resolve) => setTimeout(resolve, 50));
882
- throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
883
- } finally {
884
- this.isProcessing = false;
885
- if (this.queue.length > 0) {
886
- void this.processQueue();
887
- }
888
- }
889
- }
890
- };
900
+ // src/blockstore/store.ts
901
+ import pRetry from "p-retry";
902
+ import pMap from "p-map";
891
903
 
892
904
  // src/blockstore/loader.ts
893
- function carLogIncludesGroup(list, cids) {
894
- return list.some((arr) => {
895
- return arr.toString() === cids.toString();
896
- });
905
+ import pLimit from "p-limit";
906
+ import { CarReader } from "@ipld/car";
907
+ import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
908
+
909
+ // src/runtime/wait-pr-multiformats/block.ts
910
+ var block_exports = {};
911
+ __export(block_exports, {
912
+ Block: () => Block,
913
+ create: () => create,
914
+ createUnsafe: () => createUnsafe,
915
+ decode: () => decode3,
916
+ encode: () => encode3
917
+ });
918
+ import { bytes as binary, CID as CID2 } from "multiformats";
919
+ import { Block as mfBlock } from "multiformats/block";
920
+ var Block = mfBlock;
921
+ async function decode3({
922
+ bytes,
923
+ codec: codec3,
924
+ hasher: hasher7
925
+ }) {
926
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
927
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
928
+ const value = await Promise.resolve(codec3.decode(bytes));
929
+ const hash = await hasher7.digest(bytes);
930
+ const cid = CID2.create(1, codec3.code, hash);
931
+ return new mfBlock({ value, bytes, cid });
897
932
  }
898
- function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
899
- const byString = /* @__PURE__ */ new Map();
900
- for (const cid of list) {
901
- if (remove.has(cid.toString())) continue;
902
- byString.set(cid.toString(), cid);
903
- }
904
- return [...byString.values()];
933
+ async function encode3({
934
+ value,
935
+ codec: codec3,
936
+ hasher: hasher7
937
+ }) {
938
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
939
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
940
+ const bytes = await Promise.resolve(codec3.encode(value));
941
+ const hash = await hasher7.digest(bytes);
942
+ const cid = CID2.create(1, codec3.code, hash);
943
+ return new mfBlock({ value, bytes, cid });
905
944
  }
906
- var Loader = class {
907
- constructor(name, ebOpts, sthis) {
908
- this.commitQueue = new CommitQueue();
909
- this.isCompacting = false;
910
- this.carReaders = /* @__PURE__ */ new Map();
911
- this.seenCompacted = /* @__PURE__ */ new Set();
912
- this.processedCars = /* @__PURE__ */ new Set();
913
- this.carLog = [];
914
- this.getBlockCache = /* @__PURE__ */ new Map();
915
- this.seenMeta = /* @__PURE__ */ new Set();
916
- this.writeLimit = pLimit(1);
917
- this.onceReady = new ResolveOnce2();
918
- this.name = name;
919
- this.sthis = sthis;
920
- this.ebOpts = defaultedBlockstoreRuntime(
921
- sthis,
922
- {
923
- ...ebOpts,
924
- name
925
- },
926
- "Loader"
927
- );
928
- this.logger = this.ebOpts.logger;
929
- this.taskManager = new TaskManager(sthis, async (dbMeta) => {
930
- await this.handleDbMetasFromStore([dbMeta]);
931
- });
932
- }
933
- // readonly id = uuidv4();
934
- async keyBag() {
935
- return getKeyBag(this.sthis, this.ebOpts.keyBag);
936
- }
937
- async carStore() {
938
- return this.ebOpts.storeRuntime.makeDataStore(this);
939
- }
940
- async fileStore() {
941
- return this.ebOpts.storeRuntime.makeDataStore(this);
942
- }
943
- async WALStore() {
944
- return this.ebOpts.storeRuntime.makeWALStore(this);
945
- }
946
- async metaStore() {
947
- return this.ebOpts.storeRuntime.makeMetaStore(this);
948
- }
949
- async ready() {
950
- return this.onceReady.once(async () => {
951
- const metas = await (await this.metaStore()).load();
952
- if (this.ebOpts.meta) {
953
- await this.handleDbMetasFromStore([this.ebOpts.meta]);
954
- } else if (metas) {
955
- await this.handleDbMetasFromStore(metas);
956
- }
957
- });
958
- }
959
- async close() {
960
- const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
961
- await Promise.all(toClose.map((store) => store.close()));
962
- }
963
- async destroy() {
964
- const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
965
- await Promise.all(toDestroy.map((store) => store.destroy()));
966
- }
967
- // async snapToCar(carCid: AnyLink | string) {
968
- // await this.ready
969
- // if (typeof carCid === 'string') {
970
- // carCid = CID.parse(carCid)
971
- // }
972
- // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
973
- // this.carLog = [carCid, ...carHeader.cars]
974
- // await this.getMoreReaders(carHeader.cars)
975
- // await this._applyCarHeader(carHeader, true)
976
- // }
977
- async handleDbMetasFromStore(metas) {
978
- this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
979
- for (const meta of metas) {
980
- await this.writeLimit(async () => {
981
- await this.mergeDbMetaIntoClock(meta);
982
- });
983
- }
984
- }
985
- async mergeDbMetaIntoClock(meta) {
986
- if (this.isCompacting) {
987
- throw this.logger.Error().Msg("cannot merge while compacting").AsError();
988
- }
989
- if (this.seenMeta.has(meta.cars.toString())) return;
990
- this.seenMeta.add(meta.cars.toString());
991
- if (carLogIncludesGroup(this.carLog, meta.cars)) {
992
- return;
993
- }
994
- const carHeader = await this.loadCarHeaderFromMeta(meta);
995
- carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
996
- await this.getMoreReaders(carHeader.cars.flat());
997
- this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
998
- await this.ebOpts.applyMeta?.(carHeader.meta);
999
- }
1000
- // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1001
- // const { key } = meta;
1002
- // if (key) {
1003
- // await this.setKey(key);
1004
- // }
1005
- // }
1006
- async loadCarHeaderFromMeta({ cars: cids }) {
1007
- const reader = await this.loadCar(cids[0]);
1008
- return await parseCarFile(reader, this.logger);
1009
- }
1010
- // async _getKey(): Promise<string | undefined> {
1011
- // if (this.key) return this.key;
1012
- // // generate a random key
1013
- // if (!this.ebOpts.public) {
1014
- // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1015
- // }
1016
- // return this.key || undefined;
1017
- // }
1018
- async commitFiles(t, done) {
1019
- await this.ready();
1020
- const fstore = await this.fileStore();
1021
- const wstore = await this.WALStore();
1022
- return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1023
- }
1024
- async loadFileCar(cid) {
1025
- return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1026
- }
1027
- async commit(t, done, opts = { noLoader: false, compact: false }) {
1028
- await this.ready();
1029
- const fstore = await this.fileStore();
1030
- const params = {
1031
- encoder: (await fstore.keyedCrypto()).codec(),
1032
- carLog: this.carLog,
1033
- carStore: fstore,
1034
- WALStore: await this.WALStore(),
1035
- metaStore: await this.metaStore(),
1036
- threshold: this.ebOpts.threshold
1037
- };
1038
- return this.commitQueue.enqueue(async () => {
1039
- await this.cacheTransaction(t);
1040
- const ret = await commit(params, t, done, opts);
1041
- await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1042
- return ret.cgrp;
1043
- });
1044
- }
1045
- async updateCarLog(cids, fp, compact) {
1046
- if (compact) {
1047
- const previousCompactCid = fp.compact[fp.compact.length - 1];
1048
- fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1049
- this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1050
- await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1051
- } else {
1052
- this.carLog.unshift(cids);
1053
- }
1054
- }
1055
- async cacheTransaction(t) {
1056
- for await (const block of t.entries()) {
1057
- const sBlock = block.cid.toString();
1058
- if (!this.getBlockCache.has(sBlock)) {
1059
- this.getBlockCache.set(sBlock, block);
1060
- }
1061
- }
1062
- }
1063
- async cacheCarReader(carCidStr, reader) {
1064
- if (this.processedCars.has(carCidStr)) return;
1065
- this.processedCars.add(carCidStr);
1066
- for await (const block of reader.blocks()) {
1067
- const sBlock = block.cid.toString();
1068
- if (!this.getBlockCache.has(sBlock)) {
1069
- this.getBlockCache.set(sBlock, block);
1070
- }
1071
- }
1072
- }
1073
- async removeCidsForCompact(cid) {
1074
- const carHeader = await this.loadCarHeaderFromMeta({
1075
- cars: [cid]
1076
- });
1077
- for (const cids of carHeader.compact) {
1078
- for (const cid2 of cids) {
1079
- await (await this.carStore()).remove(cid2);
1080
- }
1081
- }
1082
- }
1083
- // async flushCars() {
1084
- // await this.ready
1085
- // // for each cid in car log, make a dbMeta
1086
- // for (const cid of this.carLog) {
1087
- // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1088
- // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1089
- // }
1090
- // }
1091
- async *entries(cache2 = true) {
1092
- await this.ready();
1093
- if (cache2) {
1094
- for (const [, block] of this.getBlockCache) {
1095
- yield block;
1096
- }
1097
- } else {
1098
- for (const [, block] of this.getBlockCache) {
1099
- yield block;
1100
- }
1101
- for (const cids of this.carLog) {
1102
- for (const cid of cids) {
1103
- const reader = await this.loadCar(cid);
1104
- if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1105
- for await (const block of reader.blocks()) {
1106
- const sCid = block.cid.toString();
1107
- if (!this.getBlockCache.has(sCid)) {
1108
- yield block;
1109
- }
1110
- }
1111
- }
1112
- }
1113
- }
1114
- }
1115
- async getBlock(cid) {
1116
- await this.ready();
1117
- const sCid = cid.toString();
1118
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1119
- const getCarCid = async (carCid) => {
1120
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1121
- const reader = await this.loadCar(carCid);
1122
- if (!reader) {
1123
- throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1124
- }
1125
- await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1126
- return;
1127
- });
1128
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1129
- throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1130
- };
1131
- const getCompactCarCids = async (carCid) => {
1132
- const reader = await this.loadCar(carCid);
1133
- if (!reader) {
1134
- throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1135
- }
1136
- const header = await parseCarFile(reader, this.logger);
1137
- const compacts = header.compact;
1138
- let got2;
1139
- const batchSize2 = 5;
1140
- for (let i = 0; i < compacts.length; i += batchSize2) {
1141
- const promises = [];
1142
- for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1143
- for (const cid2 of compacts[j]) {
1144
- promises.push(getCarCid(cid2));
1145
- }
1146
- }
1147
- try {
1148
- got2 = await Promise.any(promises);
1149
- } catch {
1150
- }
1151
- if (got2) break;
1152
- }
1153
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1154
- throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1155
- };
1156
- let got;
1157
- const batchSize = 5;
1158
- for (let i = 0; i < this.carLog.length; i += batchSize) {
1159
- const batch = this.carLog.slice(i, i + batchSize);
1160
- const promises = batch.flatMap((slice) => slice.map(getCarCid));
1161
- try {
1162
- got = await Promise.any(promises);
1163
- } catch {
1164
- }
1165
- if (got) break;
1166
- }
1167
- if (!got) {
1168
- try {
1169
- got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1170
- } catch {
1171
- }
1172
- }
1173
- return got;
945
+ async function create({
946
+ bytes,
947
+ cid,
948
+ hasher: hasher7,
949
+ codec: codec3
950
+ }) {
951
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
952
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
953
+ const value = await Promise.resolve(codec3.decode(bytes));
954
+ const hash = await hasher7.digest(bytes);
955
+ if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
956
+ throw new Error("CID hash does not match bytes");
1174
957
  }
1175
- async loadCar(cid) {
1176
- if (!this.carStore) {
1177
- throw this.logger.Error().Msg("car store not initialized").AsError();
1178
- }
1179
- const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1180
- return loaded;
958
+ return createUnsafe({
959
+ bytes,
960
+ cid,
961
+ value,
962
+ codec: codec3
963
+ });
964
+ }
965
+ async function createUnsafe({
966
+ bytes,
967
+ cid,
968
+ value: maybeValue,
969
+ codec: codec3
970
+ }) {
971
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
972
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
973
+ return new Block({
974
+ cid,
975
+ bytes,
976
+ value
977
+ });
978
+ }
979
+
980
+ // src/blockstore/loader-helpers.ts
981
+ import { sha256 as hasher2 } from "multiformats/hashes/sha2";
982
+ import * as dagCodec from "@ipld/dag-cbor";
983
+ async function parseCarFile(reader, logger) {
984
+ const roots = await reader.getRoots();
985
+ const header = await reader.get(roots[0]);
986
+ if (!header) throw logger.Error().Msg("missing header block").AsError();
987
+ const dec = await decode3({ bytes: header.bytes, hasher: hasher2, codec: dagCodec });
988
+ const fpvalue = dec.value;
989
+ if (fpvalue && !fpvalue.fp) {
990
+ throw logger.Error().Msg("missing fp").AsError();
1181
991
  }
1182
- async makeDecoderAndCarReader(cid, local, remote) {
1183
- const cidsString = cid.toString();
1184
- let loadedCar = void 0;
1185
- let activeStore = local;
1186
- try {
1187
- this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1188
- loadedCar = await local.load(cid);
1189
- this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1190
- } catch (e) {
1191
- if (remote) {
1192
- const remoteCar = await remote.load(cid);
1193
- if (remoteCar) {
1194
- this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1195
- await local.save(remoteCar);
1196
- loadedCar = remoteCar;
1197
- activeStore = remote;
1198
- }
1199
- } else {
1200
- this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1201
- }
1202
- }
1203
- if (!loadedCar) {
1204
- throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
992
+ return fpvalue.fp;
993
+ }
994
+
995
+ // src/blockstore/transaction.ts
996
+ import { MemoryBlockstore } from "@web3-storage/pail/block";
997
+ import { toCryptoRuntime as toCryptoRuntime2 } from "@adviser/cement";
998
+ var CarTransaction = class extends MemoryBlockstore {
999
+ constructor(parent, opts = { add: true, noLoader: false }) {
1000
+ super();
1001
+ if (opts.add) {
1002
+ parent.transactions.add(this);
1205
1003
  }
1206
- const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
1207
- const rawReader = await CarReader.fromBytes(bytes.value);
1208
- const readerP = Promise.resolve(rawReader);
1209
- const cachedReaderP = readerP.then(async (reader) => {
1210
- await this.cacheCarReader(cidsString, reader).catch((e) => {
1211
- this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1212
- return;
1213
- });
1214
- return reader;
1215
- });
1216
- this.carReaders.set(cidsString, cachedReaderP);
1217
- return readerP;
1004
+ this.parent = parent;
1218
1005
  }
1219
- //What if instead it returns an Array of CarHeader
1220
- async storesLoadCar(cid, local, remote) {
1221
- const cidsString = cid.toString();
1222
- let dacr = this.carReaders.get(cidsString);
1223
- if (!dacr) {
1224
- dacr = this.makeDecoderAndCarReader(cid, local, remote);
1225
- this.carReaders.set(cidsString, dacr);
1226
- }
1227
- return dacr;
1006
+ async get(cid) {
1007
+ return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
1228
1008
  }
1229
- async getMoreReaders(cids) {
1230
- const limit = pLimit(5);
1231
- const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1232
- await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1009
+ async superGet(cid) {
1010
+ return super.get(cid);
1233
1011
  }
1234
1012
  };
1235
-
1236
- // src/runtime/keyed-crypto.ts
1237
- var keyed_crypto_exports = {};
1238
- __export(keyed_crypto_exports, {
1239
- BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1240
- keyedCryptoFactory: () => keyedCryptoFactory
1241
- });
1242
- import { base58btc as base58btc2 } from "multiformats/bases/base58";
1243
- import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1244
- import * as CBOR from "cborg";
1245
- var generateIV = {
1246
- random: {
1013
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1014
+ const logger = ensureLogger(sthis, component, ctx);
1015
+ return {
1247
1016
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
1248
- calc: async (ko, crypto, data) => {
1249
- return crypto.randomBytes(ko.ivLength);
1017
+ applyMeta: (meta, snap) => {
1018
+ return Promise.resolve();
1250
1019
  },
1251
1020
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
1252
- verify: async (ko, crypto, iv, data) => {
1253
- return true;
1021
+ compact: async (blocks) => {
1022
+ return {};
1023
+ },
1024
+ autoCompact: 100,
1025
+ public: false,
1026
+ // name: undefined,
1027
+ threshold: 1e3 * 1e3,
1028
+ ...opts,
1029
+ logger,
1030
+ keyBag: opts.keyBag || {},
1031
+ crypto: toCryptoRuntime2(opts.crypto),
1032
+ storeUrls: opts.storeUrls,
1033
+ // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
1034
+ // store,
1035
+ storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
1036
+ };
1037
+ }
1038
+ function blockstoreFactory(sthis, opts) {
1039
+ return new EncryptedBlockstore(sthis, opts);
1040
+ }
1041
+ var BaseBlockstore = class {
1042
+ constructor(ebOpts) {
1043
+ this.transactions = /* @__PURE__ */ new Set();
1044
+ this.sthis = ensureSuperThis(ebOpts);
1045
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1046
+ this.sthis = ensureSuperThis(ebOpts);
1047
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1048
+ this.logger = this.ebOpts.logger;
1049
+ }
1050
+ // readonly name?: string;
1051
+ // ready: Promise<void>;
1052
+ ready() {
1053
+ return Promise.resolve();
1054
+ }
1055
+ async close() {
1056
+ }
1057
+ async destroy() {
1058
+ }
1059
+ async compact() {
1060
+ }
1061
+ async get(cid) {
1062
+ if (!cid) throw this.logger.Error().Msg("required cid").AsError();
1063
+ for (const f of this.transactions) {
1064
+ const v = await f.superGet(cid);
1065
+ if (v) return v;
1254
1066
  }
1255
- },
1256
- hash: {
1257
- calc: async (ko, crypto, data) => {
1258
- const hash = await hasher4.digest(data);
1259
- const hashBytes = new Uint8Array(hash.bytes);
1260
- const hashArray = new Uint8Array(ko.ivLength);
1261
- for (let i = 0; i < hashBytes.length; i++) {
1262
- hashArray[i % ko.ivLength] ^= hashBytes[i];
1067
+ }
1068
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1069
+ async put(cid, block) {
1070
+ throw this.logger.Error().Msg("use a transaction to put").AsError();
1071
+ }
1072
+ // TransactionMeta
1073
+ async transaction(fn, _opts) {
1074
+ this.logger.Debug().Msg("enter transaction");
1075
+ const t = new CarTransaction(this, _opts);
1076
+ this.logger.Debug().Msg("post CarTransaction");
1077
+ const done = await fn(t);
1078
+ this.logger.Debug().Msg("post fn");
1079
+ this.lastTxMeta = done;
1080
+ return { t, meta: done };
1081
+ }
1082
+ openTransaction(opts = { add: true, noLoader: false }) {
1083
+ return new CarTransaction(this, opts);
1084
+ }
1085
+ async commitTransaction(t, done, opts) {
1086
+ if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
1087
+ const cars = await this.loader?.commit(t, done, opts);
1088
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1089
+ setTimeout(() => void this.compact(), 10);
1090
+ }
1091
+ if (cars) {
1092
+ this.transactions.delete(t);
1093
+ return { meta: done, cars, t };
1094
+ }
1095
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
1096
+ }
1097
+ async *entries() {
1098
+ const seen = /* @__PURE__ */ new Set();
1099
+ for (const t of this.transactions) {
1100
+ for await (const blk of t.entries()) {
1101
+ if (seen.has(blk.cid.toString())) continue;
1102
+ seen.add(blk.cid.toString());
1103
+ yield blk;
1263
1104
  }
1264
- return hashArray;
1265
- },
1266
- verify: async function(ko, crypto, iv, data) {
1267
- return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1268
1105
  }
1269
1106
  }
1270
1107
  };
1271
- function getGenerateIVFn(url, opts) {
1272
- const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
1273
- return generateIV[ivhash] || generateIV["hash"];
1274
- }
1275
- var BlockIvKeyIdCodec = class {
1276
- constructor(ko, iv, opts) {
1277
- this.code = 3147065;
1278
- this.name = "Fireproof@encrypted-block:aes-gcm";
1279
- this.ko = ko;
1280
- this.iv = iv;
1281
- this.opts = opts || {};
1108
+ var EncryptedBlockstore = class extends BaseBlockstore {
1109
+ constructor(sthis, ebOpts) {
1110
+ super(ebOpts);
1111
+ this.compacting = false;
1112
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
1113
+ this: 1
1114
+ });
1115
+ this.loader = new Loader(sthis, ebOpts);
1282
1116
  }
1283
- async encode(data) {
1284
- const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1285
- const { iv } = this.ko.algo(calcIv);
1286
- const fprt = await this.ko.fingerPrint();
1287
- const keyId = base58btc2.decode(fprt);
1288
- this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1289
- return CBOR.encode({
1290
- iv,
1291
- keyId,
1292
- data: await this.ko._encrypt({ iv, bytes: data })
1117
+ ready() {
1118
+ return this.loader.ready();
1119
+ }
1120
+ close() {
1121
+ return this.loader.close();
1122
+ }
1123
+ destroy() {
1124
+ return this.loader.destroy();
1125
+ }
1126
+ async get(cid) {
1127
+ const got = await super.get(cid);
1128
+ if (got) return got;
1129
+ if (!this.loader) {
1130
+ return;
1131
+ }
1132
+ return falsyToUndef(await this.loader.getBlock(cid));
1133
+ }
1134
+ async transaction(fn, opts = { noLoader: false }) {
1135
+ this.logger.Debug().Msg("enter transaction");
1136
+ const { t, meta: done } = await super.transaction(fn);
1137
+ this.logger.Debug().Msg("post super.transaction");
1138
+ const cars = await this.loader.commit(t, done, opts);
1139
+ this.logger.Debug().Msg("post this.loader.commit");
1140
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1141
+ setTimeout(() => void this.compact(), 10);
1142
+ }
1143
+ if (cars) {
1144
+ this.transactions.delete(t);
1145
+ return { meta: done, cars, t };
1146
+ }
1147
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
1148
+ }
1149
+ async getFile(car, cid) {
1150
+ await this.ready();
1151
+ if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
1152
+ const reader = await this.loader.loadFileCar(
1153
+ car
1154
+ /*, isPublic */
1155
+ );
1156
+ const block = await reader.get(cid);
1157
+ if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
1158
+ return block.bytes;
1159
+ }
1160
+ async compact() {
1161
+ await this.ready();
1162
+ if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
1163
+ if (this.loader.carLog.length < 2) return;
1164
+ const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
1165
+ if (!compactFn || this.compacting) return;
1166
+ const blockLog = new CompactionFetcher(this);
1167
+ this.compacting = true;
1168
+ const meta = await compactFn(blockLog);
1169
+ await this.loader?.commit(blockLog.loggedBlocks, meta, {
1170
+ compact: true,
1171
+ noLoader: true
1293
1172
  });
1173
+ this.compacting = false;
1294
1174
  }
1295
- async decode(abytes) {
1296
- let bytes;
1297
- if (abytes instanceof Uint8Array) {
1298
- bytes = abytes;
1299
- } else {
1300
- bytes = new Uint8Array(abytes);
1175
+ async defaultCompact(blocks, logger) {
1176
+ if (!this.loader) {
1177
+ throw logger.Error().Msg("no loader").AsError();
1301
1178
  }
1302
- const { iv, keyId, data } = CBOR.decode(bytes);
1303
- const fprt = await this.ko.fingerPrint();
1304
- this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
1305
- if (base58btc2.encode(keyId) !== fprt) {
1306
- throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
1179
+ if (!this.lastTxMeta) {
1180
+ throw logger.Error().Msg("no lastTxMeta").AsError();
1307
1181
  }
1308
- const result = await this.ko._decrypt({ iv, bytes: data });
1309
- if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1310
- throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1182
+ for await (const blk of this.loader.entries(false)) {
1183
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1311
1184
  }
1312
- return result;
1313
- }
1314
- };
1315
- var keyedCrypto = class {
1316
- constructor(url, key, cyopt, sthis) {
1317
- this.ivLength = 12;
1318
- this.isEncrypting = true;
1319
- this.logger = ensureLogger(sthis, "keyedCrypto");
1320
- this.crypto = cyopt;
1321
- this.key = key;
1322
- this.url = url;
1323
- }
1324
- fingerPrint() {
1325
- return Promise.resolve(this.key.fingerPrint);
1326
- }
1327
- codec(iv, opts) {
1328
- return new BlockIvKeyIdCodec(this, iv, opts);
1185
+ for (const t of this.transactions) {
1186
+ for await (const blk of t.entries()) {
1187
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1188
+ }
1189
+ }
1190
+ return this.lastTxMeta;
1329
1191
  }
1330
- algo(iv) {
1331
- return {
1332
- name: "AES-GCM",
1333
- iv: iv || this.crypto.randomBytes(this.ivLength),
1334
- tagLength: 128
1335
- };
1192
+ async *entries() {
1193
+ for await (const blk of this.loader.entries()) {
1194
+ yield blk;
1195
+ }
1336
1196
  }
1337
- async _decrypt(data) {
1338
- this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
1339
- return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
1197
+ };
1198
+ var CompactionFetcher = class {
1199
+ constructor(blocks) {
1200
+ this.blockstore = blocks;
1201
+ this.loggedBlocks = new CarTransaction(blocks);
1340
1202
  }
1341
- async _encrypt(data) {
1342
- this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
1343
- const a = this.algo(data.iv);
1344
- return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
1203
+ async get(cid) {
1204
+ const block = await this.blockstore.get(cid);
1205
+ if (block) this.loggedBlocks.putSync(cid, block.bytes);
1206
+ return falsyToUndef(block);
1345
1207
  }
1346
1208
  };
1347
- var nullCodec = class {
1348
- constructor() {
1349
- this.code = 0;
1350
- this.name = "Fireproof@unencrypted-block";
1209
+
1210
+ // src/blockstore/commitor.ts
1211
+ import * as CBW from "@ipld/car/buffer-writer";
1212
+ import { sha256 as hasher3 } from "multiformats/hashes/sha2";
1213
+ import * as dagCodec2 from "@ipld/dag-cbor";
1214
+ async function encodeCarFile(roots, t, codec3) {
1215
+ let size = 0;
1216
+ const headerSize = CBW.headerLength({ roots });
1217
+ size += headerSize;
1218
+ for (const { cid, bytes } of t.entries()) {
1219
+ size += CBW.blockLength({ cid, bytes });
1351
1220
  }
1352
- encode(data) {
1353
- return data;
1221
+ const buffer = new Uint8Array(size);
1222
+ const writer = CBW.createWriter(buffer, { headerSize });
1223
+ for (const r of roots) {
1224
+ writer.addRoot(r);
1354
1225
  }
1355
- decode(data) {
1356
- return data;
1226
+ for (const { cid, bytes } of t.entries()) {
1227
+ writer.write({ cid, bytes });
1357
1228
  }
1358
- };
1359
- var noCrypto = class {
1360
- constructor(url, cyrt, sthis) {
1361
- this.ivLength = 0;
1362
- this.code = 0;
1363
- this.name = "Fireproof@unencrypted-block";
1364
- this.isEncrypting = false;
1365
- this._fingerPrint = "noCrypto:" + Math.random();
1366
- this.logger = ensureLogger(sthis, "noCrypto");
1367
- this.crypto = cyrt;
1368
- this.url = url;
1229
+ writer.close();
1230
+ return await encode3({ value: writer.bytes, hasher: hasher3, codec: codec3 });
1231
+ }
1232
+ async function createCarFile(encoder, cid, t) {
1233
+ return encodeCarFile([cid], t, encoder);
1234
+ }
1235
+ async function commitFiles(fileStore, walStore, t, done) {
1236
+ const { files: roots } = makeFileCarHeader(done);
1237
+ const cids = [];
1238
+ const codec3 = (await fileStore.keyedCrypto()).codec();
1239
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
1240
+ for (const car of cars) {
1241
+ const { cid, bytes } = car;
1242
+ await fileStore.save({ cid, bytes });
1243
+ await walStore.enqueueFile(
1244
+ cid
1245
+ /*, !!opts.public*/
1246
+ );
1247
+ cids.push(cid);
1369
1248
  }
1370
- fingerPrint() {
1371
- return Promise.resolve(this._fingerPrint);
1249
+ return cids;
1250
+ }
1251
+ function makeFileCarHeader(result) {
1252
+ const files = [];
1253
+ for (const [, meta] of Object.entries(result.files || {})) {
1254
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1255
+ files.push(meta.cid);
1256
+ }
1372
1257
  }
1373
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1374
- codec(iv) {
1375
- return new nullCodec();
1258
+ return { ...result, files };
1259
+ }
1260
+ async function prepareCarFilesFiles(encoder, roots, t) {
1261
+ return [await encodeCarFile(roots, t, encoder)];
1262
+ }
1263
+ function makeCarHeader(meta, cars, compact = false) {
1264
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1265
+ return { ...coreHeader, meta };
1266
+ }
1267
+ async function encodeCarHeader(fp) {
1268
+ return await encode3({
1269
+ value: { fp },
1270
+ hasher: hasher3,
1271
+ codec: dagCodec2
1272
+ });
1273
+ }
1274
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
1275
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
1276
+ const rootBlock = await encodeCarHeader(fp);
1277
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
1278
+ const cids = [];
1279
+ for (const car of cars) {
1280
+ const { cid, bytes } = car;
1281
+ await params.carStore.save({ cid, bytes });
1282
+ cids.push(cid);
1376
1283
  }
1377
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1378
- algo(iv) {
1379
- return {
1380
- name: "noCrypto",
1381
- iv: new Uint8Array(),
1382
- tagLength: 0
1383
- };
1284
+ const newDbMeta = { cars: cids };
1285
+ await params.WALStore.enqueue(newDbMeta, opts);
1286
+ await params.metaStore.save(newDbMeta);
1287
+ return { cgrp: cids, header: fp };
1288
+ }
1289
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
1290
+ const carFiles = [];
1291
+ threshold = threshold || 128e3 * 8;
1292
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1293
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1294
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
1295
+ let cidRootBlock = rootBlock;
1296
+ for (const { cid, bytes } of t.entries()) {
1297
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
1298
+ if (newsize >= threshold) {
1299
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1300
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1301
+ clonedt.putSync(cid, bytes);
1302
+ cidRootBlock = { cid, bytes };
1303
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
1304
+ } else {
1305
+ clonedt.putSync(cid, bytes);
1306
+ }
1384
1307
  }
1385
- _decrypt() {
1386
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1308
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1309
+ return carFiles;
1310
+ }
1311
+
1312
+ // src/blockstore/loader.ts
1313
+ import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1314
+
1315
+ // src/blockstore/task-manager.ts
1316
+ var TaskManager = class {
1317
+ constructor(sthis, callback) {
1318
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
1319
+ this.queue = [];
1320
+ this.isProcessing = false;
1321
+ this.logger = ensureLogger(sthis, "TaskManager");
1322
+ this.callback = callback;
1387
1323
  }
1388
- _encrypt() {
1389
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1324
+ async handleEvent(cid, parents, dbMeta) {
1325
+ for (const parent of parents) {
1326
+ this.eventsWeHandled.add(parent.toString());
1327
+ }
1328
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
1329
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
1330
+ void this.processQueue();
1390
1331
  }
1391
- };
1392
- async function keyedCryptoFactory(url, kb, sthis) {
1393
- const storekey = url.getParam("storekey");
1394
- if (storekey && storekey !== "insecure") {
1395
- let rkey = await kb.getNamedKey(storekey, true);
1396
- if (rkey.isErr()) {
1397
- try {
1398
- rkey = await kb.toKeyWithFingerPrint(storekey);
1399
- } catch (e) {
1400
- throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
1332
+ async processQueue() {
1333
+ if (this.isProcessing) return;
1334
+ this.isProcessing = true;
1335
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1336
+ const first = filteredQueue[0];
1337
+ if (!first) {
1338
+ return;
1339
+ }
1340
+ try {
1341
+ await this.callback(first.dbMeta);
1342
+ this.eventsWeHandled.add(first.cid);
1343
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1344
+ } catch (err) {
1345
+ if (first.retries++ > 3) {
1346
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
1347
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
1348
+ }
1349
+ await new Promise((resolve) => setTimeout(resolve, 50));
1350
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
1351
+ } finally {
1352
+ this.isProcessing = false;
1353
+ if (this.queue.length > 0) {
1354
+ void this.processQueue();
1401
1355
  }
1402
1356
  }
1403
- return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
1404
1357
  }
1405
- return new noCrypto(url, kb.rt.crypto, sthis);
1406
- }
1358
+ };
1407
1359
 
1408
- // src/blockstore/fragment-gateway.ts
1409
- import { Result as Result3 } from "@adviser/cement";
1410
- import { base58btc as base58btc3 } from "multiformats/bases/base58";
1411
- import { encode as encode3, decode as decode3 } from "cborg";
1412
- function getFragSize(url) {
1413
- const fragSize = url.getParam("fragSize");
1414
- let ret = 0;
1415
- if (fragSize) {
1416
- ret = parseInt(fragSize);
1417
- }
1418
- if (isNaN(ret) || ret <= 0) {
1419
- ret = 0;
1360
+ // src/blockstore/loader.ts
1361
+ function carLogIncludesGroup(list, cids) {
1362
+ return list.some((arr) => {
1363
+ return arr.toString() === cids.toString();
1364
+ });
1365
+ }
1366
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
1367
+ const byString = /* @__PURE__ */ new Map();
1368
+ for (const cid of list) {
1369
+ if (remove.has(cid.toString())) continue;
1370
+ byString.set(cid.toString(), cid);
1420
1371
  }
1421
- return ret;
1372
+ return [...byString.values()];
1422
1373
  }
1423
- async function getFrags(url, innerGW, headerSize, logger) {
1424
- const fragSize = getFragSize(url);
1425
- if (!fragSize) {
1426
- const res = await innerGW.get(url);
1427
- if (res.isErr()) {
1428
- return [res];
1429
- }
1430
- const data = res.unwrap();
1431
- return [
1432
- Result3.Ok({
1433
- fid: new Uint8Array(0),
1434
- ofs: 0,
1435
- len: data.length,
1436
- data
1374
+ var Loader = class {
1375
+ constructor(sthis, ebOpts) {
1376
+ this.commitQueue = new CommitQueue();
1377
+ this.isCompacting = false;
1378
+ this.carReaders = /* @__PURE__ */ new Map();
1379
+ this.seenCompacted = /* @__PURE__ */ new Set();
1380
+ this.processedCars = /* @__PURE__ */ new Set();
1381
+ this.carLog = [];
1382
+ this.getBlockCache = /* @__PURE__ */ new Map();
1383
+ this.seenMeta = /* @__PURE__ */ new Set();
1384
+ this.writeLimit = pLimit(1);
1385
+ this._carStore = new ResolveOnce2();
1386
+ this._fileStore = new ResolveOnce2();
1387
+ this._WALStore = new ResolveOnce2();
1388
+ this._metaStore = new ResolveOnce2();
1389
+ this.onceReady = new ResolveOnce2();
1390
+ this.sthis = sthis;
1391
+ this.ebOpts = defaultedBlockstoreRuntime(
1392
+ sthis,
1393
+ {
1394
+ ...ebOpts
1395
+ // name,
1396
+ },
1397
+ "Loader"
1398
+ );
1399
+ this.logger = this.ebOpts.logger;
1400
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
1401
+ await this.handleDbMetasFromStore([dbMeta]);
1402
+ });
1403
+ }
1404
+ async carStore() {
1405
+ return this._carStore.once(
1406
+ async () => this.ebOpts.storeRuntime.makeDataStore({
1407
+ sthis: this.sthis,
1408
+ url: this.ebOpts.storeUrls.data,
1409
+ keybag: await this.keyBag()
1437
1410
  })
1438
- ];
1411
+ );
1439
1412
  }
1440
- const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
1441
- if (firstRaw.isErr()) {
1442
- return [firstRaw];
1413
+ async fileStore() {
1414
+ return this._fileStore.once(
1415
+ async () => this.ebOpts.storeRuntime.makeDataStore({
1416
+ sthis: this.sthis,
1417
+ url: this.ebOpts.storeUrls.file,
1418
+ keybag: await this.keyBag()
1419
+ })
1420
+ );
1443
1421
  }
1444
- const firstFragment = decode3(firstRaw.unwrap());
1445
- const blockSize = firstFragment.data.length;
1446
- const ops = [Promise.resolve(Result3.Ok(firstFragment))];
1447
- const fidStr = base58btc3.encode(firstFragment.fid);
1448
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
1449
- for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
1450
- ops.push(
1451
- (async (furl, ofs2) => {
1452
- const raw2 = await innerGW.get(furl);
1453
- if (raw2.isErr()) {
1454
- return raw2;
1455
- }
1456
- const fragment = decode3(raw2.unwrap());
1457
- if (base58btc3.encode(fragment.fid) !== fidStr) {
1458
- return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
1459
- }
1460
- if (fragment.ofs !== ofs2) {
1461
- return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
1462
- }
1463
- return Result3.Ok(fragment);
1464
- })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
1422
+ async WALStore() {
1423
+ return this._WALStore.once(
1424
+ async () => this.ebOpts.storeRuntime.makeWALStore({
1425
+ sthis: this.sthis,
1426
+ url: this.ebOpts.storeUrls.wal,
1427
+ keybag: await this.keyBag()
1428
+ })
1465
1429
  );
1466
1430
  }
1467
- return Promise.all(ops);
1468
- }
1469
- var FragmentGateway = class {
1470
- constructor(sthis, innerGW) {
1471
- this.fidLength = 4;
1472
- this.headerSize = 32;
1473
- this.sthis = ensureSuperLog(sthis, "FragmentGateway");
1474
- this.logger = this.sthis.logger;
1475
- this.innerGW = innerGW;
1431
+ async metaStore() {
1432
+ return this._metaStore.once(
1433
+ async () => this.ebOpts.storeRuntime.makeMetaStore({
1434
+ sthis: this.sthis,
1435
+ url: this.ebOpts.storeUrls.meta,
1436
+ keybag: await this.keyBag()
1437
+ })
1438
+ );
1476
1439
  }
1477
- slicer(url, body) {
1478
- const fragSize = getFragSize(url);
1479
- if (!fragSize) {
1480
- return [this.innerGW.put(url, body)];
1440
+ keyBag() {
1441
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
1442
+ }
1443
+ async ready() {
1444
+ return this.onceReady.once(async () => {
1445
+ const metas = await (await this.metaStore()).load();
1446
+ if (this.ebOpts.meta) {
1447
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
1448
+ } else if (metas) {
1449
+ await this.handleDbMetasFromStore(metas);
1450
+ }
1451
+ });
1452
+ }
1453
+ async close() {
1454
+ await this.commitQueue.waitIdle();
1455
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1456
+ await Promise.all(toClose.map((store) => store.close()));
1457
+ }
1458
+ async destroy() {
1459
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1460
+ await Promise.all(toDestroy.map((store) => store.destroy()));
1461
+ }
1462
+ // async snapToCar(carCid: AnyLink | string) {
1463
+ // await this.ready
1464
+ // if (typeof carCid === 'string') {
1465
+ // carCid = CID.parse(carCid)
1466
+ // }
1467
+ // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
1468
+ // this.carLog = [carCid, ...carHeader.cars]
1469
+ // await this.getMoreReaders(carHeader.cars)
1470
+ // await this._applyCarHeader(carHeader, true)
1471
+ // }
1472
+ async handleDbMetasFromStore(metas) {
1473
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
1474
+ for (const meta of metas) {
1475
+ await this.writeLimit(async () => {
1476
+ await this.mergeDbMetaIntoClock(meta);
1477
+ });
1481
1478
  }
1482
- const blocksize = fragSize - this.headerSize;
1483
- if (blocksize <= 0) {
1484
- throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
1479
+ }
1480
+ async mergeDbMetaIntoClock(meta) {
1481
+ if (this.isCompacting) {
1482
+ throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1485
1483
  }
1486
- const ops = [];
1487
- const fid = this.sthis.nextId(this.fidLength);
1488
- const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
1489
- for (let ofs = 0; ofs < body.length; ofs += blocksize) {
1490
- const block = encode3({
1491
- fid: fid.bin,
1492
- ofs,
1493
- len: body.length,
1494
- data: body.slice(ofs, ofs + blocksize)
1495
- });
1496
- if (block.length > fragSize) {
1497
- throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
1498
- }
1499
- ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
1484
+ if (this.seenMeta.has(meta.cars.toString())) return;
1485
+ this.seenMeta.add(meta.cars.toString());
1486
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
1487
+ return;
1500
1488
  }
1501
- return ops;
1489
+ const carHeader = await this.loadCarHeaderFromMeta(meta);
1490
+ carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1491
+ await this.getMoreReaders(carHeader.cars.flat());
1492
+ this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1493
+ await this.ebOpts.applyMeta?.(carHeader.meta);
1502
1494
  }
1503
- buildUrl(baseUrl, key) {
1504
- return this.innerGW.buildUrl(baseUrl, key);
1495
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1496
+ // const { key } = meta;
1497
+ // if (key) {
1498
+ // await this.setKey(key);
1499
+ // }
1500
+ // }
1501
+ async loadCarHeaderFromMeta({ cars: cids }) {
1502
+ const reader = await this.loadCar(cids[0]);
1503
+ return await parseCarFile(reader, this.logger);
1505
1504
  }
1506
- async destroy(iurl) {
1507
- return this.innerGW.destroy(iurl);
1505
+ // async _getKey(): Promise<string | undefined> {
1506
+ // if (this.key) return this.key;
1507
+ // // generate a random key
1508
+ // if (!this.ebOpts.public) {
1509
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1510
+ // }
1511
+ // return this.key || undefined;
1512
+ // }
1513
+ async commitFiles(t, done) {
1514
+ await this.ready();
1515
+ const fstore = await this.fileStore();
1516
+ const wstore = await this.WALStore();
1517
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1508
1518
  }
1509
- async start(url) {
1510
- this.headerSize = encode3({
1511
- fid: this.sthis.nextId(this.fidLength).bin,
1512
- ofs: 1024 * 1024,
1513
- // 32bit
1514
- len: 16 * 1024 * 1024,
1515
- // 32bit
1516
- data: new Uint8Array(1024)
1517
- }).length - 1024;
1518
- return this.innerGW.start(url);
1519
+ async loadFileCar(cid) {
1520
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1519
1521
  }
1520
- async close(url) {
1521
- return this.innerGW.close(url);
1522
+ async commit(t, done, opts = { noLoader: false, compact: false }) {
1523
+ await this.ready();
1524
+ const carStore = await this.carStore();
1525
+ const params = {
1526
+ encoder: (await carStore.keyedCrypto()).codec(),
1527
+ carLog: this.carLog,
1528
+ carStore,
1529
+ WALStore: await this.WALStore(),
1530
+ metaStore: await this.metaStore(),
1531
+ threshold: this.ebOpts.threshold
1532
+ };
1533
+ return this.commitQueue.enqueue(async () => {
1534
+ await this.cacheTransaction(t);
1535
+ const ret = await commit(params, t, done, opts);
1536
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1537
+ return ret.cgrp;
1538
+ });
1522
1539
  }
1523
- async put(url, body) {
1524
- await Promise.all(this.slicer(url, body));
1525
- return Result3.Ok(void 0);
1540
+ async updateCarLog(cids, fp, compact) {
1541
+ if (compact) {
1542
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
1543
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1544
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1545
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1546
+ } else {
1547
+ this.carLog.unshift(cids);
1548
+ }
1549
+ }
1550
+ async cacheTransaction(t) {
1551
+ for await (const block of t.entries()) {
1552
+ const sBlock = block.cid.toString();
1553
+ if (!this.getBlockCache.has(sBlock)) {
1554
+ this.getBlockCache.set(sBlock, block);
1555
+ }
1556
+ }
1557
+ }
1558
+ async cacheCarReader(carCidStr, reader) {
1559
+ if (this.processedCars.has(carCidStr)) return;
1560
+ this.processedCars.add(carCidStr);
1561
+ for await (const block of reader.blocks()) {
1562
+ const sBlock = block.cid.toString();
1563
+ if (!this.getBlockCache.has(sBlock)) {
1564
+ this.getBlockCache.set(sBlock, block);
1565
+ }
1566
+ }
1526
1567
  }
1527
- async get(url) {
1528
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1529
- let buffer = void 0;
1530
- for (const rfrag of rfrags) {
1531
- if (rfrag.isErr()) {
1532
- return Result3.Err(rfrag.Err());
1568
+ async removeCidsForCompact(cid) {
1569
+ const carHeader = await this.loadCarHeaderFromMeta({
1570
+ cars: [cid]
1571
+ });
1572
+ for (const cids of carHeader.compact) {
1573
+ for (const cid2 of cids) {
1574
+ await (await this.carStore()).remove(cid2);
1533
1575
  }
1534
- const frag = rfrag.Ok();
1535
- buffer = buffer || new Uint8Array(frag.len);
1536
- buffer.set(frag.data, frag.ofs);
1537
1576
  }
1538
- return Result3.Ok(buffer || new Uint8Array(0));
1539
1577
  }
1540
- async subscribe(url, callback) {
1541
- if (this.innerGW.subscribe) {
1542
- return this.innerGW.subscribe(url, callback);
1578
+ // async flushCars() {
1579
+ // await this.ready
1580
+ // // for each cid in car log, make a dbMeta
1581
+ // for (const cid of this.carLog) {
1582
+ // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1583
+ // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1584
+ // }
1585
+ // }
1586
+ async *entries(cache2 = true) {
1587
+ await this.ready();
1588
+ if (cache2) {
1589
+ for (const [, block] of this.getBlockCache) {
1590
+ yield block;
1591
+ }
1543
1592
  } else {
1544
- return Result3.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
1593
+ for (const [, block] of this.getBlockCache) {
1594
+ yield block;
1595
+ }
1596
+ for (const cids of this.carLog) {
1597
+ for (const cid of cids) {
1598
+ const reader = await this.loadCar(cid);
1599
+ if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1600
+ for await (const block of reader.blocks()) {
1601
+ const sCid = block.cid.toString();
1602
+ if (!this.getBlockCache.has(sCid)) {
1603
+ yield block;
1604
+ }
1605
+ }
1606
+ }
1607
+ }
1545
1608
  }
1546
1609
  }
1547
- async delete(url) {
1548
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1549
- for (const rfrag of rfrags) {
1550
- if (rfrag.isErr()) {
1551
- return Result3.Err(rfrag.Err());
1610
+ async getBlock(cid) {
1611
+ await this.ready();
1612
+ const sCid = cid.toString();
1613
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1614
+ const getCarCid = async (carCid) => {
1615
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1616
+ const reader = await this.loadCar(carCid);
1617
+ if (!reader) {
1618
+ throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1552
1619
  }
1553
- const frag = rfrag.Ok();
1554
- const fidStr = base58btc3.encode(frag.fid);
1555
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
1556
- await this.innerGW.delete(fragUrl);
1620
+ await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1621
+ return;
1622
+ });
1623
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1624
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1625
+ };
1626
+ const getCompactCarCids = async (carCid) => {
1627
+ const reader = await this.loadCar(carCid);
1628
+ if (!reader) {
1629
+ throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1630
+ }
1631
+ const header = await parseCarFile(reader, this.logger);
1632
+ const compacts = header.compact;
1633
+ let got2;
1634
+ const batchSize2 = 5;
1635
+ for (let i = 0; i < compacts.length; i += batchSize2) {
1636
+ const promises = [];
1637
+ for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1638
+ for (const cid2 of compacts[j]) {
1639
+ promises.push(getCarCid(cid2));
1640
+ }
1641
+ }
1642
+ try {
1643
+ got2 = await Promise.any(promises);
1644
+ } catch {
1645
+ }
1646
+ if (got2) break;
1647
+ }
1648
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1649
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1650
+ };
1651
+ let got;
1652
+ const batchSize = 5;
1653
+ for (let i = 0; i < this.carLog.length; i += batchSize) {
1654
+ const batch = this.carLog.slice(i, i + batchSize);
1655
+ const promises = batch.flatMap((slice) => slice.map(getCarCid));
1656
+ try {
1657
+ got = await Promise.any(promises);
1658
+ } catch {
1659
+ }
1660
+ if (got) break;
1557
1661
  }
1558
- return Result3.Ok(void 0);
1559
- }
1560
- };
1561
-
1562
- // src/blockstore/meta-key-helper.ts
1563
- import { format, parse } from "@ipld/dag-json";
1564
- import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
1565
- import { CID as CID2 } from "multiformats";
1566
- import { base64pad } from "multiformats/bases/base64";
1567
- import { Result as Result4 } from "@adviser/cement";
1568
- async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
1569
- const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
1570
- if (!crdtEntries.length) {
1571
- sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
1572
- return [];
1662
+ if (!got) {
1663
+ try {
1664
+ got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1665
+ } catch {
1666
+ }
1667
+ }
1668
+ return got;
1573
1669
  }
1574
- if (!crdtEntries.map) {
1575
- sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
1576
- return [];
1670
+ async loadCar(cid) {
1671
+ if (!this.carStore) {
1672
+ throw this.logger.Error().Msg("car store not initialized").AsError();
1673
+ }
1674
+ const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1675
+ return loaded;
1577
1676
  }
1578
- return Promise.all(
1579
- crdtEntries.map(async (crdtEntry) => {
1580
- const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
1581
- const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
1582
- return {
1583
- eventCid: eventBlock.cid,
1584
- parents: crdtEntry.parents,
1585
- dbMeta
1586
- };
1587
- })
1588
- );
1589
- }
1590
- async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
1591
- try {
1592
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
1593
- const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
1594
- if (keyInfo.length) {
1595
- const dbMeta = keyInfo[0].dbMeta;
1596
- if (dbMeta.key) {
1597
- const kb = await getKeyBag(sthis);
1598
- const keyName = getStoreKeyName(uri);
1599
- const res = await kb.setNamedKey(keyName, dbMeta.key);
1600
- if (res.isErr()) {
1601
- sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
1602
- throw res.Err();
1677
+ async makeDecoderAndCarReader(cid, local, remote) {
1678
+ const cidsString = cid.toString();
1679
+ let loadedCar = void 0;
1680
+ let activeStore = local;
1681
+ try {
1682
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1683
+ loadedCar = await local.load(cid);
1684
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1685
+ } catch (e) {
1686
+ if (remote) {
1687
+ const remoteCar = await remote.load(cid);
1688
+ if (remoteCar) {
1689
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1690
+ await local.save(remoteCar);
1691
+ loadedCar = remoteCar;
1692
+ activeStore = remote;
1603
1693
  }
1694
+ } else {
1695
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1604
1696
  }
1605
- sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
1606
- return Result4.Ok(dbMeta);
1607
1697
  }
1608
- sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
1609
- return Result4.Ok(void 0);
1610
- } catch (error) {
1611
- sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
1612
- return Result4.Err(error);
1613
- }
1614
- }
1615
- async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
1616
- try {
1617
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
1618
- const keyName = getStoreKeyName(uri);
1619
- const kb = await getKeyBag(sthis);
1620
- const res = await kb.getNamedExtractableKey(keyName, true);
1621
- if (res.isErr()) {
1622
- sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
1623
- throw res.Err();
1698
+ if (!loadedCar) {
1699
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
1624
1700
  }
1625
- const keyData = await res.Ok().extract();
1626
- const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
1627
- const { dbMeta, parents } = dbMetas[0];
1628
- const parentLinks = parents.map((p) => CID2.parse(p));
1629
- dbMeta.key = keyData.keyStr;
1630
- const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
1631
- const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
1632
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
1633
- return Result4.Ok(encoded);
1634
- } catch (error) {
1635
- sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
1636
- return Result4.Err(error);
1701
+ const bytes = await decode3({ bytes: loadedCar.bytes, hasher: hasher4, codec: (await activeStore.keyedCrypto()).codec() });
1702
+ const rawReader = await CarReader.fromBytes(bytes.value);
1703
+ const readerP = Promise.resolve(rawReader);
1704
+ const cachedReaderP = readerP.then(async (reader) => {
1705
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
1706
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1707
+ return;
1708
+ });
1709
+ return reader;
1710
+ });
1711
+ this.carReaders.set(cidsString, cachedReaderP);
1712
+ return readerP;
1637
1713
  }
1638
- }
1639
- function getStoreKeyName(url) {
1640
- const storeKeyName = [url.getParam("localName") || url.getParam("name")];
1641
- const idx = url.getParam("index");
1642
- if (idx) {
1643
- storeKeyName.push(idx);
1714
+ //What if instead it returns an Array of CarHeader
1715
+ async storesLoadCar(cid, local, remote) {
1716
+ const cidsString = cid.toString();
1717
+ let dacr = this.carReaders.get(cidsString);
1718
+ if (!dacr) {
1719
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
1720
+ this.carReaders.set(cidsString, dacr);
1721
+ }
1722
+ return dacr;
1644
1723
  }
1645
- storeKeyName.push("data");
1646
- return `@${storeKeyName.join(":")}@`;
1647
- }
1648
- async function createDbMetaEventBlock(sthis, dbMeta, parents) {
1649
- const event = await EventBlock.create(
1650
- {
1651
- dbMeta: sthis.txt.encode(format(dbMeta))
1652
- },
1653
- parents
1654
- );
1655
- return event;
1656
- }
1657
- async function encodeEventsWithParents(sthis, events, parents) {
1658
- const crdtEntries = events.map((event) => {
1659
- const base64String = base64pad.encode(event.bytes);
1660
- return {
1661
- cid: event.cid.toString(),
1662
- data: base64String,
1663
- parents: parents.map((p) => p.toString())
1664
- };
1665
- });
1666
- return sthis.txt.encode(JSON.stringify(crdtEntries));
1667
- }
1724
+ async getMoreReaders(cids) {
1725
+ const limit = pLimit(5);
1726
+ const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1727
+ await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1728
+ }
1729
+ };
1668
1730
 
1669
1731
  // src/blockstore/store.ts
1670
1732
  function guardVersion(url) {
@@ -1674,16 +1736,21 @@ function guardVersion(url) {
1674
1736
  return Result5.Ok(url);
1675
1737
  }
1676
1738
  var BaseStoreImpl = class {
1677
- constructor(name, url, opts, sthis, logger) {
1739
+ constructor(sthis, url, opts, logger) {
1678
1740
  this._onStarted = [];
1679
1741
  this._onClosed = [];
1680
- this.name = name;
1681
1742
  this._url = url;
1682
1743
  this.keybag = opts.keybag;
1744
+ this.loader = opts.loader;
1683
1745
  this.sthis = sthis;
1684
- this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
1746
+ const name = this._url.getParam("name" /* NAME */);
1747
+ if (!name) {
1748
+ throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
1749
+ }
1750
+ this.name = name;
1751
+ this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
1752
+ this.realGateway = opts.gateway;
1685
1753
  this.gateway = new FragmentGateway(this.sthis, opts.gateway);
1686
- this.loader = opts.loader;
1687
1754
  }
1688
1755
  url() {
1689
1756
  return this._url;
@@ -1698,20 +1765,20 @@ var BaseStoreImpl = class {
1698
1765
  return;
1699
1766
  }
1700
1767
  async keyedCrypto() {
1701
- return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
1768
+ return keyedCryptoFactory(this._url, this.keybag, this.sthis);
1702
1769
  }
1703
1770
  async start() {
1704
1771
  this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
1705
- this._url = this._url.build().setParam("store", this.storeType).URI();
1772
+ this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
1706
1773
  const res = await this.gateway.start(this._url);
1707
1774
  if (res.isErr()) {
1708
1775
  this.logger.Error().Result("gw-start", res).Msg("started-gateway");
1709
1776
  return res;
1710
1777
  }
1711
1778
  this._url = res.Ok();
1712
- const kb = await this.keybag();
1779
+ const kb = await this.keybag;
1713
1780
  const skRes = await kb.ensureKeyFromUrl(this._url, () => {
1714
- const idx = this._url.getParam("index");
1781
+ const idx = this._url.getParam("index" /* INDEX */);
1715
1782
  const storeKeyName = [this.name];
1716
1783
  if (idx) {
1717
1784
  storeKeyName.push(idx);
@@ -1744,8 +1811,8 @@ var BaseStoreImpl = class {
1744
1811
  };
1745
1812
  var MetaStoreImpl = class extends BaseStoreImpl {
1746
1813
  // remote: boolean;
1747
- constructor(sthis, name, url, opts) {
1748
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
1814
+ constructor(sthis, url, opts) {
1815
+ super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
1749
1816
  this.storeType = "meta";
1750
1817
  this.subscribers = /* @__PURE__ */ new Map();
1751
1818
  this.parents = [];
@@ -1816,13 +1883,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
1816
1883
  return Result5.Ok(void 0);
1817
1884
  }
1818
1885
  async destroy() {
1886
+ this.logger.Debug().Msg("destroy");
1819
1887
  return this.gateway.destroy(this.url());
1820
1888
  }
1821
1889
  };
1822
1890
  var DataStoreImpl = class extends BaseStoreImpl {
1823
1891
  // readonly tag: string = "car-base";
1824
- constructor(sthis, name, url, opts) {
1825
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
1892
+ constructor(sthis, url, opts) {
1893
+ super(
1894
+ sthis,
1895
+ url,
1896
+ {
1897
+ ...opts
1898
+ },
1899
+ ensureLogger(sthis, "DataStoreImpl")
1900
+ );
1826
1901
  this.storeType = "data";
1827
1902
  }
1828
1903
  async load(cid) {
@@ -1863,23 +1938,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
1863
1938
  return Result5.Ok(void 0);
1864
1939
  }
1865
1940
  destroy() {
1941
+ this.logger.Debug().Msg("destroy");
1866
1942
  return this.gateway.destroy(this.url());
1867
1943
  }
1868
1944
  };
1869
1945
  var WALStoreImpl = class extends BaseStoreImpl {
1870
- constructor(loader, url, opts) {
1871
- super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
1946
+ constructor(sthis, url, opts) {
1947
+ super(
1948
+ sthis,
1949
+ url,
1950
+ {
1951
+ ...opts
1952
+ },
1953
+ ensureLogger(sthis, "WALStoreImpl")
1954
+ );
1872
1955
  this.storeType = "wal";
1956
+ // readonly tag: string = "rwal-base";
1957
+ // readonly loader: Loadable;
1873
1958
  this._ready = new ResolveOnce3();
1874
1959
  this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
1875
1960
  this.processing = void 0;
1876
1961
  this.processQueue = new CommitQueue();
1877
- this.loader = loader;
1878
1962
  }
1879
1963
  async ready() {
1880
1964
  return this._ready.once(async () => {
1881
1965
  const walState = await this.load().catch((e) => {
1882
- this.logger.Error().Any("error", e).Msg("error loading wal");
1966
+ this.logger.Error().Err(e).Msg("error loading wal");
1883
1967
  return void 0;
1884
1968
  });
1885
1969
  if (!walState) {
@@ -1912,7 +1996,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1912
1996
  }
1913
1997
  async process() {
1914
1998
  await this.ready();
1915
- if (!this.loader.remoteCarStore) return;
1999
+ if (!this.loader?.remoteCarStore) return;
1916
2000
  await this.processQueue.enqueue(async () => {
1917
2001
  try {
1918
2002
  await this._doProcess();
@@ -1925,73 +2009,88 @@ var WALStoreImpl = class extends BaseStoreImpl {
1925
2009
  });
1926
2010
  }
1927
2011
  async _doProcess() {
2012
+ if (!this.loader) return;
1928
2013
  if (!this.loader.remoteCarStore) return;
1929
- const rmlp = (async () => {
1930
- const operations = [...this.walState.operations];
1931
- const fileOperations = [...this.walState.fileOperations];
1932
- const uploads = [];
1933
- const noLoaderOps = [...this.walState.noLoaderOps];
1934
- const limit = pLimit2(3);
1935
- if (operations.length + fileOperations.length + noLoaderOps.length === 0) return;
1936
- for (const dbMeta of noLoaderOps) {
1937
- const uploadP = limit(async () => {
1938
- for (const cid of dbMeta.cars) {
1939
- const car = await (await this.loader.carStore()).load(cid);
1940
- if (!car) {
1941
- if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
1942
- throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
1943
- } else {
1944
- await throwFalsy(this.loader.remoteCarStore).save(car);
2014
+ const operations = [...this.walState.operations];
2015
+ const noLoaderOps = [...this.walState.noLoaderOps];
2016
+ const fileOperations = [...this.walState.fileOperations];
2017
+ if (operations.length + noLoaderOps.length + fileOperations.length === 0) return;
2018
+ const concurrencyLimit = 3;
2019
+ const retryableUpload = (fn, description) => pRetry(fn, {
2020
+ retries: 5,
2021
+ onFailedAttempt: (error) => {
2022
+ this.logger.Warn().Msg(`Attempt ${error.attemptNumber} failed for ${description}. There are ${error.retriesLeft} retries left.`);
2023
+ }
2024
+ });
2025
+ try {
2026
+ await pMap(
2027
+ noLoaderOps,
2028
+ async (dbMeta) => {
2029
+ await retryableUpload(async () => {
2030
+ if (!this.loader) return;
2031
+ for (const cid of dbMeta.cars) {
2032
+ const car = await (await this.loader.carStore()).load(cid);
2033
+ if (!car) {
2034
+ if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
2035
+ throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
2036
+ }
2037
+ } else {
2038
+ await throwFalsy(this.loader.remoteCarStore).save(car);
2039
+ }
1945
2040
  }
1946
2041
  this.walState.noLoaderOps = this.walState.noLoaderOps.filter((op) => op !== dbMeta);
1947
- }
1948
- });
1949
- uploads.push(uploadP);
1950
- }
1951
- for (const dbMeta of operations) {
1952
- const uploadP = limit(async () => {
1953
- for (const cid of dbMeta.cars) {
1954
- const car = await (await this.loader.carStore()).load(cid).catch(() => null);
1955
- if (!car) {
1956
- if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
1957
- throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
1958
- } else {
1959
- await throwFalsy(this.loader.remoteCarStore).save(car);
2042
+ }, `noLoaderOp with dbMeta.cars=${dbMeta.cars.toString()}`);
2043
+ },
2044
+ { concurrency: concurrencyLimit }
2045
+ );
2046
+ await pMap(
2047
+ operations,
2048
+ async (dbMeta) => {
2049
+ await retryableUpload(async () => {
2050
+ if (!this.loader) return;
2051
+ for (const cid of dbMeta.cars) {
2052
+ const car = await (await this.loader.carStore()).load(cid);
2053
+ if (!car) {
2054
+ if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
2055
+ throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
2056
+ }
2057
+ } else {
2058
+ await throwFalsy(this.loader.remoteCarStore).save(car);
2059
+ }
1960
2060
  }
1961
- }
1962
- this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
1963
- });
1964
- uploads.push(uploadP);
1965
- }
1966
- if (fileOperations.length) {
1967
- const dbLoader = this.loader;
1968
- for (const { cid: fileCid, public: publicFile } of fileOperations) {
1969
- const uploadP = limit(async () => {
1970
- const fileBlock = await (await dbLoader.fileStore()).load(fileCid);
1971
- await dbLoader.remoteFileStore?.save(fileBlock, { public: publicFile });
2061
+ this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
2062
+ }, `operation with dbMeta.cars=${dbMeta.cars.toString()}`);
2063
+ },
2064
+ { concurrency: concurrencyLimit }
2065
+ );
2066
+ await pMap(
2067
+ fileOperations,
2068
+ async ({ cid: fileCid, public: publicFile }) => {
2069
+ await retryableUpload(async () => {
2070
+ if (!this.loader) return;
2071
+ const fileBlock = await (await this.loader.fileStore()).load(fileCid);
2072
+ if (!fileBlock) {
2073
+ throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
2074
+ }
2075
+ await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
1972
2076
  this.walState.fileOperations = this.walState.fileOperations.filter((op) => op.cid !== fileCid);
1973
- });
1974
- uploads.push(uploadP);
1975
- }
1976
- }
1977
- try {
1978
- const res = await Promise.allSettled(uploads);
1979
- const errors = res.filter((r) => r.status === "rejected");
1980
- if (errors.length) {
1981
- throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
1982
- }
1983
- if (operations.length) {
1984
- const lastOp = operations[operations.length - 1];
1985
- await this.loader.remoteMetaStore?.save(lastOp).catch((e) => {
1986
- this.walState.operations.push(lastOp);
1987
- throw this.logger.Error().Any("error", e).Msg("error saving remote meta").AsError();
1988
- });
1989
- }
1990
- } finally {
1991
- await this.save(this.walState);
1992
- }
1993
- })();
1994
- await rmlp;
2077
+ }, `fileOperation with cid=${fileCid.toString()}`);
2078
+ },
2079
+ { concurrency: concurrencyLimit }
2080
+ );
2081
+ if (operations.length) {
2082
+ const lastOp = operations[operations.length - 1];
2083
+ await retryableUpload(async () => {
2084
+ if (!this.loader) return;
2085
+ await this.loader.remoteMetaStore?.save(lastOp);
2086
+ }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
2087
+ }
2088
+ } catch (error) {
2089
+ this.logger.Error().Any("error", error).Msg("Processing failed");
2090
+ return;
2091
+ } finally {
2092
+ await this.save(this.walState);
2093
+ }
1995
2094
  }
1996
2095
  async load() {
1997
2096
  this.logger.Debug().Msg("loading");
@@ -2008,6 +2107,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2008
2107
  }
2009
2108
  try {
2010
2109
  return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
2110
+ return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
2011
2111
  } catch (e) {
2012
2112
  throw this.logger.Error().Err(e).Msg("error parse").AsError();
2013
2113
  }
@@ -2034,72 +2134,92 @@ var WALStoreImpl = class extends BaseStoreImpl {
2034
2134
  return Result5.Ok(void 0);
2035
2135
  }
2036
2136
  destroy() {
2137
+ this.logger.Debug().Msg("destroy");
2037
2138
  return this.gateway.destroy(this.url());
2038
2139
  }
2039
2140
  };
2040
2141
 
2041
- // src/blockstore/store-factory.ts
2042
- function ensureIsIndex(url, isIndex) {
2043
- if (isIndex) {
2044
- return url.build().setParam("index", isIndex).URI();
2142
+ // src/blockstore/register-store-protocol.ts
2143
+ import { BuildURI, runtimeFn as runtimeFn2 } from "@adviser/cement";
2144
+
2145
+ // src/runtime/gateways/memory/gateway.ts
2146
+ import { Result as Result6 } from "@adviser/cement";
2147
+
2148
+ // src/runtime/gateways/memory/version.ts
2149
+ var MEMORY_VERSION = "v0.19-memory";
2150
+
2151
+ // src/runtime/gateways/memory/gateway.ts
2152
+ var MemoryGateway = class {
2153
+ constructor(memorys) {
2154
+ this.memorys = memorys;
2045
2155
  }
2046
- return url.build().delParam("index").URI();
2047
- }
2048
- function ensureName(name, url) {
2049
- if (!url.hasParam("name")) {
2050
- return url.build().setParam("name", name).URI();
2156
+ buildUrl(baseUrl, key) {
2157
+ return Promise.resolve(Result6.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
2051
2158
  }
2052
- return url;
2053
- }
2159
+ start(baseUrl) {
2160
+ return Promise.resolve(Result6.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
2161
+ }
2162
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2163
+ close(baseUrl) {
2164
+ return Promise.resolve(Result6.Ok(void 0));
2165
+ }
2166
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2167
+ destroy(baseUrl) {
2168
+ this.memorys.clear();
2169
+ return Promise.resolve(Result6.Ok(void 0));
2170
+ }
2171
+ put(url, body) {
2172
+ this.memorys.set(url.toString(), body);
2173
+ return Promise.resolve(Result6.Ok(void 0));
2174
+ }
2175
+ // get could return a NotFoundError if the key is not found
2176
+ get(url) {
2177
+ const x = this.memorys.get(url.toString());
2178
+ if (x === void 0) {
2179
+ return Promise.resolve(Result6.Err(new NotFoundError("not found")));
2180
+ }
2181
+ return Promise.resolve(Result6.Ok(x));
2182
+ }
2183
+ delete(url) {
2184
+ this.memorys.delete(url.toString());
2185
+ return Promise.resolve(Result6.Ok(void 0));
2186
+ }
2187
+ };
2188
+ var MemoryTestGateway = class {
2189
+ constructor(memorys) {
2190
+ this.memorys = memorys;
2191
+ }
2192
+ async get(url, key) {
2193
+ return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
2194
+ }
2195
+ };
2196
+
2197
+ // src/blockstore/register-store-protocol.ts
2054
2198
  var storeFactory = /* @__PURE__ */ new Map();
2055
- function buildURL(optURL, loader) {
2056
- const storeOpts = loader.ebOpts.store;
2057
- const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
2058
- let obuUrl;
2059
- if (obuItem && obuItem.overrideBaseURL) {
2060
- obuUrl = URI5.from(obuItem.overrideBaseURL);
2061
- }
2062
- const ret = ensureIsIndex(
2063
- URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2064
- storeOpts.isIndex
2065
- );
2066
- return ret;
2067
- }
2068
- var onceGateway = new KeyedResolvOnce2();
2069
- async function getGatewayFromURL(url, sthis) {
2070
- return onceGateway.get(url.toString()).once(async () => {
2071
- const item = storeFactory.get(url.protocol);
2072
- if (item) {
2073
- const ret = {
2074
- gateway: await item.gateway(sthis),
2075
- test: await item.test(sthis)
2076
- };
2077
- const res = await ret.gateway.start(url);
2078
- if (res.isErr()) {
2079
- sthis.logger.Error().Result("start", res).Msg("start failed");
2080
- return void 0;
2081
- }
2082
- return ret;
2199
+ function getDefaultURI(sthis, protocol) {
2200
+ if (protocol) {
2201
+ if (!protocol.endsWith(":")) {
2202
+ protocol += ":";
2083
2203
  }
2084
- sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2085
- return void 0;
2086
- });
2204
+ const gfi = storeFactory.get(protocol);
2205
+ if (gfi) {
2206
+ return gfi.defaultURI(sthis);
2207
+ }
2208
+ }
2209
+ const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
2210
+ if (!found) {
2211
+ throw sthis.logger.Error().Msg("no default found").AsError();
2212
+ }
2213
+ return found.defaultURI(sthis);
2087
2214
  }
2088
2215
  function registerStoreProtocol(item) {
2089
2216
  let protocol = item.protocol;
2090
2217
  if (!protocol.endsWith(":")) {
2091
2218
  protocol += ":";
2092
2219
  }
2093
- if (storeFactory.has(protocol)) {
2094
- if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
2095
- throw new Error(`we need a logger here`);
2096
- return () => {
2097
- };
2098
- }
2099
- }
2100
- if (item.overrideBaseURL) {
2220
+ if (item.isDefault) {
2101
2221
  Array.from(storeFactory.values()).forEach((items) => {
2102
- items.overrideBaseURL = void 0;
2222
+ items.isDefault = false;
2103
2223
  });
2104
2224
  }
2105
2225
  storeFactory.set(protocol, item);
@@ -2107,134 +2227,183 @@ function registerStoreProtocol(item) {
2107
2227
  storeFactory.delete(protocol);
2108
2228
  };
2109
2229
  }
2110
- var onceDataStoreFactory = new KeyedResolvOnce2();
2111
- async function dataStoreFactory(loader) {
2112
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
2113
- const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
2114
- return onceDataStoreFactory.get(url.toString()).once(async () => {
2115
- const gateway = await getGatewayFromURL(url, sthis);
2116
- if (!gateway) {
2117
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2230
+ function getGatewayFactoryItem(protocol) {
2231
+ return storeFactory.get(protocol);
2232
+ }
2233
+ function fileGatewayFactoryItem() {
2234
+ return {
2235
+ protocol: "file:",
2236
+ isDefault: true,
2237
+ defaultURI: (sthis) => {
2238
+ return BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
2239
+ },
2240
+ gateway: async (sthis) => {
2241
+ const { FileGateway } = await import("./gateway-GK5QZ6KP.js");
2242
+ return new FileGateway(sthis);
2243
+ },
2244
+ test: async (sthis) => {
2245
+ const { FileTestStore } = await import("./gateway-GK5QZ6KP.js");
2246
+ return new FileTestStore(sthis);
2247
+ }
2248
+ };
2249
+ }
2250
+ if (runtimeFn2().isBrowser) {
2251
+ registerStoreProtocol({
2252
+ protocol: "indexdb:",
2253
+ isDefault: true,
2254
+ defaultURI: () => {
2255
+ return BuildURI.from("indexdb://").pathname("fp").URI();
2256
+ },
2257
+ gateway: async (logger) => {
2258
+ const { IndexDBGateway } = await import("./gateway-TQTGDRCN.js");
2259
+ return new IndexDBGateway(logger);
2260
+ },
2261
+ test: async (logger) => {
2262
+ const { IndexDBTestStore } = await import("./gateway-TQTGDRCN.js");
2263
+ return new IndexDBTestStore(logger);
2118
2264
  }
2119
- const store = new DataStoreImpl(sthis, loader.name, url, {
2120
- gateway: gateway.gateway,
2121
- keybag: () => getKeyBag(loader.sthis, {
2122
- ...loader.ebOpts.keyBag
2123
- })
2124
- });
2125
- return store;
2126
2265
  });
2266
+ } else {
2267
+ registerStoreProtocol(fileGatewayFactoryItem());
2127
2268
  }
2128
- var onceMetaStoreFactory = new KeyedResolvOnce2();
2129
- async function metaStoreFactory(loader) {
2130
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
2131
- const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
2132
- return onceMetaStoreFactory.get(url.toString()).once(async () => {
2133
- sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
2134
- const gateway = await getGatewayFromURL(url, sthis);
2135
- if (!gateway) {
2136
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2137
- }
2138
- const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
2139
- gateway: gateway.gateway,
2140
- keybag: () => getKeyBag(loader.sthis, {
2141
- ...loader.ebOpts.keyBag
2142
- })
2143
- });
2144
- return store;
2269
+ var memory = /* @__PURE__ */ new Map();
2270
+ registerStoreProtocol({
2271
+ protocol: "memory:",
2272
+ isDefault: false,
2273
+ defaultURI: () => {
2274
+ return BuildURI.from("memory://").pathname("ram").URI();
2275
+ },
2276
+ gateway: async () => {
2277
+ return new MemoryGateway(memory);
2278
+ },
2279
+ test: async () => {
2280
+ return new MemoryTestGateway(memory);
2281
+ }
2282
+ });
2283
+
2284
+ // src/blockstore/store-factory.ts
2285
+ var onceGateway = new KeyedResolvOnce2();
2286
+ var gatewayInstances = new KeyedResolvOnce2();
2287
+ async function getStartedGateway(sthis, url) {
2288
+ return onceGateway.get(url.toString()).once(async () => {
2289
+ const item = getGatewayFactoryItem(url.protocol);
2290
+ if (item) {
2291
+ const ret = {
2292
+ url,
2293
+ ...await gatewayInstances.get(url.protocol).once(async () => ({
2294
+ gateway: await item.gateway(sthis),
2295
+ test: await item.test(sthis)
2296
+ }))
2297
+ };
2298
+ const res = await ret.gateway.start(url);
2299
+ if (res.isErr()) {
2300
+ return Result7.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
2301
+ }
2302
+ ret.url = res.Ok();
2303
+ return Result7.Ok(ret);
2304
+ }
2305
+ return Result7.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
2145
2306
  });
2146
2307
  }
2147
- var onceRemoteWalFactory = new KeyedResolvOnce2();
2148
- async function remoteWalFactory(loader) {
2149
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
2150
- const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
2151
- return onceRemoteWalFactory.get(url.toString()).once(async () => {
2152
- const gateway = await getGatewayFromURL(url, sthis);
2153
- if (!gateway) {
2154
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2155
- }
2156
- sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
2157
- const store = new WALStoreImpl(loader, url, {
2158
- gateway: gateway.gateway,
2159
- keybag: () => getKeyBag(loader.sthis, {
2160
- ...loader.ebOpts.keyBag
2161
- })
2162
- });
2163
- return store;
2308
+ async function dataStoreFactory(sfi) {
2309
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
2310
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2311
+ if (rgateway.isErr()) {
2312
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2313
+ }
2314
+ const gateway = rgateway.Ok();
2315
+ const store = new DataStoreImpl(sfi.sthis, gateway.url, {
2316
+ gateway: gateway.gateway,
2317
+ keybag: sfi.keybag
2318
+ });
2319
+ return store;
2320
+ }
2321
+ async function metaStoreFactory(sfi) {
2322
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
2323
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2324
+ if (rgateway.isErr()) {
2325
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2326
+ }
2327
+ const gateway = rgateway.Ok();
2328
+ const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
2329
+ gateway: gateway.gateway,
2330
+ keybag: sfi.keybag
2331
+ });
2332
+ return store;
2333
+ }
2334
+ async function WALStoreFactory(sfi) {
2335
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
2336
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2337
+ if (rgateway.isErr()) {
2338
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2339
+ }
2340
+ const gateway = rgateway.Ok();
2341
+ const store = new WALStoreImpl(sfi.sthis, gateway.url, {
2342
+ gateway: gateway.gateway,
2343
+ keybag: sfi.keybag
2164
2344
  });
2345
+ return store;
2165
2346
  }
2166
2347
  async function testStoreFactory(url, sthis) {
2167
- sthis = ensureSuperLog(sthis, "testStoreFactory");
2168
- const gateway = await getGatewayFromURL(url, sthis);
2169
- if (!gateway) {
2348
+ const rgateway = await getStartedGateway(sthis, url);
2349
+ if (!rgateway) {
2170
2350
  throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2171
2351
  }
2172
- return gateway.test;
2352
+ return rgateway.Ok().test;
2173
2353
  }
2174
- async function ensureStart(store, logger) {
2354
+ async function ensureStart(store) {
2175
2355
  const ret = await store.start();
2176
2356
  if (ret.isErr()) {
2177
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
2357
+ throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
2178
2358
  }
2179
- logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2359
+ store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2180
2360
  return store;
2181
2361
  }
2182
- function toStoreRuntime(opts, sthis) {
2183
- const logger = ensureLogger(sthis, "toStoreRuntime", {});
2362
+ function ensureStoreEnDeFile(ende) {
2363
+ ende = ende || {};
2184
2364
  return {
2185
- makeMetaStore: async (loader) => {
2186
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
2187
- return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
2188
- },
2189
- makeDataStore: async (loader) => {
2190
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
2191
- return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
2192
- },
2193
- makeWALStore: async (loader) => {
2194
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
2195
- return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
2196
- },
2197
- encodeFile: opts.encodeFile || encodeFile,
2198
- decodeFile: opts.decodeFile || decodeFile
2365
+ encodeFile: ende.encodeFile || encodeFile,
2366
+ decodeFile: ende.decodeFile || decodeFile
2367
+ };
2368
+ }
2369
+ function toStoreRuntime(sthis, endeOpts = {}) {
2370
+ return {
2371
+ makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
2372
+ // async (loader: Loadable) => {
2373
+ // logger
2374
+ // .Debug()
2375
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
2376
+ // .Msg("makeMetaStore");
2377
+ // return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
2378
+ // },
2379
+ makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
2380
+ // async (loader: Loadable) => {
2381
+ // logger
2382
+ // .Debug()
2383
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
2384
+ // .Msg("makeDataStore");
2385
+ // return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
2386
+ // },
2387
+ makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
2388
+ // async (loader: Loadable) => {
2389
+ // logger
2390
+ // .Debug()
2391
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
2392
+ // .Msg("makeRemoteWAL");
2393
+ // return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
2394
+ // },
2395
+ ...ensureStoreEnDeFile(endeOpts)
2199
2396
  };
2200
2397
  }
2201
- registerStoreProtocol({
2202
- protocol: "file:",
2203
- gateway: async (sthis) => {
2204
- const { FileGateway } = await import("./gateway-5FCWPX5W.js");
2205
- return new FileGateway(sthis);
2206
- },
2207
- test: async (sthis) => {
2208
- const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
2209
- return new FileTestStore(sthis);
2210
- }
2211
- });
2212
- registerStoreProtocol({
2213
- protocol: "indexdb:",
2214
- gateway: async (sthis) => {
2215
- const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
2216
- return new IndexDBGateway(sthis);
2217
- },
2218
- test: async (sthis) => {
2219
- const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
2220
- return new IndexDBTestStore(sthis);
2221
- }
2222
- });
2223
2398
 
2224
2399
  // src/blockstore/store-remote.ts
2225
2400
  async function RemoteDataStore(sthis, name, url, opts) {
2226
- const ds = new DataStoreImpl(sthis, name, url, opts);
2401
+ const ds = new DataStoreImpl(sthis, url, opts);
2227
2402
  await ds.start();
2228
2403
  return ds;
2229
2404
  }
2230
2405
  async function RemoteMetaStore(sthis, name, url, opts) {
2231
- const ms = new MetaStoreImpl(
2232
- sthis,
2233
- name,
2234
- url,
2235
- opts
2236
- /* , true*/
2237
- );
2406
+ const ms = new MetaStoreImpl(sthis, url, opts);
2238
2407
  await ms.start();
2239
2408
  return ms;
2240
2409
  }
@@ -2259,14 +2428,17 @@ var ConnectionBase = class {
2259
2428
  if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
2260
2429
  this.loader = loader;
2261
2430
  await this.onConnect();
2262
- const metaUrl = this.url.build().defParam("store", "meta").URI();
2263
- const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
2264
- if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2431
+ const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
2432
+ const rgateway = await getStartedGateway(loader.sthis, metaUrl);
2433
+ if (rgateway.isErr())
2434
+ throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2435
+ const name = metaUrl.toString();
2265
2436
  const dbName = metaUrl.getParam("name");
2266
- if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
2267
- const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
2437
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
2438
+ const gateway = rgateway.Ok();
2439
+ const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
2268
2440
  gateway: gateway.gateway,
2269
- keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
2441
+ keybag: await loader.keyBag(),
2270
2442
  loader
2271
2443
  });
2272
2444
  this.loader.remoteMetaStore = remote;
@@ -2279,14 +2451,15 @@ var ConnectionBase = class {
2279
2451
  async connectStorage_X({ loader }) {
2280
2452
  if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
2281
2453
  this.loader = loader;
2282
- const dataUrl = this.url.build().defParam("store", "data").URI();
2283
- const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
2284
- if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2454
+ const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
2455
+ const rgateway = await getStartedGateway(loader.sthis, dataUrl);
2456
+ if (rgateway.isErr())
2457
+ throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2285
2458
  const name = dataUrl.getParam("name");
2286
2459
  if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
2287
2460
  loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
2288
- gateway: gateway.gateway,
2289
- keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
2461
+ gateway: rgateway.Ok().gateway,
2462
+ keybag: await loader.keyBag()
2290
2463
  });
2291
2464
  loader.remoteFileStore = loader.remoteCarStore;
2292
2465
  }
@@ -2326,6 +2499,12 @@ var ConnectionBase = class {
2326
2499
  };
2327
2500
 
2328
2501
  // src/crdt-helpers.ts
2502
+ import { parse as parse3 } from "multiformats/link";
2503
+ import { sha256 as hasher5 } from "multiformats/hashes/sha2";
2504
+ import * as codec from "@ipld/dag-cbor";
2505
+ import { put, get, entries, root } from "@web3-storage/pail/crdt";
2506
+ import { EventFetcher, vis } from "@web3-storage/pail/clock";
2507
+ import * as Batch from "@web3-storage/pail/crdt/batch";
2329
2508
  function time(tag) {
2330
2509
  }
2331
2510
  function timeEnd(tag) {
@@ -2373,7 +2552,7 @@ async function writeDocContent(store, blocks, update, logger) {
2373
2552
  await processFiles(store, blocks, update.value, logger);
2374
2553
  value = { doc: update.value };
2375
2554
  }
2376
- const block = await encode({ value, hasher: hasher5, codec });
2555
+ const block = await encode3({ value, hasher: hasher5, codec });
2377
2556
  blocks.putSync(block.cid, block.bytes);
2378
2557
  return block.cid;
2379
2558
  }
@@ -2464,7 +2643,7 @@ function readFileset(blocks, files, isPublic = false) {
2464
2643
  async function getValueFromLink(blocks, link, logger) {
2465
2644
  const block = await blocks.get(link);
2466
2645
  if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
2467
- const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
2646
+ const { value } = await decode3({ bytes: block.bytes, hasher: hasher5, codec });
2468
2647
  const cvalue = {
2469
2648
  ...value,
2470
2649
  cid: link
@@ -2585,7 +2764,7 @@ async function doCompact(blockLog, head, logger) {
2585
2764
  async function getBlock(blocks, cidString) {
2586
2765
  const block = await blocks.get(parse3(cidString));
2587
2766
  if (!block) throw new Error(`Missing block ${cidString}`);
2588
- const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
2767
+ const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: hasher5 });
2589
2768
  return new Block({ cid, value, bytes: block.bytes });
2590
2769
  }
2591
2770
 
@@ -2643,7 +2822,8 @@ function makeProllyGetBlock(blocks) {
2643
2822
  return create({ cid, bytes, hasher: hasher6, codec: codec2 });
2644
2823
  };
2645
2824
  }
2646
- async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
2825
+ async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
2826
+ logger.Debug().Msg("enter bulkIndex");
2647
2827
  if (!indexEntries.length) return inIndex;
2648
2828
  if (!inIndex.root) {
2649
2829
  if (!inIndex.cid) {
@@ -2660,18 +2840,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
2660
2840
  returnNode = node;
2661
2841
  }
2662
2842
  if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
2843
+ logger.Debug().Msg("exit !root bulkIndex");
2663
2844
  return { root: returnNode, cid: returnRootBlock.cid };
2664
2845
  } else {
2665
2846
  inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
2666
2847
  }
2667
2848
  }
2849
+ logger.Debug().Msg("pre bulk bulkIndex");
2668
2850
  const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
2669
2851
  if (root3) {
2852
+ logger.Debug().Msg("pre root put bulkIndex");
2670
2853
  for await (const block of newBlocks) {
2671
2854
  await tblocks.put(block.cid, block.bytes);
2672
2855
  }
2673
2856
  return { root: root3, cid: (await root3.block).cid };
2674
2857
  } else {
2858
+ logger.Debug().Msg("pre !root bulkIndex");
2675
2859
  return { root: void 0, cid: void 0 };
2676
2860
  }
2677
2861
  }
@@ -2711,17 +2895,17 @@ function encodeKey(key) {
2711
2895
  }
2712
2896
 
2713
2897
  // src/indexer.ts
2714
- function index(sthis, { _crdt }, name, mapFn, meta) {
2715
- if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2716
- if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
2717
- if (_crdt.indexers.has(name)) {
2718
- const idx = _crdt.indexers.get(name);
2898
+ function index(refDb, name, mapFn, meta) {
2899
+ if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2900
+ if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
2901
+ if (refDb.crdt.indexers.has(name)) {
2902
+ const idx = refDb.crdt.indexers.get(name);
2719
2903
  idx.applyMapFn(name, mapFn, meta);
2720
2904
  } else {
2721
- const idx = new Index(sthis, _crdt, name, mapFn, meta);
2722
- _crdt.indexers.set(name, idx);
2905
+ const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
2906
+ refDb.crdt.indexers.set(name, idx);
2723
2907
  }
2724
- return _crdt.indexers.get(name);
2908
+ return refDb.crdt.indexers.get(name);
2725
2909
  }
2726
2910
  var Index = class {
2727
2911
  constructor(sthis, crdt, name, mapFn, meta) {
@@ -2740,18 +2924,9 @@ var Index = class {
2740
2924
  return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
2741
2925
  });
2742
2926
  }
2743
- close() {
2744
- return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
2745
- });
2746
- }
2747
- destroy() {
2748
- return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
2749
- });
2750
- }
2751
2927
  applyMapFn(name, mapFn, meta) {
2752
2928
  if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2753
2929
  if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
2754
- this.name = name;
2755
2930
  try {
2756
2931
  if (meta) {
2757
2932
  if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
@@ -2799,9 +2974,13 @@ var Index = class {
2799
2974
  }
2800
2975
  }
2801
2976
  async query(opts = {}) {
2977
+ this.logger.Debug().Msg("enter query");
2802
2978
  await this.ready();
2979
+ this.logger.Debug().Msg("post ready query");
2803
2980
  await this._updateIndex();
2981
+ this.logger.Debug().Msg("post _updateIndex query");
2804
2982
  await this._hydrateIndex();
2983
+ this.logger.Debug().Msg("post _hydrateIndex query");
2805
2984
  if (!this.byKey.root) {
2806
2985
  return await applyQuery(this.crdt, { result: [] }, opts);
2807
2986
  }
@@ -2857,13 +3036,16 @@ var Index = class {
2857
3036
  }
2858
3037
  async _updateIndex() {
2859
3038
  await this.ready();
3039
+ this.logger.Debug().Msg("enter _updateIndex");
2860
3040
  if (this.initError) throw this.initError;
2861
3041
  if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
2862
3042
  let result, head;
2863
3043
  if (!this.indexHead || this.indexHead.length === 0) {
2864
3044
  ({ result, head } = await this.crdt.allDocs());
3045
+ this.logger.Debug().Msg("enter crdt.allDocs");
2865
3046
  } else {
2866
3047
  ({ result, head } = await this.crdt.changes(this.indexHead));
3048
+ this.logger.Debug().Msg("enter crdt.changes");
2867
3049
  }
2868
3050
  if (result.length === 0) {
2869
3051
  this.indexHead = head;
@@ -2896,9 +3078,22 @@ var Index = class {
2896
3078
  if (result.length === 0) {
2897
3079
  return indexerMeta;
2898
3080
  }
3081
+ this.logger.Debug().Msg("pre this.blockstore.transaction");
2899
3082
  const { meta } = await this.blockstore.transaction(async (tblocks) => {
2900
- this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
2901
- this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
3083
+ this.byId = await bulkIndex(
3084
+ this.logger,
3085
+ tblocks,
3086
+ this.byId,
3087
+ removeIdIndexEntries.concat(byIdIndexEntries),
3088
+ byIdOpts
3089
+ );
3090
+ this.byKey = await bulkIndex(
3091
+ this.logger,
3092
+ tblocks,
3093
+ this.byKey,
3094
+ staleKeyIndexEntries.concat(indexEntries),
3095
+ byKeyOpts
3096
+ );
2902
3097
  this.indexHead = head;
2903
3098
  if (this.byId.cid && this.byKey.cid) {
2904
3099
  const idxMeta = {
@@ -2910,8 +3105,10 @@ var Index = class {
2910
3105
  };
2911
3106
  indexerMeta.indexes?.set(this.name, idxMeta);
2912
3107
  }
3108
+ this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
2913
3109
  return indexerMeta;
2914
3110
  });
3111
+ this.logger.Debug().Msg("post this.blockstore.transaction");
2915
3112
  return meta;
2916
3113
  }
2917
3114
  };
@@ -3039,21 +3236,23 @@ var CRDTClock = class {
3039
3236
  throw this.logger.Error().Msg("missing blockstore").AsError();
3040
3237
  }
3041
3238
  await validateBlocks(this.logger, newHead, this.blockstore);
3042
- const { meta } = await this.blockstore.transaction(
3043
- async (tblocks) => {
3044
- const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3045
- const result = await root2(tblocks, advancedHead);
3046
- for (const { cid, bytes } of [
3047
- ...result.additions
3048
- // ...result.removals
3049
- ]) {
3050
- tblocks.putSync(cid, bytes);
3051
- }
3052
- return { head: advancedHead };
3053
- },
3054
- { noLoader, add: false }
3055
- );
3056
- this.setHead(meta.head);
3239
+ if (!this.transaction) {
3240
+ this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
3241
+ }
3242
+ const tblocks = this.transaction;
3243
+ const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3244
+ const result = await root2(tblocks, advancedHead);
3245
+ for (const { cid, bytes } of [
3246
+ ...result.additions
3247
+ // ...result.removals
3248
+ ]) {
3249
+ tblocks.putSync(cid, bytes);
3250
+ }
3251
+ if (!noLoader) {
3252
+ await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
3253
+ this.transaction = void 0;
3254
+ }
3255
+ this.setHead(advancedHead);
3057
3256
  }
3058
3257
  };
3059
3258
  function sortClockHead(clockHead) {
@@ -3085,15 +3284,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
3085
3284
 
3086
3285
  // src/crdt.ts
3087
3286
  var CRDT = class {
3088
- constructor(sthis, name, opts = {}) {
3287
+ constructor(sthis, opts) {
3089
3288
  this.indexers = /* @__PURE__ */ new Map();
3090
3289
  this.onceReady = new ResolveOnce5();
3091
3290
  this.sthis = sthis;
3092
- this.name = name;
3093
3291
  this.logger = ensureLogger(sthis, "CRDT");
3094
3292
  this.opts = opts;
3095
3293
  this.blockstore = blockstoreFactory(sthis, {
3096
- name,
3097
3294
  applyMeta: async (meta) => {
3098
3295
  const crdtMeta = meta;
3099
3296
  if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
@@ -3103,23 +3300,27 @@ var CRDT = class {
3103
3300
  await doCompact(blocks, this.clock.head, this.logger);
3104
3301
  return { head: this.clock.head };
3105
3302
  },
3106
- autoCompact: this.opts.autoCompact || 100,
3107
- store: { ...this.opts.store, isIndex: void 0 },
3108
- public: this.opts.public,
3109
- meta: this.opts.meta,
3110
- threshold: this.opts.threshold
3303
+ // autoCompact: this.opts.autoCompact || 100,
3304
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
3305
+ storeUrls: this.opts.storeUrls.data,
3306
+ keyBag: this.opts.keyBag,
3307
+ // public: this.opts.public,
3308
+ meta: this.opts.meta
3309
+ // threshold: this.opts.threshold,
3111
3310
  });
3112
3311
  this.indexBlockstore = blockstoreFactory(sthis, {
3113
- name,
3312
+ // name: opts.name,
3114
3313
  applyMeta: async (meta) => {
3115
3314
  const idxCarMeta = meta;
3116
3315
  if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
3117
- for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
3118
- index(this.sthis, { _crdt: this }, name2, void 0, idx);
3316
+ for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
3317
+ index({ crdt: this }, name, void 0, idx);
3119
3318
  }
3120
3319
  },
3121
- store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
3122
- public: this.opts.public
3320
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
3321
+ storeUrls: this.opts.storeUrls.idx,
3322
+ keyBag: this.opts.keyBag
3323
+ // public: this.opts.public,
3123
3324
  });
3124
3325
  this.clock = new CRDTClock(this.blockstore);
3125
3326
  this.clock.onZoom(() => {
@@ -3201,51 +3402,164 @@ var CRDT = class {
3201
3402
  };
3202
3403
 
3203
3404
  // src/database.ts
3204
- var Database = class {
3205
- constructor(name, opts) {
3206
- this.opts = {};
3405
+ var databases = new KeyedResolvOnce3();
3406
+ function toSortedArray(set) {
3407
+ if (!set) return [];
3408
+ return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
3409
+ }
3410
+ function keyConfigOpts(sthis, name, opts) {
3411
+ return JSON.stringify(
3412
+ toSortedArray({
3413
+ name,
3414
+ stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
3415
+ })
3416
+ );
3417
+ }
3418
+ function isDatabase(db) {
3419
+ return db instanceof DatabaseImpl || db instanceof DatabaseShell;
3420
+ }
3421
+ function DatabaseFactory(name, opts) {
3422
+ const sthis = ensureSuperThis(opts);
3423
+ return new DatabaseShell(
3424
+ databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
3425
+ const db = new DatabaseImpl(sthis, {
3426
+ name,
3427
+ meta: opts?.meta,
3428
+ keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
3429
+ storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
3430
+ storeEnDe: {
3431
+ encodeFile,
3432
+ decodeFile,
3433
+ ...opts?.storeEnDe
3434
+ }
3435
+ });
3436
+ db.onClosed(() => {
3437
+ databases.unget(key);
3438
+ });
3439
+ return db;
3440
+ })
3441
+ );
3442
+ }
3443
+ var DatabaseShell = class {
3444
+ constructor(ref) {
3445
+ this.ref = ref;
3446
+ ref.addShell(this);
3447
+ }
3448
+ get id() {
3449
+ return this.ref.id;
3450
+ }
3451
+ get logger() {
3452
+ return this.ref.logger;
3453
+ }
3454
+ get sthis() {
3455
+ return this.ref.sthis;
3456
+ }
3457
+ get crdt() {
3458
+ return this.ref.crdt;
3459
+ }
3460
+ name() {
3461
+ return this.ref.name();
3462
+ }
3463
+ onClosed(fn) {
3464
+ return this.ref.onClosed(fn);
3465
+ }
3466
+ close() {
3467
+ return this.ref.shellClose(this);
3468
+ }
3469
+ destroy() {
3470
+ return this.ref.destroy();
3471
+ }
3472
+ ready() {
3473
+ return this.ref.ready();
3474
+ }
3475
+ get(id) {
3476
+ return this.ref.get(id);
3477
+ }
3478
+ put(doc) {
3479
+ return this.ref.put(doc);
3480
+ }
3481
+ del(id) {
3482
+ return this.ref.del(id);
3483
+ }
3484
+ changes(since, opts) {
3485
+ return this.ref.changes(since, opts);
3486
+ }
3487
+ allDocs(opts) {
3488
+ return this.ref.allDocs(opts);
3489
+ }
3490
+ allDocuments() {
3491
+ return this.ref.allDocuments();
3492
+ }
3493
+ subscribe(listener, updates) {
3494
+ return this.ref.subscribe(listener, updates);
3495
+ }
3496
+ query(field, opts) {
3497
+ return this.ref.query(field, opts);
3498
+ }
3499
+ compact() {
3500
+ return this.ref.compact();
3501
+ }
3502
+ };
3503
+ var DatabaseImpl = class {
3504
+ constructor(sthis, opts) {
3207
3505
  this._listening = false;
3208
3506
  this._listeners = /* @__PURE__ */ new Set();
3209
3507
  this._noupdate_listeners = /* @__PURE__ */ new Set();
3508
+ // readonly blockstore: BaseBlockstore;
3509
+ this.shells = /* @__PURE__ */ new Set();
3510
+ this._onClosedFns = /* @__PURE__ */ new Set();
3210
3511
  this._ready = new ResolveOnce6();
3211
- this.name = name;
3212
- this.opts = opts || this.opts;
3213
- this.sthis = ensureSuperThis(this.opts);
3512
+ this.opts = opts;
3513
+ this.sthis = sthis;
3514
+ this.id = sthis.timeOrderedNextId().str;
3214
3515
  this.logger = ensureLogger(this.sthis, "Database");
3215
- this._crdt = new CRDT(this.sthis, name, this.opts);
3216
- this.blockstore = this._crdt.blockstore;
3516
+ this.crdt = new CRDT(this.sthis, this.opts);
3217
3517
  this._writeQueue = writeQueue(async (updates) => {
3218
- return await this._crdt.bulk(updates);
3518
+ return await this.crdt.bulk(updates);
3219
3519
  });
3220
- this._crdt.clock.onTock(() => {
3520
+ this.crdt.clock.onTock(() => {
3221
3521
  this._no_update_notify();
3222
3522
  });
3223
3523
  }
3224
- static {
3225
- this.databases = /* @__PURE__ */ new Map();
3524
+ addShell(shell) {
3525
+ this.shells.add(shell);
3526
+ }
3527
+ onClosed(fn) {
3528
+ this._onClosedFns.add(fn);
3226
3529
  }
3227
3530
  async close() {
3228
- await this.ready();
3229
- await this._crdt.close();
3230
- await this.blockstore.close();
3531
+ throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
3532
+ }
3533
+ async shellClose(db) {
3534
+ if (!this.shells.has(db)) {
3535
+ throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
3536
+ }
3537
+ this.shells.delete(db);
3538
+ if (this.shells.size === 0) {
3539
+ await this.ready();
3540
+ await this.crdt.close();
3541
+ this._onClosedFns.forEach((fn) => fn());
3542
+ }
3231
3543
  }
3232
3544
  async destroy() {
3233
3545
  await this.ready();
3234
- await this._crdt.destroy();
3235
- await this.blockstore.destroy();
3546
+ await this.crdt.destroy();
3236
3547
  }
3237
3548
  async ready() {
3238
- return this._ready.once(async () => {
3549
+ const ret = await this._ready.once(async () => {
3239
3550
  await this.sthis.start();
3240
- await this._crdt.ready();
3241
- await this.blockstore.ready();
3551
+ await this.crdt.ready();
3242
3552
  });
3553
+ return ret;
3554
+ }
3555
+ name() {
3556
+ return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
3243
3557
  }
3244
3558
  async get(id) {
3245
- if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
3559
+ if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
3246
3560
  await this.ready();
3247
3561
  this.logger.Debug().Str("id", id).Msg("get");
3248
- const got = await this._crdt.get(id).catch((e) => {
3562
+ const got = await this.crdt.get(id).catch((e) => {
3249
3563
  throw new NotFoundError(`Not found: ${id} - ${e.message}`);
3250
3564
  });
3251
3565
  if (!got) throw new NotFoundError(`Not found: ${id}`);
@@ -3264,34 +3578,34 @@ var Database = class {
3264
3578
  _id: docId
3265
3579
  }
3266
3580
  });
3267
- return { id: docId, clock: result?.head, name: this.name };
3581
+ return { id: docId, clock: result?.head, name: this.name() };
3268
3582
  }
3269
3583
  async del(id) {
3270
3584
  await this.ready();
3271
3585
  this.logger.Debug().Str("id", id).Msg("del");
3272
3586
  const result = await this._writeQueue.push({ id, del: true });
3273
- return { id, clock: result?.head, name: this.name };
3587
+ return { id, clock: result?.head, name: this.name() };
3274
3588
  }
3275
3589
  async changes(since = [], opts = {}) {
3276
3590
  await this.ready();
3277
3591
  this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
3278
- const { result, head } = await this._crdt.changes(since, opts);
3592
+ const { result, head } = await this.crdt.changes(since, opts);
3279
3593
  const rows = result.map(({ id: key, value, del, clock }) => ({
3280
3594
  key,
3281
3595
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
3282
3596
  clock
3283
3597
  }));
3284
- return { rows, clock: head, name: this.name };
3598
+ return { rows, clock: head, name: this.name() };
3285
3599
  }
3286
3600
  async allDocs(opts = {}) {
3287
3601
  await this.ready();
3288
3602
  this.logger.Debug().Msg("allDocs");
3289
- const { result, head } = await this._crdt.allDocs();
3603
+ const { result, head } = await this.crdt.allDocs();
3290
3604
  const rows = result.map(({ id: key, value, del }) => ({
3291
3605
  key,
3292
3606
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
3293
3607
  }));
3294
- return { rows, clock: head, name: this.name };
3608
+ return { rows, clock: head, name: this.name() };
3295
3609
  }
3296
3610
  async allDocuments() {
3297
3611
  return this.allDocs();
@@ -3301,7 +3615,7 @@ var Database = class {
3301
3615
  if (updates) {
3302
3616
  if (!this._listening) {
3303
3617
  this._listening = true;
3304
- this._crdt.clock.onTick((updates2) => {
3618
+ this.crdt.clock.onTick((updates2) => {
3305
3619
  void this._notify(updates2);
3306
3620
  });
3307
3621
  }
@@ -3320,13 +3634,13 @@ var Database = class {
3320
3634
  async query(field, opts = {}) {
3321
3635
  await this.ready();
3322
3636
  this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
3323
- const _crdt = this._crdt;
3324
- const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
3637
+ const _crdt = this.crdt;
3638
+ const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
3325
3639
  return await idx.query(opts);
3326
3640
  }
3327
3641
  async compact() {
3328
3642
  await this.ready();
3329
- await this._crdt.compact();
3643
+ await this.crdt.compact();
3330
3644
  }
3331
3645
  async _notify(updates) {
3332
3646
  await this.ready();
@@ -3350,23 +3664,62 @@ var Database = class {
3350
3664
  }
3351
3665
  }
3352
3666
  };
3353
- function toSortedArray(set) {
3354
- if (!set) return [];
3355
- return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
3667
+ function defaultURI(sthis, curi, uri, store, ctx) {
3668
+ ctx = ctx || {};
3669
+ const ret = (curi ? URI8.from(curi) : uri).build().setParam("store" /* STORE */, store);
3670
+ if (!ret.hasParam("name" /* NAME */)) {
3671
+ const name = sthis.pathOps.basename(ret.URI().pathname);
3672
+ if (!name) {
3673
+ throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
3674
+ }
3675
+ ret.setParam("name" /* NAME */, name);
3676
+ }
3677
+ if (ctx.idx) {
3678
+ ret.defParam("index" /* INDEX */, "idx");
3679
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
3680
+ } else {
3681
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
3682
+ }
3683
+ if (store === "data") {
3684
+ if (ctx.file) {
3685
+ } else {
3686
+ ret.defParam("suffix" /* SUFFIX */, ".car");
3687
+ }
3688
+ }
3689
+ return ret.URI();
3356
3690
  }
3357
- function fireproof(name, opts) {
3358
- const key = JSON.stringify(
3359
- toSortedArray({
3360
- name,
3361
- stores: toSortedArray(opts?.store?.stores)
3362
- })
3363
- );
3364
- let db = Database.databases.get(key);
3365
- if (!db) {
3366
- db = new Database(name, opts);
3367
- Database.databases.set(key, db);
3691
+ function toStoreURIRuntime(sthis, name, sopts) {
3692
+ sopts = sopts || {};
3693
+ if (!sopts.base) {
3694
+ const fp_env = sthis.env.get("FP_STORAGE_URL");
3695
+ if (fp_env) {
3696
+ sopts = { ...sopts, base: BuildURI2.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
3697
+ } else {
3698
+ sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
3699
+ }
3368
3700
  }
3369
- return db;
3701
+ const bbase = BuildURI2.from(sopts.base);
3702
+ if (name) {
3703
+ bbase.setParam("name" /* NAME */, name);
3704
+ }
3705
+ const base = bbase.URI();
3706
+ return {
3707
+ idx: {
3708
+ data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
3709
+ file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
3710
+ meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
3711
+ wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
3712
+ },
3713
+ data: {
3714
+ data: defaultURI(sthis, sopts.data?.data, base, "data"),
3715
+ file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
3716
+ meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
3717
+ wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
3718
+ }
3719
+ };
3720
+ }
3721
+ function fireproof(name, opts) {
3722
+ return DatabaseFactory(name, opts);
3370
3723
  }
3371
3724
  function makeName(fnString) {
3372
3725
  const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
@@ -3397,7 +3750,7 @@ __export(runtime_exports, {
3397
3750
  kb: () => key_bag_exports,
3398
3751
  kc: () => keyed_crypto_exports,
3399
3752
  mf: () => wait_pr_multiformats_exports,
3400
- runtimeFn: () => runtimeFn2,
3753
+ runtimeFn: () => runtimeFn3,
3401
3754
  toArrayBuffer: () => toArrayBuffer
3402
3755
  });
3403
3756
 
@@ -3412,23 +3765,24 @@ __export(wait_pr_multiformats_exports, {
3412
3765
  var codec_interface_exports = {};
3413
3766
 
3414
3767
  // src/runtime/index.ts
3415
- import { runtimeFn as runtimeFn2 } from "@adviser/cement";
3768
+ import { runtimeFn as runtimeFn3 } from "@adviser/cement";
3416
3769
 
3417
3770
  // src/version.ts
3418
3771
  var PACKAGE_VERSION = Object.keys({
3419
- "0.19.99": "xxxx"
3772
+ "0.19.101": "xxxx"
3420
3773
  })[0];
3421
3774
  export {
3422
3775
  CRDT,
3423
- Database,
3776
+ DatabaseFactory,
3777
+ DatabaseShell,
3424
3778
  Index,
3425
3779
  NotFoundError,
3426
3780
  PACKAGE_VERSION,
3781
+ PARAM,
3427
3782
  Result,
3428
3783
  UInt8ArrayEqual,
3429
3784
  blockstore_exports as blockstore,
3430
3785
  blockstore_exports as bs,
3431
- dataDir,
3432
3786
  ensureLogger,
3433
3787
  ensureSuperLog,
3434
3788
  ensureSuperThis,
@@ -3439,10 +3793,13 @@ export {
3439
3793
  getName,
3440
3794
  getStore,
3441
3795
  index,
3796
+ isDatabase,
3442
3797
  isFalsy,
3443
3798
  isNotFoundError,
3799
+ keyConfigOpts,
3444
3800
  runtime_exports as rt,
3445
3801
  runtime_exports as runtime,
3446
- throwFalsy
3802
+ throwFalsy,
3803
+ toStoreURIRuntime
3447
3804
  };
3448
3805
  //# sourceMappingURL=index.js.map