@fireproof/core 0.19.100 → 0.19.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/{chunk-OFGPKRCM.js → chunk-3EB3ENHT.js} +54 -25
  2. package/chunk-3EB3ENHT.js.map +1 -0
  3. package/chunk-HQ7D3PEU.js +61 -0
  4. package/chunk-HQ7D3PEU.js.map +1 -0
  5. package/chunk-PZ5AY32C.js +10 -0
  6. package/deno-filesystem-Q2IJ7YDR.js +57 -0
  7. package/deno-filesystem-Q2IJ7YDR.js.map +1 -0
  8. package/{gateway-5FCWPX5W.js → gateway-GK5QZ6KP.js} +13 -12
  9. package/gateway-GK5QZ6KP.js.map +1 -0
  10. package/{gateway-H7UD6TNB.js → gateway-TQTGDRCN.js} +9 -8
  11. package/gateway-TQTGDRCN.js.map +1 -0
  12. package/index.cjs +2158 -1718
  13. package/index.cjs.map +1 -1
  14. package/index.d.cts +261 -117
  15. package/index.d.ts +261 -117
  16. package/index.global.js +11834 -11354
  17. package/index.global.js.map +1 -1
  18. package/index.js +1865 -1519
  19. package/index.js.map +1 -1
  20. package/{key-bag-file-WADZBHYG.js → key-bag-file-VOSSK46F.js} +4 -3
  21. package/{key-bag-file-WADZBHYG.js.map → key-bag-file-VOSSK46F.js.map} +1 -1
  22. package/{key-bag-indexdb-PGVAI3FJ.js → key-bag-indexdb-AXTQOSMC.js} +4 -3
  23. package/{key-bag-indexdb-PGVAI3FJ.js.map → key-bag-indexdb-AXTQOSMC.js.map} +1 -1
  24. package/key-bag-memory-LWE6ARPX.js +29 -0
  25. package/key-bag-memory-LWE6ARPX.js.map +1 -0
  26. package/metafile-cjs.json +1 -1
  27. package/metafile-esm.json +1 -1
  28. package/metafile-iife.json +1 -1
  29. package/{node-filesystem-INX4ZTHE.js → node-filesystem-CFRXFSO7.js} +6 -9
  30. package/node-filesystem-CFRXFSO7.js.map +1 -0
  31. package/package.json +1 -1
  32. package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +129 -0
  33. package/tests/blockstore/keyed-crypto.test.ts +63 -227
  34. package/tests/blockstore/loader.test.ts +19 -11
  35. package/tests/blockstore/store.test.ts +23 -19
  36. package/tests/blockstore/transaction.test.ts +12 -12
  37. package/tests/fireproof/all-gateway.test.ts +201 -193
  38. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +324 -316
  39. package/tests/fireproof/crdt.test.ts +67 -16
  40. package/tests/fireproof/database.test.ts +183 -21
  41. package/tests/fireproof/fireproof.test.ts +83 -74
  42. package/tests/fireproof/hello.test.ts +18 -14
  43. package/tests/fireproof/indexer.test.ts +53 -43
  44. package/tests/fireproof/utils.test.ts +18 -6
  45. package/tests/gateway/file/loader-config.test.ts +303 -0
  46. package/tests/gateway/indexdb/loader-config.test.ts +75 -0
  47. package/tests/helpers.ts +27 -9
  48. package/tests/react/useFireproof.test.tsx +1 -1
  49. package/{utils-QO2HIWGI.js → utils-STA2C35G.js} +4 -3
  50. package/utils-STA2C35G.js.map +1 -0
  51. package/chunk-OFGPKRCM.js.map +0 -1
  52. package/chunk-WS3YRPIA.js +0 -75
  53. package/chunk-WS3YRPIA.js.map +0 -1
  54. package/gateway-5FCWPX5W.js.map +0 -1
  55. package/gateway-H7UD6TNB.js.map +0 -1
  56. package/mem-filesystem-YPPJV7Q2.js +0 -41
  57. package/mem-filesystem-YPPJV7Q2.js.map +0 -1
  58. package/node-filesystem-INX4ZTHE.js.map +0 -1
  59. package/tests/fireproof/config.test.ts +0 -172
  60. /package/{utils-QO2HIWGI.js.map → chunk-PZ5AY32C.js.map} +0 -0
  61. /package/tests/fireproof/{fireproof.test.fixture.ts → fireproof.fixture.ts} +0 -0
package/index.js CHANGED
@@ -1,3 +1,6 @@
1
+ import {
2
+ INDEXDB_VERSION
3
+ } from "./chunk-PB4BKL4O.js";
1
4
  import {
2
5
  FILESTORE_VERSION
3
6
  } from "./chunk-7EWIAXTM.js";
@@ -6,28 +9,30 @@ import {
6
9
  getFileSystem,
7
10
  getPath,
8
11
  toArrayBuffer
9
- } from "./chunk-WS3YRPIA.js";
10
- import {
11
- INDEXDB_VERSION
12
- } from "./chunk-PB4BKL4O.js";
12
+ } from "./chunk-HQ7D3PEU.js";
13
13
  import {
14
14
  NotFoundError,
15
+ PARAM,
15
16
  Result,
16
17
  UInt8ArrayEqual,
17
- __export,
18
- dataDir,
19
18
  ensureLogger,
20
19
  ensureSuperLog,
21
20
  ensureSuperThis,
22
21
  exceptionWrapper,
22
+ falsyToUndef,
23
23
  getKey,
24
24
  getName,
25
25
  getStore,
26
- isNotFoundError
27
- } from "./chunk-OFGPKRCM.js";
26
+ isFalsy,
27
+ isNotFoundError,
28
+ throwFalsy
29
+ } from "./chunk-3EB3ENHT.js";
30
+ import {
31
+ __export
32
+ } from "./chunk-PZ5AY32C.js";
28
33
 
29
34
  // src/database.ts
30
- import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
35
+ import { BuildURI as BuildURI2, KeyedResolvOnce as KeyedResolvOnce3, ResolveOnce as ResolveOnce6, URI as URI8 } from "@adviser/cement";
31
36
 
32
37
  // src/write-queue.ts
33
38
  function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -72,85 +77,6 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
72
77
  // src/crdt.ts
73
78
  import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
74
79
 
75
- // src/runtime/wait-pr-multiformats/block.ts
76
- var block_exports = {};
77
- __export(block_exports, {
78
- Block: () => Block,
79
- create: () => create,
80
- createUnsafe: () => createUnsafe,
81
- decode: () => decode,
82
- encode: () => encode
83
- });
84
- import { bytes as binary, CID } from "multiformats";
85
- import { Block as mfBlock } from "multiformats/block";
86
- var Block = mfBlock;
87
- async function decode({
88
- bytes,
89
- codec: codec3,
90
- hasher: hasher7
91
- }) {
92
- if (bytes == null) throw new Error('Missing required argument "bytes"');
93
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
94
- const value = await Promise.resolve(codec3.decode(bytes));
95
- const hash = await hasher7.digest(bytes);
96
- const cid = CID.create(1, codec3.code, hash);
97
- return new mfBlock({ value, bytes, cid });
98
- }
99
- async function encode({
100
- value,
101
- codec: codec3,
102
- hasher: hasher7
103
- }) {
104
- if (typeof value === "undefined") throw new Error('Missing required argument "value"');
105
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
106
- const bytes = await Promise.resolve(codec3.encode(value));
107
- const hash = await hasher7.digest(bytes);
108
- const cid = CID.create(1, codec3.code, hash);
109
- return new mfBlock({ value, bytes, cid });
110
- }
111
- async function create({
112
- bytes,
113
- cid,
114
- hasher: hasher7,
115
- codec: codec3
116
- }) {
117
- if (bytes == null) throw new Error('Missing required argument "bytes"');
118
- if (hasher7 == null) throw new Error('Missing required argument "hasher"');
119
- const value = await Promise.resolve(codec3.decode(bytes));
120
- const hash = await hasher7.digest(bytes);
121
- if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
122
- throw new Error("CID hash does not match bytes");
123
- }
124
- return createUnsafe({
125
- bytes,
126
- cid,
127
- value,
128
- codec: codec3
129
- });
130
- }
131
- async function createUnsafe({
132
- bytes,
133
- cid,
134
- value: maybeValue,
135
- codec: codec3
136
- }) {
137
- const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
138
- if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
139
- return new Block({
140
- cid,
141
- bytes,
142
- value
143
- });
144
- }
145
-
146
- // src/crdt-helpers.ts
147
- import { parse as parse3 } from "multiformats/link";
148
- import { sha256 as hasher5 } from "multiformats/hashes/sha2";
149
- import * as codec from "@ipld/dag-cbor";
150
- import { put, get, entries, root } from "@web3-storage/pail/crdt";
151
- import { EventFetcher, vis } from "@web3-storage/pail/clock";
152
- import * as Batch from "@web3-storage/pail/crdt/batch";
153
-
154
80
  // src/blockstore/index.ts
155
81
  var blockstore_exports = {};
156
82
  __export(blockstore_exports, {
@@ -162,8 +88,11 @@ __export(blockstore_exports, {
162
88
  FragmentGateway: () => FragmentGateway,
163
89
  Loader: () => Loader,
164
90
  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
165
- ensureStart: () => ensureStart,
166
- getGatewayFromURL: () => getGatewayFromURL,
91
+ ensureStoreEnDeFile: () => ensureStoreEnDeFile,
92
+ fileGatewayFactoryItem: () => fileGatewayFactoryItem,
93
+ getDefaultURI: () => getDefaultURI,
94
+ getGatewayFactoryItem: () => getGatewayFactoryItem,
95
+ getStartedGateway: () => getStartedGateway,
167
96
  parseCarFile: () => parseCarFile,
168
97
  registerStoreProtocol: () => registerStoreProtocol,
169
98
  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
@@ -178,7 +107,7 @@ function toCIDBlock(block) {
178
107
  }
179
108
 
180
109
  // src/blockstore/store-factory.ts
181
- import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
110
+ import { KeyedResolvOnce as KeyedResolvOnce2, Result as Result7 } from "@adviser/cement";
182
111
 
183
112
  // src/runtime/files.ts
184
113
  var files_exports = {};
@@ -253,327 +182,436 @@ var UnixFSFileBuilder = class {
253
182
 
254
183
  // src/blockstore/store.ts
255
184
  import { format as format2, parse as parse2 } from "@ipld/dag-json";
256
- import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result5 } from "@adviser/cement";
257
-
258
- // src/types.ts
259
- function isFalsy(value) {
260
- return value === false && value === null && value === void 0;
261
- }
262
- function throwFalsy(value) {
263
- if (isFalsy(value)) {
264
- throw new Error("value is Falsy");
265
- }
266
- return value;
267
- }
268
- function falsyToUndef(value) {
269
- if (isFalsy(value)) {
270
- return void 0;
271
- }
272
- return value;
273
- }
274
-
275
- // src/blockstore/loader.ts
276
- import pLimit from "p-limit";
277
- import { CarReader } from "@ipld/car";
278
- import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
185
+ import { ResolveOnce as ResolveOnce3, Result as Result5, exception2Result } from "@adviser/cement";
279
186
 
280
- // src/blockstore/loader-helpers.ts
281
- import { sha256 as hasher } from "multiformats/hashes/sha2";
282
- import * as dagCodec from "@ipld/dag-cbor";
283
- async function parseCarFile(reader, logger) {
284
- const roots = await reader.getRoots();
285
- const header = await reader.get(roots[0]);
286
- if (!header) throw logger.Error().Msg("missing header block").AsError();
287
- const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
288
- const fpvalue = dec.value;
289
- if (fpvalue && !fpvalue.fp) {
290
- throw logger.Error().Msg("missing fp").AsError();
187
+ // src/blockstore/commit-queue.ts
188
+ import { Future } from "@adviser/cement";
189
+ var CommitQueue = class {
190
+ constructor() {
191
+ this.queue = [];
192
+ this.processing = false;
193
+ this._waitIdleItems = /* @__PURE__ */ new Set();
291
194
  }
292
- return fpvalue.fp;
293
- }
294
-
295
- // src/blockstore/transaction.ts
296
- import { MemoryBlockstore } from "@web3-storage/pail/block";
297
- import { toCryptoRuntime } from "@adviser/cement";
298
- var CarTransaction = class extends MemoryBlockstore {
299
- constructor(parent, opts = { add: true, noLoader: false }) {
300
- super();
301
- if (opts.add) {
302
- parent.transactions.add(this);
195
+ waitIdle() {
196
+ if (this.queue.length === 0 && !this.processing) {
197
+ return Promise.resolve();
303
198
  }
304
- this.parent = parent;
199
+ const fn = new Future();
200
+ this._waitIdleItems.add(fn);
201
+ return fn.asPromise();
305
202
  }
306
- async get(cid) {
307
- return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
203
+ async enqueue(fn) {
204
+ return new Promise((resolve, reject) => {
205
+ const queueFn = async () => {
206
+ try {
207
+ resolve(await fn());
208
+ } catch (e) {
209
+ reject(e);
210
+ } finally {
211
+ this.processing = false;
212
+ this.processNext();
213
+ }
214
+ };
215
+ this.queue.push(queueFn);
216
+ if (!this.processing) {
217
+ this.processNext();
218
+ }
219
+ });
308
220
  }
309
- async superGet(cid) {
310
- return super.get(cid);
221
+ processNext() {
222
+ if (this.queue.length > 0 && !this.processing) {
223
+ this.processing = true;
224
+ const queueFn = this.queue.shift();
225
+ if (queueFn) {
226
+ queueFn().finally(() => {
227
+ });
228
+ }
229
+ }
230
+ if (this.queue.length === 0 && !this.processing) {
231
+ const toResolve = Array.from(this._waitIdleItems);
232
+ this._waitIdleItems.clear();
233
+ toResolve.map((fn) => fn.resolve());
234
+ }
311
235
  }
312
236
  };
313
- function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
314
- const logger = ensureLogger(sthis, component, ctx);
315
- const store = opts.store || {};
316
- return {
237
+
238
+ // src/runtime/keyed-crypto.ts
239
+ var keyed_crypto_exports = {};
240
+ __export(keyed_crypto_exports, {
241
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
242
+ keyedCryptoFactory: () => keyedCryptoFactory
243
+ });
244
+ import { base58btc } from "multiformats/bases/base58";
245
+ import { sha256 as hasher } from "multiformats/hashes/sha2";
246
+ import * as CBOR from "cborg";
247
+ var generateIV = {
248
+ random: {
317
249
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
318
- applyMeta: (meta, snap) => {
319
- return Promise.resolve();
250
+ calc: async (ko, crypto, data) => {
251
+ return crypto.randomBytes(ko.ivLength);
320
252
  },
321
253
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
322
- compact: async (blocks) => {
323
- return {};
254
+ verify: async (ko, crypto, iv, data) => {
255
+ return true;
256
+ }
257
+ },
258
+ hash: {
259
+ calc: async (ko, crypto, data) => {
260
+ const hash = await hasher.digest(data);
261
+ const hashBytes = new Uint8Array(hash.bytes);
262
+ const hashArray = new Uint8Array(ko.ivLength);
263
+ for (let i = 0; i < hashBytes.length; i++) {
264
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
265
+ }
266
+ return hashArray;
324
267
  },
325
- autoCompact: 100,
326
- public: false,
327
- name: void 0,
328
- threshold: 1e3 * 1e3,
329
- ...opts,
330
- logger,
331
- keyBag: opts.keyBag || {},
332
- crypto: toCryptoRuntime(opts.crypto),
333
- store,
334
- storeRuntime: toStoreRuntime(store, sthis)
335
- };
336
- }
337
- function blockstoreFactory(sthis, opts) {
338
- if (opts.name) {
339
- return new EncryptedBlockstore(sthis, opts);
340
- } else {
341
- return new BaseBlockstore(opts);
268
+ verify: async function(ko, crypto, iv, data) {
269
+ return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
270
+ }
342
271
  }
272
+ };
273
+ function getGenerateIVFn(url, opts) {
274
+ const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
275
+ return generateIV[ivhash] || generateIV["hash"];
343
276
  }
344
- var BaseBlockstore = class {
345
- constructor(ebOpts = {}) {
346
- this.transactions = /* @__PURE__ */ new Set();
347
- this.sthis = ensureSuperThis(ebOpts);
348
- this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
349
- this.logger = this.ebOpts.logger;
350
- }
351
- // ready: Promise<void>;
352
- ready() {
353
- return Promise.resolve();
354
- }
355
- async close() {
277
+ var BlockIvKeyIdCodec = class {
278
+ constructor(ko, iv, opts) {
279
+ this.code = 3147065;
280
+ this.name = "Fireproof@encrypted-block:aes-gcm";
281
+ this.ko = ko;
282
+ this.iv = iv;
283
+ this.opts = opts || {};
356
284
  }
357
- async destroy() {
285
+ async encode(data) {
286
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
287
+ const { iv } = this.ko.algo(calcIv);
288
+ const fprt = await this.ko.fingerPrint();
289
+ const keyId = base58btc.decode(fprt);
290
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
291
+ return CBOR.encode({
292
+ iv,
293
+ keyId,
294
+ data: await this.ko._encrypt({ iv, bytes: data })
295
+ });
358
296
  }
359
- async get(cid) {
360
- if (!cid) throw this.logger.Error().Msg("required cid").AsError();
361
- for (const f of this.transactions) {
362
- const v = await f.superGet(cid);
363
- if (v) return v;
297
+ async decode(abytes) {
298
+ let bytes;
299
+ if (abytes instanceof Uint8Array) {
300
+ bytes = abytes;
301
+ } else {
302
+ bytes = new Uint8Array(abytes);
364
303
  }
365
- }
366
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
367
- async put(cid, block) {
368
- throw this.logger.Error().Msg("use a transaction to put").AsError();
369
- }
370
- // TransactionMeta
371
- async transaction(fn, _opts) {
372
- const t = new CarTransaction(this, _opts);
373
- const done = await fn(t);
374
- this.lastTxMeta = done;
375
- return { t, meta: done };
376
- }
377
- async *entries() {
378
- const seen = /* @__PURE__ */ new Set();
379
- for (const t of this.transactions) {
380
- for await (const blk of t.entries()) {
381
- if (seen.has(blk.cid.toString())) continue;
382
- seen.add(blk.cid.toString());
383
- yield blk;
384
- }
304
+ const { iv, keyId, data } = CBOR.decode(bytes);
305
+ const fprt = await this.ko.fingerPrint();
306
+ this.ko.logger.Debug().Str("fp", base58btc.encode(keyId)).Msg("decode");
307
+ if (base58btc.encode(keyId) !== fprt) {
308
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
385
309
  }
386
- }
387
- };
388
- var EncryptedBlockstore = class extends BaseBlockstore {
389
- constructor(sthis, ebOpts) {
390
- super(ebOpts);
391
- this.compacting = false;
392
- this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
393
- const { name } = ebOpts;
394
- if (!name) {
395
- throw this.logger.Error().Msg("name required").AsError();
310
+ const result = await this.ko._decrypt({ iv, bytes: data });
311
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
312
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
396
313
  }
397
- this.name = name;
398
- this.loader = new Loader(this.name, ebOpts, sthis);
314
+ return result;
399
315
  }
400
- ready() {
401
- return this.loader.ready();
316
+ };
317
+ var keyedCrypto = class {
318
+ constructor(url, key, cyopt, sthis) {
319
+ this.ivLength = 12;
320
+ this.isEncrypting = true;
321
+ this.logger = ensureLogger(sthis, "keyedCrypto");
322
+ this.crypto = cyopt;
323
+ this.key = key;
324
+ this.url = url;
402
325
  }
403
- close() {
404
- return this.loader.close();
326
+ fingerPrint() {
327
+ return Promise.resolve(this.key.fingerPrint);
405
328
  }
406
- destroy() {
407
- return this.loader.destroy();
329
+ codec(iv, opts) {
330
+ return new BlockIvKeyIdCodec(this, iv, opts);
408
331
  }
409
- async get(cid) {
410
- const got = await super.get(cid);
411
- if (got) return got;
412
- if (!this.loader) {
413
- return;
414
- }
415
- return falsyToUndef(await this.loader.getBlock(cid));
332
+ algo(iv) {
333
+ return {
334
+ name: "AES-GCM",
335
+ iv: iv || this.crypto.randomBytes(this.ivLength),
336
+ tagLength: 128
337
+ };
416
338
  }
417
- async transaction(fn, opts = { noLoader: false }) {
418
- const { t, meta: done } = await super.transaction(fn);
419
- const cars = await this.loader.commit(t, done, opts);
420
- if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
421
- setTimeout(() => void this.compact(), 10);
422
- }
423
- if (cars) {
424
- this.transactions.delete(t);
425
- return { meta: done, cars, t };
426
- }
427
- throw this.logger.Error().Msg("failed to commit car files").AsError();
339
+ async _decrypt(data) {
340
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
341
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
428
342
  }
429
- async getFile(car, cid) {
430
- await this.ready();
431
- if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
432
- const reader = await this.loader.loadFileCar(
433
- car
434
- /*, isPublic */
435
- );
436
- const block = await reader.get(cid);
437
- if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
438
- return block.bytes;
343
+ async _encrypt(data) {
344
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
345
+ const a = this.algo(data.iv);
346
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
439
347
  }
440
- async compact() {
441
- await this.ready();
442
- if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
443
- if (this.loader.carLog.length < 2) return;
444
- const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
445
- if (!compactFn || this.compacting) return;
446
- const blockLog = new CompactionFetcher(this);
447
- this.compacting = true;
448
- const meta = await compactFn(blockLog);
449
- await this.loader?.commit(blockLog.loggedBlocks, meta, {
450
- compact: true,
451
- noLoader: true
452
- });
453
- this.compacting = false;
348
+ };
349
+ var nullCodec = class {
350
+ constructor() {
351
+ this.code = 0;
352
+ this.name = "Fireproof@unencrypted-block";
454
353
  }
455
- async defaultCompact(blocks, logger) {
456
- if (!this.loader) {
457
- throw logger.Error().Msg("no loader").AsError();
458
- }
459
- if (!this.lastTxMeta) {
460
- throw logger.Error().Msg("no lastTxMeta").AsError();
461
- }
462
- for await (const blk of this.loader.entries(false)) {
463
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
464
- }
465
- for (const t of this.transactions) {
466
- for await (const blk of t.entries()) {
467
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
468
- }
469
- }
470
- return this.lastTxMeta;
354
+ encode(data) {
355
+ return data;
471
356
  }
472
- async *entries() {
473
- for await (const blk of this.loader.entries()) {
474
- yield blk;
475
- }
357
+ decode(data) {
358
+ return data;
476
359
  }
477
360
  };
478
- var CompactionFetcher = class {
479
- constructor(blocks) {
480
- this.blockstore = blocks;
481
- this.loggedBlocks = new CarTransaction(blocks);
361
+ var noCrypto = class {
362
+ constructor(url, cyrt, sthis) {
363
+ this.ivLength = 0;
364
+ this.code = 0;
365
+ this.name = "Fireproof@unencrypted-block";
366
+ this.isEncrypting = false;
367
+ this._fingerPrint = "noCrypto:" + Math.random();
368
+ this.logger = ensureLogger(sthis, "noCrypto");
369
+ this.crypto = cyrt;
370
+ this.url = url;
482
371
  }
483
- async get(cid) {
484
- const block = await this.blockstore.get(cid);
485
- if (block) this.loggedBlocks.putSync(cid, block.bytes);
486
- return falsyToUndef(block);
372
+ fingerPrint() {
373
+ return Promise.resolve(this._fingerPrint);
487
374
  }
488
- };
489
-
490
- // src/blockstore/commit-queue.ts
491
- import { Future } from "@adviser/cement";
492
- var CommitQueue = class {
493
- constructor() {
494
- this.queue = [];
495
- this.processing = false;
496
- this._waitIdleItems = /* @__PURE__ */ new Set();
375
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
376
+ codec(iv) {
377
+ return new nullCodec();
497
378
  }
498
- waitIdle() {
499
- if (this.queue.length === 0 && !this.processing) {
500
- return Promise.resolve();
501
- }
502
- const fn = new Future();
503
- this._waitIdleItems.add(fn);
504
- return fn.asPromise();
379
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
380
+ algo(iv) {
381
+ return {
382
+ name: "noCrypto",
383
+ iv: new Uint8Array(),
384
+ tagLength: 0
385
+ };
505
386
  }
506
- async enqueue(fn) {
507
- return new Promise((resolve, reject) => {
508
- const queueFn = async () => {
509
- try {
510
- resolve(await fn());
511
- } catch (e) {
512
- reject(e);
513
- } finally {
514
- this.processing = false;
515
- this.processNext();
516
- }
517
- };
518
- this.queue.push(queueFn);
519
- if (!this.processing) {
520
- this.processNext();
521
- }
522
- });
387
+ _decrypt() {
388
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
523
389
  }
524
- processNext() {
525
- if (this.queue.length > 0 && !this.processing) {
526
- this.processing = true;
527
- const queueFn = this.queue.shift();
528
- if (queueFn) {
529
- queueFn().finally(() => {
530
- });
390
+ _encrypt() {
391
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
392
+ }
393
+ };
394
+ async function keyedCryptoFactory(url, kb, sthis) {
395
+ const storekey = url.getParam("storekey" /* STORE_KEY */);
396
+ if (storekey && storekey !== "insecure") {
397
+ let rkey = await kb.getNamedKey(storekey, true);
398
+ if (rkey.isErr()) {
399
+ try {
400
+ rkey = await kb.toKeyWithFingerPrint(storekey);
401
+ } catch (e) {
402
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
531
403
  }
532
404
  }
533
- if (this.queue.length === 0 && !this.processing) {
534
- const toResolve = Array.from(this._waitIdleItems);
535
- this._waitIdleItems.clear();
536
- toResolve.map((fn) => fn.resolve());
537
- }
405
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
538
406
  }
539
- };
407
+ return new noCrypto(url, kb.rt.crypto, sthis);
408
+ }
540
409
 
541
- // src/runtime/key-bag.ts
542
- var key_bag_exports = {};
543
- __export(key_bag_exports, {
544
- KeyBag: () => KeyBag,
545
- getKeyBag: () => getKeyBag,
546
- registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
547
- });
548
- import {
549
- KeyedResolvOnce,
550
- ResolveOnce,
551
- ResolveSeq,
552
- Result as Result2,
553
- runtimeFn,
554
- toCryptoRuntime as toCryptoRuntime2,
555
- URI
556
- } from "@adviser/cement";
557
- import { base58btc } from "multiformats/bases/base58";
558
- var KeyBag = class {
559
- constructor(rt) {
560
- this.rt = rt;
561
- this._warnOnce = new ResolveOnce();
562
- this._seq = new ResolveSeq();
563
- this.logger = ensureLogger(rt.sthis, "KeyBag");
564
- this.logger.Debug().Msg("KeyBag created");
410
+ // src/blockstore/fragment-gateway.ts
411
+ import { Result as Result2 } from "@adviser/cement";
412
+ import { base58btc as base58btc2 } from "multiformats/bases/base58";
413
+ import { encode as encode2, decode as decode2 } from "cborg";
414
+ function getFragSize(url) {
415
+ const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
416
+ let ret = 0;
417
+ if (fragSize) {
418
+ ret = parseInt(fragSize);
565
419
  }
566
- async subtleKey(key) {
567
- const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
568
- if (extractable) {
569
- this._warnOnce.once(
570
- () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
420
+ if (isNaN(ret) || ret <= 0) {
421
+ ret = 0;
422
+ }
423
+ return ret;
424
+ }
425
+ async function getFrags(url, innerGW, headerSize, logger) {
426
+ const fragSize = getFragSize(url);
427
+ if (!fragSize) {
428
+ const res = await innerGW.get(url);
429
+ if (res.isErr()) {
430
+ return [res];
431
+ }
432
+ const data = res.unwrap();
433
+ return [
434
+ Result2.Ok({
435
+ fid: new Uint8Array(0),
436
+ ofs: 0,
437
+ len: data.length,
438
+ data
439
+ })
440
+ ];
441
+ }
442
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
443
+ if (firstRaw.isErr()) {
444
+ return [firstRaw];
445
+ }
446
+ const firstFragment = decode2(firstRaw.unwrap());
447
+ const blockSize = firstFragment.data.length;
448
+ const ops = [Promise.resolve(Result2.Ok(firstFragment))];
449
+ const fidStr = base58btc2.encode(firstFragment.fid);
450
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
451
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
452
+ ops.push(
453
+ (async (furl, ofs2) => {
454
+ const raw2 = await innerGW.get(furl);
455
+ if (raw2.isErr()) {
456
+ return raw2;
457
+ }
458
+ const fragment = decode2(raw2.unwrap());
459
+ if (base58btc2.encode(fragment.fid) !== fidStr) {
460
+ return Result2.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
461
+ }
462
+ if (fragment.ofs !== ofs2) {
463
+ return Result2.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
464
+ }
465
+ return Result2.Ok(fragment);
466
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
467
+ );
468
+ }
469
+ return Promise.all(ops);
470
+ }
471
+ var FragmentGateway = class {
472
+ constructor(sthis, innerGW) {
473
+ this.fidLength = 4;
474
+ this.headerSize = 32;
475
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
476
+ this.logger = this.sthis.logger;
477
+ this.innerGW = innerGW;
478
+ }
479
+ slicer(url, body) {
480
+ const fragSize = getFragSize(url);
481
+ if (!fragSize) {
482
+ return [this.innerGW.put(url, body)];
483
+ }
484
+ const blocksize = fragSize - this.headerSize;
485
+ if (blocksize <= 0) {
486
+ throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
487
+ }
488
+ const ops = [];
489
+ const fid = this.sthis.nextId(this.fidLength);
490
+ const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
491
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
492
+ const block = encode2({
493
+ fid: fid.bin,
494
+ ofs,
495
+ len: body.length,
496
+ data: body.slice(ofs, ofs + blocksize)
497
+ });
498
+ if (block.length > fragSize) {
499
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
500
+ }
501
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
502
+ }
503
+ return ops;
504
+ }
505
+ buildUrl(baseUrl, key) {
506
+ return this.innerGW.buildUrl(baseUrl, key);
507
+ }
508
+ async destroy(iurl) {
509
+ return this.innerGW.destroy(iurl);
510
+ }
511
+ async start(url) {
512
+ this.headerSize = encode2({
513
+ fid: this.sthis.nextId(this.fidLength).bin,
514
+ ofs: 1024 * 1024,
515
+ // 32bit
516
+ len: 16 * 1024 * 1024,
517
+ // 32bit
518
+ data: new Uint8Array(1024)
519
+ }).length - 1024;
520
+ return this.innerGW.start(url);
521
+ }
522
+ async close(url) {
523
+ return this.innerGW.close(url);
524
+ }
525
+ async put(url, body) {
526
+ await Promise.all(this.slicer(url, body));
527
+ return Result2.Ok(void 0);
528
+ }
529
+ async get(url) {
530
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
531
+ let buffer = void 0;
532
+ for (const rfrag of rfrags) {
533
+ if (rfrag.isErr()) {
534
+ return Result2.Err(rfrag.Err());
535
+ }
536
+ const frag = rfrag.Ok();
537
+ buffer = buffer || new Uint8Array(frag.len);
538
+ buffer.set(frag.data, frag.ofs);
539
+ }
540
+ return Result2.Ok(buffer || new Uint8Array(0));
541
+ }
542
+ async subscribe(url, callback) {
543
+ if (this.innerGW.subscribe) {
544
+ return this.innerGW.subscribe(url, callback);
545
+ } else {
546
+ return Result2.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
547
+ }
548
+ }
549
+ async delete(url) {
550
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
551
+ for (const rfrag of rfrags) {
552
+ if (rfrag.isErr()) {
553
+ return Result2.Err(rfrag.Err());
554
+ }
555
+ const frag = rfrag.Ok();
556
+ let fragUrl;
557
+ if (rfrags.length > 1) {
558
+ const fidStr = base58btc2.encode(frag.fid);
559
+ fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
560
+ } else {
561
+ fragUrl = url;
562
+ }
563
+ await this.innerGW.delete(fragUrl);
564
+ }
565
+ return Result2.Ok(void 0);
566
+ }
567
+ };
568
+
569
+ // src/blockstore/meta-key-helper.ts
570
+ import { format, parse } from "@ipld/dag-json";
571
+ import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
572
+ import { CID } from "multiformats";
573
+ import { base64pad } from "multiformats/bases/base64";
574
+ import { Result as Result4 } from "@adviser/cement";
575
+
576
+ // src/runtime/key-bag.ts
577
+ var key_bag_exports = {};
578
+ __export(key_bag_exports, {
579
+ KeyBag: () => KeyBag,
580
+ defaultKeyBagOpts: () => defaultKeyBagOpts,
581
+ getKeyBag: () => getKeyBag,
582
+ registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
583
+ });
584
+ import {
585
+ KeyedResolvOnce,
586
+ ResolveOnce,
587
+ ResolveSeq,
588
+ Result as Result3,
589
+ runtimeFn,
590
+ toCryptoRuntime,
591
+ URI as URI2
592
+ } from "@adviser/cement";
593
+ import { base58btc as base58btc3 } from "multiformats/bases/base58";
594
+ var KeyBag = class {
595
+ constructor(rt) {
596
+ this.rt = rt;
597
+ this._warnOnce = new ResolveOnce();
598
+ this._seq = new ResolveSeq();
599
+ this.logger = ensureLogger(rt.sthis, "KeyBag", {
600
+ id: rt.id()
601
+ });
602
+ this.logger.Debug().Msg("KeyBag created");
603
+ }
604
+ async subtleKey(key) {
605
+ const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
606
+ if (extractable) {
607
+ this._warnOnce.once(
608
+ () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
571
609
  );
572
610
  }
573
611
  return await this.rt.crypto.importKey(
574
612
  "raw",
575
613
  // raw or jwk
576
- base58btc.decode(key),
614
+ base58btc3.decode(key),
577
615
  // hexStringToUint8Array(key), // raw data
578
616
  "AES-GCM",
579
617
  extractable,
@@ -581,9 +619,9 @@ var KeyBag = class {
581
619
  );
582
620
  }
583
621
  async ensureKeyFromUrl(url, keyFactory) {
584
- const storeKey = url.getParam("storekey");
622
+ const storeKey = url.getParam("storekey" /* STORE_KEY */);
585
623
  if (storeKey === "insecure") {
586
- return Result2.Ok(url);
624
+ return Result3.Ok(url);
587
625
  }
588
626
  if (!storeKey) {
589
627
  const keyName = `@${keyFactory()}@`;
@@ -591,8 +629,8 @@ var KeyBag = class {
591
629
  if (ret.isErr()) {
592
630
  return ret;
593
631
  }
594
- const urb = url.build().setParam("storekey", keyName);
595
- return Result2.Ok(urb.URI());
632
+ const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
633
+ return Result3.Ok(urb.URI());
596
634
  }
597
635
  if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
598
636
  const ret = await this.getNamedKey(storeKey);
@@ -600,15 +638,15 @@ var KeyBag = class {
600
638
  return ret;
601
639
  }
602
640
  }
603
- return Result2.Ok(url);
641
+ return Result3.Ok(url);
604
642
  }
605
643
  async toKeyWithFingerPrint(keyStr) {
606
- const material = base58btc.decode(keyStr);
644
+ const material = base58btc3.decode(keyStr);
607
645
  const key = await this.subtleKey(keyStr);
608
646
  const fpr = await this.rt.crypto.digestSHA256(material);
609
- return Result2.Ok({
647
+ return Result3.Ok({
610
648
  key,
611
- fingerPrint: base58btc.encode(new Uint8Array(fpr))
649
+ fingerPrint: base58btc3.encode(new Uint8Array(fpr))
612
650
  });
613
651
  }
614
652
  async setNamedKey(name, key) {
@@ -631,13 +669,13 @@ var KeyBag = class {
631
669
  return ret;
632
670
  }
633
671
  const named = ret.Ok();
634
- return Result2.Ok({
672
+ return Result3.Ok({
635
673
  ...named,
636
674
  extract: async () => {
637
675
  const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
638
676
  return {
639
677
  key: ext,
640
- keyStr: base58btc.encode(ext)
678
+ keyStr: base58btc3.encode(ext)
641
679
  };
642
680
  }
643
681
  });
@@ -654,9 +692,9 @@ var KeyBag = class {
654
692
  }
655
693
  if (failIfNotFound) {
656
694
  this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
657
- return Result2.Err(new Error(`Key not found: ${name}`));
695
+ return Result3.Err(new Error(`Key not found: ${name}`));
658
696
  }
659
- const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
697
+ const ret = await this._setNamedKey(name, base58btc3.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
660
698
  this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
661
699
  return ret;
662
700
  });
@@ -667,14 +705,14 @@ var keyBagProviderFactories = new Map(
667
705
  {
668
706
  protocol: "file:",
669
707
  factory: async (url, sthis) => {
670
- const { KeyBagProviderFile } = await import("./key-bag-file-WADZBHYG.js");
708
+ const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
671
709
  return new KeyBagProviderFile(url, sthis);
672
710
  }
673
711
  },
674
712
  {
675
713
  protocol: "indexdb:",
676
714
  factory: async (url, sthis) => {
677
- const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-PGVAI3FJ.js");
715
+ const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
678
716
  return new KeyBagProviderIndexDB(url, sthis);
679
717
  }
680
718
  }
@@ -688,44 +726,63 @@ function registerKeyBagProviderFactory(item) {
688
726
  });
689
727
  }
690
728
  function defaultKeyBagOpts(sthis, kbo) {
729
+ kbo = kbo || {};
691
730
  if (kbo.keyRuntime) {
692
731
  return kbo.keyRuntime;
693
732
  }
694
733
  const logger = ensureLogger(sthis, "KeyBag");
695
734
  let url;
696
735
  if (kbo.url) {
697
- url = URI.from(kbo.url);
736
+ url = URI2.from(kbo.url);
698
737
  logger.Debug().Url(url).Msg("from opts");
699
738
  } else {
700
739
  let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
701
740
  if (runtimeFn().isBrowser) {
702
- url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
741
+ url = URI2.from(bagFnameOrUrl || "indexdb://fp-keybag");
703
742
  } else {
704
743
  if (!bagFnameOrUrl) {
705
744
  const home = sthis.env.get("HOME");
706
745
  bagFnameOrUrl = `${home}/.fireproof/keybag`;
707
- url = URI.from(`file://${bagFnameOrUrl}`);
746
+ url = URI2.from(`file://${bagFnameOrUrl}`);
708
747
  } else {
709
- url = URI.from(bagFnameOrUrl);
748
+ url = URI2.from(bagFnameOrUrl);
710
749
  }
711
750
  }
712
751
  logger.Debug().Url(url).Msg("from env");
713
752
  }
714
- const kitem = keyBagProviderFactories.get(url.protocol);
715
- if (!kitem) {
716
- throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
753
+ let keyProviderFactory;
754
+ switch (url.protocol) {
755
+ case "file:":
756
+ keyProviderFactory = async () => {
757
+ const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
758
+ return new KeyBagProviderFile(url, sthis);
759
+ };
760
+ break;
761
+ case "indexdb:":
762
+ keyProviderFactory = async () => {
763
+ const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
764
+ return new KeyBagProviderIndexDB(url, sthis);
765
+ };
766
+ break;
767
+ case "memory:":
768
+ keyProviderFactory = async () => {
769
+ const { KeyBagProviderMemory } = await import("./key-bag-memory-LWE6ARPX.js");
770
+ return new KeyBagProviderMemory(url, sthis);
771
+ };
772
+ break;
773
+ default:
774
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
717
775
  }
718
- const getBag = async () => kitem.factory(url, sthis);
719
776
  if (url.hasParam("masterkey")) {
720
777
  throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
721
778
  }
722
779
  return {
723
780
  url,
724
- crypto: kbo.crypto || toCryptoRuntime2({}),
781
+ crypto: kbo.crypto || toCryptoRuntime({}),
725
782
  sthis,
726
783
  logger,
727
784
  keyLength: kbo.keyLength || 16,
728
- getBag,
785
+ getBag: keyProviderFactory,
729
786
  id: () => {
730
787
  return url.toString();
731
788
  }
@@ -738,936 +795,940 @@ async function getKeyBag(sthis, kbo = {}) {
738
795
  return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
739
796
  }
740
797
 
741
- // src/blockstore/commitor.ts
742
- import * as CBW from "@ipld/car/buffer-writer";
743
- import { sha256 as hasher2 } from "multiformats/hashes/sha2";
744
- import * as dagCodec2 from "@ipld/dag-cbor";
745
- async function encodeCarFile(roots, t, codec3) {
746
- let size = 0;
747
- const headerSize = CBW.headerLength({ roots });
748
- size += headerSize;
749
- for (const { cid, bytes } of t.entries()) {
750
- size += CBW.blockLength({ cid, bytes });
751
- }
752
- const buffer = new Uint8Array(size);
753
- const writer = CBW.createWriter(buffer, { headerSize });
754
- for (const r of roots) {
755
- writer.addRoot(r);
756
- }
757
- for (const { cid, bytes } of t.entries()) {
758
- writer.write({ cid, bytes });
798
+ // src/blockstore/meta-key-helper.ts
799
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
800
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
801
+ if (!crdtEntries.length) {
802
+ sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
803
+ return [];
759
804
  }
760
- writer.close();
761
- return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
762
- }
763
- async function createCarFile(encoder, cid, t) {
764
- return encodeCarFile([cid], t, encoder);
765
- }
766
- async function commitFiles(fileStore, walStore, t, done) {
767
- const { files: roots } = makeFileCarHeader(done);
768
- const cids = [];
769
- const codec3 = (await fileStore.keyedCrypto()).codec();
770
- const cars = await prepareCarFilesFiles(codec3, roots, t);
771
- for (const car of cars) {
772
- const { cid, bytes } = car;
773
- await fileStore.save({ cid, bytes });
774
- await walStore.enqueueFile(
775
- cid
776
- /*, !!opts.public*/
777
- );
778
- cids.push(cid);
805
+ if (!crdtEntries.map) {
806
+ sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
807
+ return [];
779
808
  }
780
- return cids;
809
+ return Promise.all(
810
+ crdtEntries.map(async (crdtEntry) => {
811
+ const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
812
+ const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
813
+ return {
814
+ eventCid: eventBlock.cid,
815
+ parents: crdtEntry.parents,
816
+ dbMeta
817
+ };
818
+ })
819
+ );
781
820
  }
782
- function makeFileCarHeader(result) {
783
- const files = [];
784
- for (const [, meta] of Object.entries(result.files || {})) {
785
- if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
786
- files.push(meta.cid);
821
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
822
+ try {
823
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
824
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
825
+ if (keyInfo.length) {
826
+ const dbMeta = keyInfo[0].dbMeta;
827
+ if (dbMeta.key) {
828
+ const kb = await getKeyBag(sthis);
829
+ const keyName = getStoreKeyName(uri);
830
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
831
+ if (res.isErr()) {
832
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
833
+ throw res.Err();
834
+ }
835
+ }
836
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
837
+ return Result4.Ok(dbMeta);
787
838
  }
839
+ sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
840
+ return Result4.Ok(void 0);
841
+ } catch (error) {
842
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
843
+ return Result4.Err(error);
788
844
  }
789
- return { ...result, files };
790
845
  }
791
- async function prepareCarFilesFiles(encoder, roots, t) {
792
- return [await encodeCarFile(roots, t, encoder)];
793
- }
794
- function makeCarHeader(meta, cars, compact = false) {
795
- const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
796
- return { ...coreHeader, meta };
797
- }
798
- async function encodeCarHeader(fp) {
799
- return await encode({
800
- value: { fp },
801
- hasher: hasher2,
802
- codec: dagCodec2
803
- });
804
- }
805
- async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
806
- const fp = makeCarHeader(done, params.carLog, !!opts.compact);
807
- const rootBlock = await encodeCarHeader(fp);
808
- const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
809
- const cids = [];
810
- for (const car of cars) {
811
- const { cid, bytes } = car;
812
- await params.carStore.save({ cid, bytes });
813
- cids.push(cid);
846
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
847
+ try {
848
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
849
+ const keyName = getStoreKeyName(uri);
850
+ const kb = await getKeyBag(sthis);
851
+ const res = await kb.getNamedExtractableKey(keyName, true);
852
+ if (res.isErr()) {
853
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
854
+ throw res.Err();
855
+ }
856
+ const keyData = await res.Ok().extract();
857
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
858
+ const { dbMeta, parents } = dbMetas[0];
859
+ const parentLinks = parents.map((p) => CID.parse(p));
860
+ dbMeta.key = keyData.keyStr;
861
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
862
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
863
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
864
+ return Result4.Ok(encoded);
865
+ } catch (error) {
866
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
867
+ return Result4.Err(error);
814
868
  }
815
- const newDbMeta = { cars: cids };
816
- await params.WALStore.enqueue(newDbMeta, opts);
817
- await params.metaStore.save(newDbMeta);
818
- return { cgrp: cids, header: fp };
819
869
  }
820
- async function prepareCarFiles(encoder, threshold, rootBlock, t) {
821
- const carFiles = [];
822
- threshold = threshold || 128e3 * 8;
823
- let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
824
- clonedt.putSync(rootBlock.cid, rootBlock.bytes);
825
- let newsize = CBW.blockLength(toCIDBlock(rootBlock));
826
- let cidRootBlock = rootBlock;
827
- for (const { cid, bytes } of t.entries()) {
828
- newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
829
- if (newsize >= threshold) {
830
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
831
- clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
832
- clonedt.putSync(cid, bytes);
833
- cidRootBlock = { cid, bytes };
834
- newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
835
- } else {
836
- clonedt.putSync(cid, bytes);
837
- }
870
+ function getStoreKeyName(url) {
871
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
872
+ const idx = url.getParam("index");
873
+ if (idx) {
874
+ storeKeyName.push(idx);
838
875
  }
839
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
840
- return carFiles;
876
+ storeKeyName.push("data");
877
+ return `@${storeKeyName.join(":")}@`;
878
+ }
879
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
880
+ const event = await EventBlock.create(
881
+ {
882
+ dbMeta: sthis.txt.encode(format(dbMeta))
883
+ },
884
+ parents
885
+ );
886
+ return event;
887
+ }
888
+ async function encodeEventsWithParents(sthis, events, parents) {
889
+ const crdtEntries = events.map((event) => {
890
+ const base64String = base64pad.encode(event.bytes);
891
+ return {
892
+ cid: event.cid.toString(),
893
+ data: base64String,
894
+ parents: parents.map((p) => p.toString())
895
+ };
896
+ });
897
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
841
898
  }
842
899
 
843
- // src/blockstore/loader.ts
844
- import { sha256 as hasher3 } from "multiformats/hashes/sha2";
845
-
846
- // src/blockstore/task-manager.ts
847
- var TaskManager = class {
848
- constructor(sthis, callback) {
849
- this.eventsWeHandled = /* @__PURE__ */ new Set();
850
- this.queue = [];
851
- this.isProcessing = false;
852
- this.logger = ensureLogger(sthis, "TaskManager");
853
- this.callback = callback;
854
- }
855
- async handleEvent(cid, parents, dbMeta) {
856
- for (const parent of parents) {
857
- this.eventsWeHandled.add(parent.toString());
858
- }
859
- this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
860
- this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
861
- void this.processQueue();
862
- }
863
- async processQueue() {
864
- if (this.isProcessing) return;
865
- this.isProcessing = true;
866
- const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
867
- const first = filteredQueue[0];
868
- if (!first) {
869
- return;
870
- }
871
- try {
872
- await this.callback(first.dbMeta);
873
- this.eventsWeHandled.add(first.cid);
874
- this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
875
- } catch (err) {
876
- if (first.retries++ > 3) {
877
- this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
878
- this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
879
- }
880
- await new Promise((resolve) => setTimeout(resolve, 50));
881
- throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
882
- } finally {
883
- this.isProcessing = false;
884
- if (this.queue.length > 0) {
885
- void this.processQueue();
886
- }
887
- }
888
- }
889
- };
900
+ // src/blockstore/store.ts
901
+ import pRetry from "p-retry";
902
+ import pMap from "p-map";
890
903
 
891
904
  // src/blockstore/loader.ts
892
- function carLogIncludesGroup(list, cids) {
893
- return list.some((arr) => {
894
- return arr.toString() === cids.toString();
895
- });
896
- }
897
- function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
898
- const byString = /* @__PURE__ */ new Map();
899
- for (const cid of list) {
900
- if (remove.has(cid.toString())) continue;
901
- byString.set(cid.toString(), cid);
902
- }
903
- return [...byString.values()];
905
+ import pLimit from "p-limit";
906
+ import { CarReader } from "@ipld/car";
907
+ import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
908
+
909
+ // src/runtime/wait-pr-multiformats/block.ts
910
+ var block_exports = {};
911
+ __export(block_exports, {
912
+ Block: () => Block,
913
+ create: () => create,
914
+ createUnsafe: () => createUnsafe,
915
+ decode: () => decode3,
916
+ encode: () => encode3
917
+ });
918
+ import { bytes as binary, CID as CID2 } from "multiformats";
919
+ import { Block as mfBlock } from "multiformats/block";
920
+ var Block = mfBlock;
921
+ async function decode3({
922
+ bytes,
923
+ codec: codec3,
924
+ hasher: hasher7
925
+ }) {
926
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
927
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
928
+ const value = await Promise.resolve(codec3.decode(bytes));
929
+ const hash = await hasher7.digest(bytes);
930
+ const cid = CID2.create(1, codec3.code, hash);
931
+ return new mfBlock({ value, bytes, cid });
904
932
  }
905
- var Loader = class {
906
- constructor(name, ebOpts, sthis) {
907
- this.commitQueue = new CommitQueue();
908
- this.isCompacting = false;
909
- this.carReaders = /* @__PURE__ */ new Map();
910
- this.seenCompacted = /* @__PURE__ */ new Set();
911
- this.processedCars = /* @__PURE__ */ new Set();
912
- this.carLog = [];
913
- this.getBlockCache = /* @__PURE__ */ new Map();
914
- this.seenMeta = /* @__PURE__ */ new Set();
915
- this.writeLimit = pLimit(1);
916
- this.onceReady = new ResolveOnce2();
917
- this.name = name;
918
- this.sthis = sthis;
919
- this.ebOpts = defaultedBlockstoreRuntime(
920
- sthis,
921
- {
922
- ...ebOpts,
923
- name
924
- },
925
- "Loader"
926
- );
927
- this.logger = this.ebOpts.logger;
928
- this.taskManager = new TaskManager(sthis, async (dbMeta) => {
929
- await this.handleDbMetasFromStore([dbMeta]);
930
- });
931
- }
932
- // readonly id = uuidv4();
933
- async keyBag() {
934
- return getKeyBag(this.sthis, this.ebOpts.keyBag);
935
- }
936
- async carStore() {
937
- return this.ebOpts.storeRuntime.makeDataStore(this);
938
- }
939
- async fileStore() {
940
- return this.ebOpts.storeRuntime.makeDataStore(this);
941
- }
942
- async WALStore() {
943
- return this.ebOpts.storeRuntime.makeWALStore(this);
944
- }
945
- async metaStore() {
946
- return this.ebOpts.storeRuntime.makeMetaStore(this);
947
- }
948
- async ready() {
949
- return this.onceReady.once(async () => {
950
- const metas = await (await this.metaStore()).load();
951
- if (this.ebOpts.meta) {
952
- await this.handleDbMetasFromStore([this.ebOpts.meta]);
953
- } else if (metas) {
954
- await this.handleDbMetasFromStore(metas);
955
- }
956
- });
957
- }
958
- async close() {
959
- const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
960
- await Promise.all(toClose.map((store) => store.close()));
961
- }
962
- async destroy() {
963
- const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
964
- await Promise.all(toDestroy.map((store) => store.destroy()));
965
- }
966
- // async snapToCar(carCid: AnyLink | string) {
967
- // await this.ready
968
- // if (typeof carCid === 'string') {
969
- // carCid = CID.parse(carCid)
970
- // }
971
- // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
972
- // this.carLog = [carCid, ...carHeader.cars]
973
- // await this.getMoreReaders(carHeader.cars)
974
- // await this._applyCarHeader(carHeader, true)
975
- // }
976
- async handleDbMetasFromStore(metas) {
977
- this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
978
- for (const meta of metas) {
979
- await this.writeLimit(async () => {
980
- await this.mergeDbMetaIntoClock(meta);
981
- });
982
- }
983
- }
984
- async mergeDbMetaIntoClock(meta) {
985
- if (this.isCompacting) {
986
- throw this.logger.Error().Msg("cannot merge while compacting").AsError();
987
- }
988
- if (this.seenMeta.has(meta.cars.toString())) return;
989
- this.seenMeta.add(meta.cars.toString());
990
- if (carLogIncludesGroup(this.carLog, meta.cars)) {
991
- return;
992
- }
993
- const carHeader = await this.loadCarHeaderFromMeta(meta);
994
- carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
995
- await this.getMoreReaders(carHeader.cars.flat());
996
- this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
997
- await this.ebOpts.applyMeta?.(carHeader.meta);
998
- }
999
- // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1000
- // const { key } = meta;
1001
- // if (key) {
1002
- // await this.setKey(key);
1003
- // }
1004
- // }
1005
- async loadCarHeaderFromMeta({ cars: cids }) {
1006
- const reader = await this.loadCar(cids[0]);
1007
- return await parseCarFile(reader, this.logger);
1008
- }
1009
- // async _getKey(): Promise<string | undefined> {
1010
- // if (this.key) return this.key;
1011
- // // generate a random key
1012
- // if (!this.ebOpts.public) {
1013
- // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1014
- // }
1015
- // return this.key || undefined;
1016
- // }
1017
- async commitFiles(t, done) {
1018
- await this.ready();
1019
- const fstore = await this.fileStore();
1020
- const wstore = await this.WALStore();
1021
- return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1022
- }
1023
- async loadFileCar(cid) {
1024
- return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1025
- }
1026
- async commit(t, done, opts = { noLoader: false, compact: false }) {
1027
- await this.ready();
1028
- const fstore = await this.fileStore();
1029
- const params = {
1030
- encoder: (await fstore.keyedCrypto()).codec(),
1031
- carLog: this.carLog,
1032
- carStore: fstore,
1033
- WALStore: await this.WALStore(),
1034
- metaStore: await this.metaStore(),
1035
- threshold: this.ebOpts.threshold
1036
- };
1037
- return this.commitQueue.enqueue(async () => {
1038
- await this.cacheTransaction(t);
1039
- const ret = await commit(params, t, done, opts);
1040
- await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1041
- return ret.cgrp;
1042
- });
1043
- }
1044
- async updateCarLog(cids, fp, compact) {
1045
- if (compact) {
1046
- const previousCompactCid = fp.compact[fp.compact.length - 1];
1047
- fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1048
- this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1049
- await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1050
- } else {
1051
- this.carLog.unshift(cids);
1052
- }
1053
- }
1054
- async cacheTransaction(t) {
1055
- for await (const block of t.entries()) {
1056
- const sBlock = block.cid.toString();
1057
- if (!this.getBlockCache.has(sBlock)) {
1058
- this.getBlockCache.set(sBlock, block);
1059
- }
1060
- }
1061
- }
1062
- async cacheCarReader(carCidStr, reader) {
1063
- if (this.processedCars.has(carCidStr)) return;
1064
- this.processedCars.add(carCidStr);
1065
- for await (const block of reader.blocks()) {
1066
- const sBlock = block.cid.toString();
1067
- if (!this.getBlockCache.has(sBlock)) {
1068
- this.getBlockCache.set(sBlock, block);
1069
- }
1070
- }
1071
- }
1072
- async removeCidsForCompact(cid) {
1073
- const carHeader = await this.loadCarHeaderFromMeta({
1074
- cars: [cid]
1075
- });
1076
- for (const cids of carHeader.compact) {
1077
- for (const cid2 of cids) {
1078
- await (await this.carStore()).remove(cid2);
1079
- }
1080
- }
1081
- }
1082
- // async flushCars() {
1083
- // await this.ready
1084
- // // for each cid in car log, make a dbMeta
1085
- // for (const cid of this.carLog) {
1086
- // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1087
- // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1088
- // }
1089
- // }
1090
- async *entries(cache2 = true) {
1091
- await this.ready();
1092
- if (cache2) {
1093
- for (const [, block] of this.getBlockCache) {
1094
- yield block;
1095
- }
1096
- } else {
1097
- for (const [, block] of this.getBlockCache) {
1098
- yield block;
1099
- }
1100
- for (const cids of this.carLog) {
1101
- for (const cid of cids) {
1102
- const reader = await this.loadCar(cid);
1103
- if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1104
- for await (const block of reader.blocks()) {
1105
- const sCid = block.cid.toString();
1106
- if (!this.getBlockCache.has(sCid)) {
1107
- yield block;
1108
- }
1109
- }
1110
- }
1111
- }
1112
- }
1113
- }
1114
- async getBlock(cid) {
1115
- await this.ready();
1116
- const sCid = cid.toString();
1117
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1118
- const getCarCid = async (carCid) => {
1119
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1120
- const reader = await this.loadCar(carCid);
1121
- if (!reader) {
1122
- throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1123
- }
1124
- await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1125
- return;
1126
- });
1127
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1128
- throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1129
- };
1130
- const getCompactCarCids = async (carCid) => {
1131
- const reader = await this.loadCar(carCid);
1132
- if (!reader) {
1133
- throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1134
- }
1135
- const header = await parseCarFile(reader, this.logger);
1136
- const compacts = header.compact;
1137
- let got2;
1138
- const batchSize2 = 5;
1139
- for (let i = 0; i < compacts.length; i += batchSize2) {
1140
- const promises = [];
1141
- for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1142
- for (const cid2 of compacts[j]) {
1143
- promises.push(getCarCid(cid2));
1144
- }
1145
- }
1146
- try {
1147
- got2 = await Promise.any(promises);
1148
- } catch {
1149
- }
1150
- if (got2) break;
1151
- }
1152
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1153
- throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1154
- };
1155
- let got;
1156
- const batchSize = 5;
1157
- for (let i = 0; i < this.carLog.length; i += batchSize) {
1158
- const batch = this.carLog.slice(i, i + batchSize);
1159
- const promises = batch.flatMap((slice) => slice.map(getCarCid));
1160
- try {
1161
- got = await Promise.any(promises);
1162
- } catch {
1163
- }
1164
- if (got) break;
1165
- }
1166
- if (!got) {
1167
- try {
1168
- got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1169
- } catch {
1170
- }
1171
- }
1172
- return got;
933
+ async function encode3({
934
+ value,
935
+ codec: codec3,
936
+ hasher: hasher7
937
+ }) {
938
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
939
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
940
+ const bytes = await Promise.resolve(codec3.encode(value));
941
+ const hash = await hasher7.digest(bytes);
942
+ const cid = CID2.create(1, codec3.code, hash);
943
+ return new mfBlock({ value, bytes, cid });
944
+ }
945
+ async function create({
946
+ bytes,
947
+ cid,
948
+ hasher: hasher7,
949
+ codec: codec3
950
+ }) {
951
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
952
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
953
+ const value = await Promise.resolve(codec3.decode(bytes));
954
+ const hash = await hasher7.digest(bytes);
955
+ if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
956
+ throw new Error("CID hash does not match bytes");
1173
957
  }
1174
- async loadCar(cid) {
1175
- if (!this.carStore) {
1176
- throw this.logger.Error().Msg("car store not initialized").AsError();
1177
- }
1178
- const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1179
- return loaded;
958
+ return createUnsafe({
959
+ bytes,
960
+ cid,
961
+ value,
962
+ codec: codec3
963
+ });
964
+ }
965
+ async function createUnsafe({
966
+ bytes,
967
+ cid,
968
+ value: maybeValue,
969
+ codec: codec3
970
+ }) {
971
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
972
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
973
+ return new Block({
974
+ cid,
975
+ bytes,
976
+ value
977
+ });
978
+ }
979
+
980
+ // src/blockstore/loader-helpers.ts
981
+ import { sha256 as hasher2 } from "multiformats/hashes/sha2";
982
+ import * as dagCodec from "@ipld/dag-cbor";
983
+ async function parseCarFile(reader, logger) {
984
+ const roots = await reader.getRoots();
985
+ const header = await reader.get(roots[0]);
986
+ if (!header) throw logger.Error().Msg("missing header block").AsError();
987
+ const dec = await decode3({ bytes: header.bytes, hasher: hasher2, codec: dagCodec });
988
+ const fpvalue = dec.value;
989
+ if (fpvalue && !fpvalue.fp) {
990
+ throw logger.Error().Msg("missing fp").AsError();
1180
991
  }
1181
- async makeDecoderAndCarReader(cid, local, remote) {
1182
- const cidsString = cid.toString();
1183
- let loadedCar = void 0;
1184
- let activeStore = local;
1185
- try {
1186
- this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1187
- loadedCar = await local.load(cid);
1188
- this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1189
- } catch (e) {
1190
- if (remote) {
1191
- const remoteCar = await remote.load(cid);
1192
- if (remoteCar) {
1193
- this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1194
- await local.save(remoteCar);
1195
- loadedCar = remoteCar;
1196
- activeStore = remote;
1197
- }
1198
- } else {
1199
- this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1200
- }
1201
- }
1202
- if (!loadedCar) {
1203
- throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
992
+ return fpvalue.fp;
993
+ }
994
+
995
+ // src/blockstore/transaction.ts
996
+ import { MemoryBlockstore } from "@web3-storage/pail/block";
997
+ import { toCryptoRuntime as toCryptoRuntime2 } from "@adviser/cement";
998
+ var CarTransaction = class extends MemoryBlockstore {
999
+ constructor(parent, opts = { add: true, noLoader: false }) {
1000
+ super();
1001
+ if (opts.add) {
1002
+ parent.transactions.add(this);
1204
1003
  }
1205
- const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
1206
- const rawReader = await CarReader.fromBytes(bytes.value);
1207
- const readerP = Promise.resolve(rawReader);
1208
- const cachedReaderP = readerP.then(async (reader) => {
1209
- await this.cacheCarReader(cidsString, reader).catch((e) => {
1210
- this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1211
- return;
1212
- });
1213
- return reader;
1214
- });
1215
- this.carReaders.set(cidsString, cachedReaderP);
1216
- return readerP;
1004
+ this.parent = parent;
1217
1005
  }
1218
- //What if instead it returns an Array of CarHeader
1219
- async storesLoadCar(cid, local, remote) {
1220
- const cidsString = cid.toString();
1221
- let dacr = this.carReaders.get(cidsString);
1222
- if (!dacr) {
1223
- dacr = this.makeDecoderAndCarReader(cid, local, remote);
1224
- this.carReaders.set(cidsString, dacr);
1225
- }
1226
- return dacr;
1006
+ async get(cid) {
1007
+ return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
1227
1008
  }
1228
- async getMoreReaders(cids) {
1229
- const limit = pLimit(5);
1230
- const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1231
- await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1009
+ async superGet(cid) {
1010
+ return super.get(cid);
1232
1011
  }
1233
1012
  };
1234
-
1235
- // src/runtime/keyed-crypto.ts
1236
- var keyed_crypto_exports = {};
1237
- __export(keyed_crypto_exports, {
1238
- BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1239
- keyedCryptoFactory: () => keyedCryptoFactory
1240
- });
1241
- import { base58btc as base58btc2 } from "multiformats/bases/base58";
1242
- import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1243
- import * as CBOR from "cborg";
1244
- var generateIV = {
1245
- random: {
1013
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1014
+ const logger = ensureLogger(sthis, component, ctx);
1015
+ return {
1246
1016
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
1247
- calc: async (ko, crypto, data) => {
1248
- return crypto.randomBytes(ko.ivLength);
1017
+ applyMeta: (meta, snap) => {
1018
+ return Promise.resolve();
1249
1019
  },
1250
1020
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
1251
- verify: async (ko, crypto, iv, data) => {
1252
- return true;
1021
+ compact: async (blocks) => {
1022
+ return {};
1023
+ },
1024
+ autoCompact: 100,
1025
+ public: false,
1026
+ // name: undefined,
1027
+ threshold: 1e3 * 1e3,
1028
+ ...opts,
1029
+ logger,
1030
+ keyBag: opts.keyBag || {},
1031
+ crypto: toCryptoRuntime2(opts.crypto),
1032
+ storeUrls: opts.storeUrls,
1033
+ // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
1034
+ // store,
1035
+ storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
1036
+ };
1037
+ }
1038
+ function blockstoreFactory(sthis, opts) {
1039
+ return new EncryptedBlockstore(sthis, opts);
1040
+ }
1041
+ var BaseBlockstore = class {
1042
+ constructor(ebOpts) {
1043
+ this.transactions = /* @__PURE__ */ new Set();
1044
+ this.sthis = ensureSuperThis(ebOpts);
1045
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1046
+ this.sthis = ensureSuperThis(ebOpts);
1047
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1048
+ this.logger = this.ebOpts.logger;
1049
+ }
1050
+ // readonly name?: string;
1051
+ // ready: Promise<void>;
1052
+ ready() {
1053
+ return Promise.resolve();
1054
+ }
1055
+ async close() {
1056
+ }
1057
+ async destroy() {
1058
+ }
1059
+ async compact() {
1060
+ }
1061
+ async get(cid) {
1062
+ if (!cid) throw this.logger.Error().Msg("required cid").AsError();
1063
+ for (const f of this.transactions) {
1064
+ const v = await f.superGet(cid);
1065
+ if (v) return v;
1253
1066
  }
1254
- },
1255
- hash: {
1256
- calc: async (ko, crypto, data) => {
1257
- const hash = await hasher4.digest(data);
1258
- const hashBytes = new Uint8Array(hash.bytes);
1259
- const hashArray = new Uint8Array(ko.ivLength);
1260
- for (let i = 0; i < hashBytes.length; i++) {
1261
- hashArray[i % ko.ivLength] ^= hashBytes[i];
1067
+ }
1068
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1069
+ async put(cid, block) {
1070
+ throw this.logger.Error().Msg("use a transaction to put").AsError();
1071
+ }
1072
+ // TransactionMeta
1073
+ async transaction(fn, _opts) {
1074
+ this.logger.Debug().Msg("enter transaction");
1075
+ const t = new CarTransaction(this, _opts);
1076
+ this.logger.Debug().Msg("post CarTransaction");
1077
+ const done = await fn(t);
1078
+ this.logger.Debug().Msg("post fn");
1079
+ this.lastTxMeta = done;
1080
+ return { t, meta: done };
1081
+ }
1082
+ openTransaction(opts = { add: true, noLoader: false }) {
1083
+ return new CarTransaction(this, opts);
1084
+ }
1085
+ async commitTransaction(t, done, opts) {
1086
+ if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
1087
+ const cars = await this.loader?.commit(t, done, opts);
1088
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1089
+ setTimeout(() => void this.compact(), 10);
1090
+ }
1091
+ if (cars) {
1092
+ this.transactions.delete(t);
1093
+ return { meta: done, cars, t };
1094
+ }
1095
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
1096
+ }
1097
+ async *entries() {
1098
+ const seen = /* @__PURE__ */ new Set();
1099
+ for (const t of this.transactions) {
1100
+ for await (const blk of t.entries()) {
1101
+ if (seen.has(blk.cid.toString())) continue;
1102
+ seen.add(blk.cid.toString());
1103
+ yield blk;
1262
1104
  }
1263
- return hashArray;
1264
- },
1265
- verify: async function(ko, crypto, iv, data) {
1266
- return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1267
1105
  }
1268
1106
  }
1269
1107
  };
1270
- function getGenerateIVFn(url, opts) {
1271
- const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
1272
- return generateIV[ivhash] || generateIV["hash"];
1273
- }
1274
- var BlockIvKeyIdCodec = class {
1275
- constructor(ko, iv, opts) {
1276
- this.code = 3147065;
1277
- this.name = "Fireproof@encrypted-block:aes-gcm";
1278
- this.ko = ko;
1279
- this.iv = iv;
1280
- this.opts = opts || {};
1281
- }
1282
- async encode(data) {
1283
- const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1284
- const { iv } = this.ko.algo(calcIv);
1285
- const fprt = await this.ko.fingerPrint();
1286
- const keyId = base58btc2.decode(fprt);
1287
- this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1288
- return CBOR.encode({
1289
- iv,
1290
- keyId,
1291
- data: await this.ko._encrypt({ iv, bytes: data })
1108
+ var EncryptedBlockstore = class extends BaseBlockstore {
1109
+ constructor(sthis, ebOpts) {
1110
+ super(ebOpts);
1111
+ this.compacting = false;
1112
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
1113
+ this: 1
1292
1114
  });
1115
+ this.loader = new Loader(sthis, ebOpts);
1293
1116
  }
1294
- async decode(abytes) {
1295
- let bytes;
1296
- if (abytes instanceof Uint8Array) {
1297
- bytes = abytes;
1298
- } else {
1299
- bytes = new Uint8Array(abytes);
1117
+ ready() {
1118
+ return this.loader.ready();
1119
+ }
1120
+ close() {
1121
+ return this.loader.close();
1122
+ }
1123
+ destroy() {
1124
+ return this.loader.destroy();
1125
+ }
1126
+ async get(cid) {
1127
+ const got = await super.get(cid);
1128
+ if (got) return got;
1129
+ if (!this.loader) {
1130
+ return;
1300
1131
  }
1301
- const { iv, keyId, data } = CBOR.decode(bytes);
1302
- const fprt = await this.ko.fingerPrint();
1303
- this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
1304
- if (base58btc2.encode(keyId) !== fprt) {
1305
- throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
1132
+ return falsyToUndef(await this.loader.getBlock(cid));
1133
+ }
1134
+ async transaction(fn, opts = { noLoader: false }) {
1135
+ this.logger.Debug().Msg("enter transaction");
1136
+ const { t, meta: done } = await super.transaction(fn);
1137
+ this.logger.Debug().Msg("post super.transaction");
1138
+ const cars = await this.loader.commit(t, done, opts);
1139
+ this.logger.Debug().Msg("post this.loader.commit");
1140
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1141
+ setTimeout(() => void this.compact(), 10);
1306
1142
  }
1307
- const result = await this.ko._decrypt({ iv, bytes: data });
1308
- if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1309
- throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1143
+ if (cars) {
1144
+ this.transactions.delete(t);
1145
+ return { meta: done, cars, t };
1310
1146
  }
1311
- return result;
1147
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
1312
1148
  }
1313
- };
1314
- var keyedCrypto = class {
1315
- constructor(url, key, cyopt, sthis) {
1316
- this.ivLength = 12;
1317
- this.isEncrypting = true;
1318
- this.logger = ensureLogger(sthis, "keyedCrypto");
1319
- this.crypto = cyopt;
1320
- this.key = key;
1321
- this.url = url;
1149
+ async getFile(car, cid) {
1150
+ await this.ready();
1151
+ if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
1152
+ const reader = await this.loader.loadFileCar(
1153
+ car
1154
+ /*, isPublic */
1155
+ );
1156
+ const block = await reader.get(cid);
1157
+ if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
1158
+ return block.bytes;
1322
1159
  }
1323
- fingerPrint() {
1324
- return Promise.resolve(this.key.fingerPrint);
1160
+ async compact() {
1161
+ await this.ready();
1162
+ if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
1163
+ if (this.loader.carLog.length < 2) return;
1164
+ const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
1165
+ if (!compactFn || this.compacting) return;
1166
+ const blockLog = new CompactionFetcher(this);
1167
+ this.compacting = true;
1168
+ const meta = await compactFn(blockLog);
1169
+ await this.loader?.commit(blockLog.loggedBlocks, meta, {
1170
+ compact: true,
1171
+ noLoader: true
1172
+ });
1173
+ this.compacting = false;
1325
1174
  }
1326
- codec(iv, opts) {
1327
- return new BlockIvKeyIdCodec(this, iv, opts);
1175
+ async defaultCompact(blocks, logger) {
1176
+ if (!this.loader) {
1177
+ throw logger.Error().Msg("no loader").AsError();
1178
+ }
1179
+ if (!this.lastTxMeta) {
1180
+ throw logger.Error().Msg("no lastTxMeta").AsError();
1181
+ }
1182
+ for await (const blk of this.loader.entries(false)) {
1183
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1184
+ }
1185
+ for (const t of this.transactions) {
1186
+ for await (const blk of t.entries()) {
1187
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1188
+ }
1189
+ }
1190
+ return this.lastTxMeta;
1328
1191
  }
1329
- algo(iv) {
1330
- return {
1331
- name: "AES-GCM",
1332
- iv: iv || this.crypto.randomBytes(this.ivLength),
1333
- tagLength: 128
1334
- };
1192
+ async *entries() {
1193
+ for await (const blk of this.loader.entries()) {
1194
+ yield blk;
1195
+ }
1335
1196
  }
1336
- async _decrypt(data) {
1337
- this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
1338
- return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
1197
+ };
1198
+ var CompactionFetcher = class {
1199
+ constructor(blocks) {
1200
+ this.blockstore = blocks;
1201
+ this.loggedBlocks = new CarTransaction(blocks);
1339
1202
  }
1340
- async _encrypt(data) {
1341
- this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
1342
- const a = this.algo(data.iv);
1343
- return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
1203
+ async get(cid) {
1204
+ const block = await this.blockstore.get(cid);
1205
+ if (block) this.loggedBlocks.putSync(cid, block.bytes);
1206
+ return falsyToUndef(block);
1344
1207
  }
1345
1208
  };
1346
- var nullCodec = class {
1347
- constructor() {
1348
- this.code = 0;
1349
- this.name = "Fireproof@unencrypted-block";
1209
+
1210
+ // src/blockstore/commitor.ts
1211
+ import * as CBW from "@ipld/car/buffer-writer";
1212
+ import { sha256 as hasher3 } from "multiformats/hashes/sha2";
1213
+ import * as dagCodec2 from "@ipld/dag-cbor";
1214
+ async function encodeCarFile(roots, t, codec3) {
1215
+ let size = 0;
1216
+ const headerSize = CBW.headerLength({ roots });
1217
+ size += headerSize;
1218
+ for (const { cid, bytes } of t.entries()) {
1219
+ size += CBW.blockLength({ cid, bytes });
1350
1220
  }
1351
- encode(data) {
1352
- return data;
1221
+ const buffer = new Uint8Array(size);
1222
+ const writer = CBW.createWriter(buffer, { headerSize });
1223
+ for (const r of roots) {
1224
+ writer.addRoot(r);
1353
1225
  }
1354
- decode(data) {
1355
- return data;
1226
+ for (const { cid, bytes } of t.entries()) {
1227
+ writer.write({ cid, bytes });
1356
1228
  }
1357
- };
1358
- var noCrypto = class {
1359
- constructor(url, cyrt, sthis) {
1360
- this.ivLength = 0;
1361
- this.code = 0;
1362
- this.name = "Fireproof@unencrypted-block";
1363
- this.isEncrypting = false;
1364
- this._fingerPrint = "noCrypto:" + Math.random();
1365
- this.logger = ensureLogger(sthis, "noCrypto");
1366
- this.crypto = cyrt;
1367
- this.url = url;
1229
+ writer.close();
1230
+ return await encode3({ value: writer.bytes, hasher: hasher3, codec: codec3 });
1231
+ }
1232
+ async function createCarFile(encoder, cid, t) {
1233
+ return encodeCarFile([cid], t, encoder);
1234
+ }
1235
+ async function commitFiles(fileStore, walStore, t, done) {
1236
+ const { files: roots } = makeFileCarHeader(done);
1237
+ const cids = [];
1238
+ const codec3 = (await fileStore.keyedCrypto()).codec();
1239
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
1240
+ for (const car of cars) {
1241
+ const { cid, bytes } = car;
1242
+ await fileStore.save({ cid, bytes });
1243
+ await walStore.enqueueFile(
1244
+ cid
1245
+ /*, !!opts.public*/
1246
+ );
1247
+ cids.push(cid);
1368
1248
  }
1369
- fingerPrint() {
1370
- return Promise.resolve(this._fingerPrint);
1249
+ return cids;
1250
+ }
1251
+ function makeFileCarHeader(result) {
1252
+ const files = [];
1253
+ for (const [, meta] of Object.entries(result.files || {})) {
1254
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1255
+ files.push(meta.cid);
1256
+ }
1371
1257
  }
1372
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1373
- codec(iv) {
1374
- return new nullCodec();
1258
+ return { ...result, files };
1259
+ }
1260
+ async function prepareCarFilesFiles(encoder, roots, t) {
1261
+ return [await encodeCarFile(roots, t, encoder)];
1262
+ }
1263
+ function makeCarHeader(meta, cars, compact = false) {
1264
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1265
+ return { ...coreHeader, meta };
1266
+ }
1267
+ async function encodeCarHeader(fp) {
1268
+ return await encode3({
1269
+ value: { fp },
1270
+ hasher: hasher3,
1271
+ codec: dagCodec2
1272
+ });
1273
+ }
1274
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
1275
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
1276
+ const rootBlock = await encodeCarHeader(fp);
1277
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
1278
+ const cids = [];
1279
+ for (const car of cars) {
1280
+ const { cid, bytes } = car;
1281
+ await params.carStore.save({ cid, bytes });
1282
+ cids.push(cid);
1375
1283
  }
1376
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1377
- algo(iv) {
1378
- return {
1379
- name: "noCrypto",
1380
- iv: new Uint8Array(),
1381
- tagLength: 0
1382
- };
1284
+ const newDbMeta = { cars: cids };
1285
+ await params.WALStore.enqueue(newDbMeta, opts);
1286
+ await params.metaStore.save(newDbMeta);
1287
+ return { cgrp: cids, header: fp };
1288
+ }
1289
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
1290
+ const carFiles = [];
1291
+ threshold = threshold || 128e3 * 8;
1292
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1293
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1294
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
1295
+ let cidRootBlock = rootBlock;
1296
+ for (const { cid, bytes } of t.entries()) {
1297
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
1298
+ if (newsize >= threshold) {
1299
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1300
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1301
+ clonedt.putSync(cid, bytes);
1302
+ cidRootBlock = { cid, bytes };
1303
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
1304
+ } else {
1305
+ clonedt.putSync(cid, bytes);
1306
+ }
1383
1307
  }
1384
- _decrypt() {
1385
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1308
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1309
+ return carFiles;
1310
+ }
1311
+
1312
+ // src/blockstore/loader.ts
1313
+ import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1314
+
1315
+ // src/blockstore/task-manager.ts
1316
+ var TaskManager = class {
1317
+ constructor(sthis, callback) {
1318
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
1319
+ this.queue = [];
1320
+ this.isProcessing = false;
1321
+ this.logger = ensureLogger(sthis, "TaskManager");
1322
+ this.callback = callback;
1386
1323
  }
1387
- _encrypt() {
1388
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1324
+ async handleEvent(cid, parents, dbMeta) {
1325
+ for (const parent of parents) {
1326
+ this.eventsWeHandled.add(parent.toString());
1327
+ }
1328
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
1329
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
1330
+ void this.processQueue();
1389
1331
  }
1390
- };
1391
- async function keyedCryptoFactory(url, kb, sthis) {
1392
- const storekey = url.getParam("storekey");
1393
- if (storekey && storekey !== "insecure") {
1394
- let rkey = await kb.getNamedKey(storekey, true);
1395
- if (rkey.isErr()) {
1396
- try {
1397
- rkey = await kb.toKeyWithFingerPrint(storekey);
1398
- } catch (e) {
1399
- throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
1332
+ async processQueue() {
1333
+ if (this.isProcessing) return;
1334
+ this.isProcessing = true;
1335
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1336
+ const first = filteredQueue[0];
1337
+ if (!first) {
1338
+ return;
1339
+ }
1340
+ try {
1341
+ await this.callback(first.dbMeta);
1342
+ this.eventsWeHandled.add(first.cid);
1343
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1344
+ } catch (err) {
1345
+ if (first.retries++ > 3) {
1346
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
1347
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
1348
+ }
1349
+ await new Promise((resolve) => setTimeout(resolve, 50));
1350
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
1351
+ } finally {
1352
+ this.isProcessing = false;
1353
+ if (this.queue.length > 0) {
1354
+ void this.processQueue();
1400
1355
  }
1401
1356
  }
1402
- return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
1403
1357
  }
1404
- return new noCrypto(url, kb.rt.crypto, sthis);
1405
- }
1358
+ };
1406
1359
 
1407
- // src/blockstore/fragment-gateway.ts
1408
- import { Result as Result3 } from "@adviser/cement";
1409
- import { base58btc as base58btc3 } from "multiformats/bases/base58";
1410
- import { encode as encode3, decode as decode3 } from "cborg";
1411
- function getFragSize(url) {
1412
- const fragSize = url.getParam("fragSize");
1413
- let ret = 0;
1414
- if (fragSize) {
1415
- ret = parseInt(fragSize);
1416
- }
1417
- if (isNaN(ret) || ret <= 0) {
1418
- ret = 0;
1360
+ // src/blockstore/loader.ts
1361
+ function carLogIncludesGroup(list, cids) {
1362
+ return list.some((arr) => {
1363
+ return arr.toString() === cids.toString();
1364
+ });
1365
+ }
1366
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
1367
+ const byString = /* @__PURE__ */ new Map();
1368
+ for (const cid of list) {
1369
+ if (remove.has(cid.toString())) continue;
1370
+ byString.set(cid.toString(), cid);
1419
1371
  }
1420
- return ret;
1372
+ return [...byString.values()];
1421
1373
  }
1422
- async function getFrags(url, innerGW, headerSize, logger) {
1423
- const fragSize = getFragSize(url);
1424
- if (!fragSize) {
1425
- const res = await innerGW.get(url);
1426
- if (res.isErr()) {
1427
- return [res];
1428
- }
1429
- const data = res.unwrap();
1430
- return [
1431
- Result3.Ok({
1432
- fid: new Uint8Array(0),
1433
- ofs: 0,
1434
- len: data.length,
1435
- data
1374
+ var Loader = class {
1375
+ constructor(sthis, ebOpts) {
1376
+ this.commitQueue = new CommitQueue();
1377
+ this.isCompacting = false;
1378
+ this.carReaders = /* @__PURE__ */ new Map();
1379
+ this.seenCompacted = /* @__PURE__ */ new Set();
1380
+ this.processedCars = /* @__PURE__ */ new Set();
1381
+ this.carLog = [];
1382
+ this.getBlockCache = /* @__PURE__ */ new Map();
1383
+ this.seenMeta = /* @__PURE__ */ new Set();
1384
+ this.writeLimit = pLimit(1);
1385
+ this._carStore = new ResolveOnce2();
1386
+ this._fileStore = new ResolveOnce2();
1387
+ this._WALStore = new ResolveOnce2();
1388
+ this._metaStore = new ResolveOnce2();
1389
+ this.onceReady = new ResolveOnce2();
1390
+ this.sthis = sthis;
1391
+ this.ebOpts = defaultedBlockstoreRuntime(
1392
+ sthis,
1393
+ {
1394
+ ...ebOpts
1395
+ // name,
1396
+ },
1397
+ "Loader"
1398
+ );
1399
+ this.logger = this.ebOpts.logger;
1400
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
1401
+ await this.handleDbMetasFromStore([dbMeta]);
1402
+ });
1403
+ }
1404
+ async carStore() {
1405
+ return this._carStore.once(
1406
+ async () => this.ebOpts.storeRuntime.makeDataStore({
1407
+ sthis: this.sthis,
1408
+ url: this.ebOpts.storeUrls.data,
1409
+ keybag: await this.keyBag()
1436
1410
  })
1437
- ];
1411
+ );
1438
1412
  }
1439
- const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
1440
- if (firstRaw.isErr()) {
1441
- return [firstRaw];
1413
+ async fileStore() {
1414
+ return this._fileStore.once(
1415
+ async () => this.ebOpts.storeRuntime.makeDataStore({
1416
+ sthis: this.sthis,
1417
+ url: this.ebOpts.storeUrls.file,
1418
+ keybag: await this.keyBag()
1419
+ })
1420
+ );
1442
1421
  }
1443
- const firstFragment = decode3(firstRaw.unwrap());
1444
- const blockSize = firstFragment.data.length;
1445
- const ops = [Promise.resolve(Result3.Ok(firstFragment))];
1446
- const fidStr = base58btc3.encode(firstFragment.fid);
1447
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
1448
- for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
1449
- ops.push(
1450
- (async (furl, ofs2) => {
1451
- const raw2 = await innerGW.get(furl);
1452
- if (raw2.isErr()) {
1453
- return raw2;
1454
- }
1455
- const fragment = decode3(raw2.unwrap());
1456
- if (base58btc3.encode(fragment.fid) !== fidStr) {
1457
- return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
1458
- }
1459
- if (fragment.ofs !== ofs2) {
1460
- return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
1461
- }
1462
- return Result3.Ok(fragment);
1463
- })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
1422
+ async WALStore() {
1423
+ return this._WALStore.once(
1424
+ async () => this.ebOpts.storeRuntime.makeWALStore({
1425
+ sthis: this.sthis,
1426
+ url: this.ebOpts.storeUrls.wal,
1427
+ keybag: await this.keyBag()
1428
+ })
1464
1429
  );
1465
1430
  }
1466
- return Promise.all(ops);
1467
- }
1468
- var FragmentGateway = class {
1469
- constructor(sthis, innerGW) {
1470
- this.fidLength = 4;
1471
- this.headerSize = 32;
1472
- this.sthis = ensureSuperLog(sthis, "FragmentGateway");
1473
- this.logger = this.sthis.logger;
1474
- this.innerGW = innerGW;
1431
+ async metaStore() {
1432
+ return this._metaStore.once(
1433
+ async () => this.ebOpts.storeRuntime.makeMetaStore({
1434
+ sthis: this.sthis,
1435
+ url: this.ebOpts.storeUrls.meta,
1436
+ keybag: await this.keyBag()
1437
+ })
1438
+ );
1475
1439
  }
1476
- slicer(url, body) {
1477
- const fragSize = getFragSize(url);
1478
- if (!fragSize) {
1479
- return [this.innerGW.put(url, body)];
1440
+ keyBag() {
1441
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
1442
+ }
1443
+ async ready() {
1444
+ return this.onceReady.once(async () => {
1445
+ const metas = await (await this.metaStore()).load();
1446
+ if (this.ebOpts.meta) {
1447
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
1448
+ } else if (metas) {
1449
+ await this.handleDbMetasFromStore(metas);
1450
+ }
1451
+ });
1452
+ }
1453
+ async close() {
1454
+ await this.commitQueue.waitIdle();
1455
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1456
+ await Promise.all(toClose.map((store) => store.close()));
1457
+ }
1458
+ async destroy() {
1459
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1460
+ await Promise.all(toDestroy.map((store) => store.destroy()));
1461
+ }
1462
+ // async snapToCar(carCid: AnyLink | string) {
1463
+ // await this.ready
1464
+ // if (typeof carCid === 'string') {
1465
+ // carCid = CID.parse(carCid)
1466
+ // }
1467
+ // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
1468
+ // this.carLog = [carCid, ...carHeader.cars]
1469
+ // await this.getMoreReaders(carHeader.cars)
1470
+ // await this._applyCarHeader(carHeader, true)
1471
+ // }
1472
+ async handleDbMetasFromStore(metas) {
1473
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
1474
+ for (const meta of metas) {
1475
+ await this.writeLimit(async () => {
1476
+ await this.mergeDbMetaIntoClock(meta);
1477
+ });
1480
1478
  }
1481
- const blocksize = fragSize - this.headerSize;
1482
- if (blocksize <= 0) {
1483
- throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
1479
+ }
1480
+ async mergeDbMetaIntoClock(meta) {
1481
+ if (this.isCompacting) {
1482
+ throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1484
1483
  }
1485
- const ops = [];
1486
- const fid = this.sthis.nextId(this.fidLength);
1487
- const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
1488
- for (let ofs = 0; ofs < body.length; ofs += blocksize) {
1489
- const block = encode3({
1490
- fid: fid.bin,
1491
- ofs,
1492
- len: body.length,
1493
- data: body.slice(ofs, ofs + blocksize)
1494
- });
1495
- if (block.length > fragSize) {
1496
- throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
1497
- }
1498
- ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
1484
+ if (this.seenMeta.has(meta.cars.toString())) return;
1485
+ this.seenMeta.add(meta.cars.toString());
1486
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
1487
+ return;
1499
1488
  }
1500
- return ops;
1489
+ const carHeader = await this.loadCarHeaderFromMeta(meta);
1490
+ carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1491
+ await this.getMoreReaders(carHeader.cars.flat());
1492
+ this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1493
+ await this.ebOpts.applyMeta?.(carHeader.meta);
1501
1494
  }
1502
- buildUrl(baseUrl, key) {
1503
- return this.innerGW.buildUrl(baseUrl, key);
1495
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1496
+ // const { key } = meta;
1497
+ // if (key) {
1498
+ // await this.setKey(key);
1499
+ // }
1500
+ // }
1501
+ async loadCarHeaderFromMeta({ cars: cids }) {
1502
+ const reader = await this.loadCar(cids[0]);
1503
+ return await parseCarFile(reader, this.logger);
1504
1504
  }
1505
- async destroy(iurl) {
1506
- return this.innerGW.destroy(iurl);
1505
+ // async _getKey(): Promise<string | undefined> {
1506
+ // if (this.key) return this.key;
1507
+ // // generate a random key
1508
+ // if (!this.ebOpts.public) {
1509
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1510
+ // }
1511
+ // return this.key || undefined;
1512
+ // }
1513
+ async commitFiles(t, done) {
1514
+ await this.ready();
1515
+ const fstore = await this.fileStore();
1516
+ const wstore = await this.WALStore();
1517
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1507
1518
  }
1508
- async start(url) {
1509
- this.headerSize = encode3({
1510
- fid: this.sthis.nextId(this.fidLength).bin,
1511
- ofs: 1024 * 1024,
1512
- // 32bit
1513
- len: 16 * 1024 * 1024,
1514
- // 32bit
1515
- data: new Uint8Array(1024)
1516
- }).length - 1024;
1517
- return this.innerGW.start(url);
1519
+ async loadFileCar(cid) {
1520
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1518
1521
  }
1519
- async close(url) {
1520
- return this.innerGW.close(url);
1522
+ async commit(t, done, opts = { noLoader: false, compact: false }) {
1523
+ await this.ready();
1524
+ const carStore = await this.carStore();
1525
+ const params = {
1526
+ encoder: (await carStore.keyedCrypto()).codec(),
1527
+ carLog: this.carLog,
1528
+ carStore,
1529
+ WALStore: await this.WALStore(),
1530
+ metaStore: await this.metaStore(),
1531
+ threshold: this.ebOpts.threshold
1532
+ };
1533
+ return this.commitQueue.enqueue(async () => {
1534
+ await this.cacheTransaction(t);
1535
+ const ret = await commit(params, t, done, opts);
1536
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1537
+ return ret.cgrp;
1538
+ });
1539
+ }
1540
+ async updateCarLog(cids, fp, compact) {
1541
+ if (compact) {
1542
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
1543
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1544
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1545
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1546
+ } else {
1547
+ this.carLog.unshift(cids);
1548
+ }
1549
+ }
1550
+ async cacheTransaction(t) {
1551
+ for await (const block of t.entries()) {
1552
+ const sBlock = block.cid.toString();
1553
+ if (!this.getBlockCache.has(sBlock)) {
1554
+ this.getBlockCache.set(sBlock, block);
1555
+ }
1556
+ }
1521
1557
  }
1522
- async put(url, body) {
1523
- await Promise.all(this.slicer(url, body));
1524
- return Result3.Ok(void 0);
1558
+ async cacheCarReader(carCidStr, reader) {
1559
+ if (this.processedCars.has(carCidStr)) return;
1560
+ this.processedCars.add(carCidStr);
1561
+ for await (const block of reader.blocks()) {
1562
+ const sBlock = block.cid.toString();
1563
+ if (!this.getBlockCache.has(sBlock)) {
1564
+ this.getBlockCache.set(sBlock, block);
1565
+ }
1566
+ }
1525
1567
  }
1526
- async get(url) {
1527
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1528
- let buffer = void 0;
1529
- for (const rfrag of rfrags) {
1530
- if (rfrag.isErr()) {
1531
- return Result3.Err(rfrag.Err());
1568
+ async removeCidsForCompact(cid) {
1569
+ const carHeader = await this.loadCarHeaderFromMeta({
1570
+ cars: [cid]
1571
+ });
1572
+ for (const cids of carHeader.compact) {
1573
+ for (const cid2 of cids) {
1574
+ await (await this.carStore()).remove(cid2);
1532
1575
  }
1533
- const frag = rfrag.Ok();
1534
- buffer = buffer || new Uint8Array(frag.len);
1535
- buffer.set(frag.data, frag.ofs);
1536
1576
  }
1537
- return Result3.Ok(buffer || new Uint8Array(0));
1538
1577
  }
1539
- async subscribe(url, callback) {
1540
- if (this.innerGW.subscribe) {
1541
- return this.innerGW.subscribe(url, callback);
1578
+ // async flushCars() {
1579
+ // await this.ready
1580
+ // // for each cid in car log, make a dbMeta
1581
+ // for (const cid of this.carLog) {
1582
+ // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1583
+ // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1584
+ // }
1585
+ // }
1586
+ async *entries(cache2 = true) {
1587
+ await this.ready();
1588
+ if (cache2) {
1589
+ for (const [, block] of this.getBlockCache) {
1590
+ yield block;
1591
+ }
1542
1592
  } else {
1543
- return Result3.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
1593
+ for (const [, block] of this.getBlockCache) {
1594
+ yield block;
1595
+ }
1596
+ for (const cids of this.carLog) {
1597
+ for (const cid of cids) {
1598
+ const reader = await this.loadCar(cid);
1599
+ if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1600
+ for await (const block of reader.blocks()) {
1601
+ const sCid = block.cid.toString();
1602
+ if (!this.getBlockCache.has(sCid)) {
1603
+ yield block;
1604
+ }
1605
+ }
1606
+ }
1607
+ }
1544
1608
  }
1545
1609
  }
1546
- async delete(url) {
1547
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1548
- for (const rfrag of rfrags) {
1549
- if (rfrag.isErr()) {
1550
- return Result3.Err(rfrag.Err());
1610
+ async getBlock(cid) {
1611
+ await this.ready();
1612
+ const sCid = cid.toString();
1613
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1614
+ const getCarCid = async (carCid) => {
1615
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1616
+ const reader = await this.loadCar(carCid);
1617
+ if (!reader) {
1618
+ throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1551
1619
  }
1552
- const frag = rfrag.Ok();
1553
- const fidStr = base58btc3.encode(frag.fid);
1554
- const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
1555
- await this.innerGW.delete(fragUrl);
1620
+ await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1621
+ return;
1622
+ });
1623
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1624
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1625
+ };
1626
+ const getCompactCarCids = async (carCid) => {
1627
+ const reader = await this.loadCar(carCid);
1628
+ if (!reader) {
1629
+ throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1630
+ }
1631
+ const header = await parseCarFile(reader, this.logger);
1632
+ const compacts = header.compact;
1633
+ let got2;
1634
+ const batchSize2 = 5;
1635
+ for (let i = 0; i < compacts.length; i += batchSize2) {
1636
+ const promises = [];
1637
+ for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1638
+ for (const cid2 of compacts[j]) {
1639
+ promises.push(getCarCid(cid2));
1640
+ }
1641
+ }
1642
+ try {
1643
+ got2 = await Promise.any(promises);
1644
+ } catch {
1645
+ }
1646
+ if (got2) break;
1647
+ }
1648
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1649
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1650
+ };
1651
+ let got;
1652
+ const batchSize = 5;
1653
+ for (let i = 0; i < this.carLog.length; i += batchSize) {
1654
+ const batch = this.carLog.slice(i, i + batchSize);
1655
+ const promises = batch.flatMap((slice) => slice.map(getCarCid));
1656
+ try {
1657
+ got = await Promise.any(promises);
1658
+ } catch {
1659
+ }
1660
+ if (got) break;
1556
1661
  }
1557
- return Result3.Ok(void 0);
1558
- }
1559
- };
1560
-
1561
- // src/blockstore/meta-key-helper.ts
1562
- import { format, parse } from "@ipld/dag-json";
1563
- import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
1564
- import { CID as CID2 } from "multiformats";
1565
- import { base64pad } from "multiformats/bases/base64";
1566
- import { Result as Result4 } from "@adviser/cement";
1567
- async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
1568
- const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
1569
- if (!crdtEntries.length) {
1570
- sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
1571
- return [];
1662
+ if (!got) {
1663
+ try {
1664
+ got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1665
+ } catch {
1666
+ }
1667
+ }
1668
+ return got;
1572
1669
  }
1573
- if (!crdtEntries.map) {
1574
- sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
1575
- return [];
1670
+ async loadCar(cid) {
1671
+ if (!this.carStore) {
1672
+ throw this.logger.Error().Msg("car store not initialized").AsError();
1673
+ }
1674
+ const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1675
+ return loaded;
1576
1676
  }
1577
- return Promise.all(
1578
- crdtEntries.map(async (crdtEntry) => {
1579
- const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
1580
- const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
1581
- return {
1582
- eventCid: eventBlock.cid,
1583
- parents: crdtEntry.parents,
1584
- dbMeta
1585
- };
1586
- })
1587
- );
1588
- }
1589
- async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
1590
- try {
1591
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
1592
- const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
1593
- if (keyInfo.length) {
1594
- const dbMeta = keyInfo[0].dbMeta;
1595
- if (dbMeta.key) {
1596
- const kb = await getKeyBag(sthis);
1597
- const keyName = getStoreKeyName(uri);
1598
- const res = await kb.setNamedKey(keyName, dbMeta.key);
1599
- if (res.isErr()) {
1600
- sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
1601
- throw res.Err();
1677
+ async makeDecoderAndCarReader(cid, local, remote) {
1678
+ const cidsString = cid.toString();
1679
+ let loadedCar = void 0;
1680
+ let activeStore = local;
1681
+ try {
1682
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1683
+ loadedCar = await local.load(cid);
1684
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1685
+ } catch (e) {
1686
+ if (remote) {
1687
+ const remoteCar = await remote.load(cid);
1688
+ if (remoteCar) {
1689
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1690
+ await local.save(remoteCar);
1691
+ loadedCar = remoteCar;
1692
+ activeStore = remote;
1602
1693
  }
1694
+ } else {
1695
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1603
1696
  }
1604
- sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
1605
- return Result4.Ok(dbMeta);
1606
1697
  }
1607
- sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
1608
- return Result4.Ok(void 0);
1609
- } catch (error) {
1610
- sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
1611
- return Result4.Err(error);
1612
- }
1613
- }
1614
- async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
1615
- try {
1616
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
1617
- const keyName = getStoreKeyName(uri);
1618
- const kb = await getKeyBag(sthis);
1619
- const res = await kb.getNamedExtractableKey(keyName, true);
1620
- if (res.isErr()) {
1621
- sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
1622
- throw res.Err();
1698
+ if (!loadedCar) {
1699
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
1623
1700
  }
1624
- const keyData = await res.Ok().extract();
1625
- const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
1626
- const { dbMeta, parents } = dbMetas[0];
1627
- const parentLinks = parents.map((p) => CID2.parse(p));
1628
- dbMeta.key = keyData.keyStr;
1629
- const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
1630
- const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
1631
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
1632
- return Result4.Ok(encoded);
1633
- } catch (error) {
1634
- sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
1635
- return Result4.Err(error);
1636
- }
1637
- }
1638
- function getStoreKeyName(url) {
1639
- const storeKeyName = [url.getParam("localName") || url.getParam("name")];
1640
- const idx = url.getParam("index");
1641
- if (idx) {
1642
- storeKeyName.push(idx);
1643
- }
1644
- storeKeyName.push("data");
1645
- return `@${storeKeyName.join(":")}@`;
1646
- }
1647
- async function createDbMetaEventBlock(sthis, dbMeta, parents) {
1648
- const event = await EventBlock.create(
1649
- {
1650
- dbMeta: sthis.txt.encode(format(dbMeta))
1651
- },
1652
- parents
1653
- );
1654
- return event;
1655
- }
1656
- async function encodeEventsWithParents(sthis, events, parents) {
1657
- const crdtEntries = events.map((event) => {
1658
- const base64String = base64pad.encode(event.bytes);
1659
- return {
1660
- cid: event.cid.toString(),
1661
- data: base64String,
1662
- parents: parents.map((p) => p.toString())
1663
- };
1664
- });
1665
- return sthis.txt.encode(JSON.stringify(crdtEntries));
1666
- }
1701
+ const bytes = await decode3({ bytes: loadedCar.bytes, hasher: hasher4, codec: (await activeStore.keyedCrypto()).codec() });
1702
+ const rawReader = await CarReader.fromBytes(bytes.value);
1703
+ const readerP = Promise.resolve(rawReader);
1704
+ const cachedReaderP = readerP.then(async (reader) => {
1705
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
1706
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1707
+ return;
1708
+ });
1709
+ return reader;
1710
+ });
1711
+ this.carReaders.set(cidsString, cachedReaderP);
1712
+ return readerP;
1713
+ }
1714
+ //What if instead it returns an Array of CarHeader
1715
+ async storesLoadCar(cid, local, remote) {
1716
+ const cidsString = cid.toString();
1717
+ let dacr = this.carReaders.get(cidsString);
1718
+ if (!dacr) {
1719
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
1720
+ this.carReaders.set(cidsString, dacr);
1721
+ }
1722
+ return dacr;
1723
+ }
1724
+ async getMoreReaders(cids) {
1725
+ const limit = pLimit(5);
1726
+ const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1727
+ await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1728
+ }
1729
+ };
1667
1730
 
1668
1731
  // src/blockstore/store.ts
1669
- import pRetry from "p-retry";
1670
- import pMap from "p-map";
1671
1732
  function guardVersion(url) {
1672
1733
  if (!url.hasParam("version")) {
1673
1734
  return Result5.Err(`missing version: ${url.toString()}`);
@@ -1675,16 +1736,21 @@ function guardVersion(url) {
1675
1736
  return Result5.Ok(url);
1676
1737
  }
1677
1738
  var BaseStoreImpl = class {
1678
- constructor(name, url, opts, sthis, logger) {
1739
+ constructor(sthis, url, opts, logger) {
1679
1740
  this._onStarted = [];
1680
1741
  this._onClosed = [];
1681
- this.name = name;
1682
1742
  this._url = url;
1683
1743
  this.keybag = opts.keybag;
1744
+ this.loader = opts.loader;
1684
1745
  this.sthis = sthis;
1685
- this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
1746
+ const name = this._url.getParam("name" /* NAME */);
1747
+ if (!name) {
1748
+ throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
1749
+ }
1750
+ this.name = name;
1751
+ this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
1752
+ this.realGateway = opts.gateway;
1686
1753
  this.gateway = new FragmentGateway(this.sthis, opts.gateway);
1687
- this.loader = opts.loader;
1688
1754
  }
1689
1755
  url() {
1690
1756
  return this._url;
@@ -1699,20 +1765,20 @@ var BaseStoreImpl = class {
1699
1765
  return;
1700
1766
  }
1701
1767
  async keyedCrypto() {
1702
- return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
1768
+ return keyedCryptoFactory(this._url, this.keybag, this.sthis);
1703
1769
  }
1704
1770
  async start() {
1705
1771
  this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
1706
- this._url = this._url.build().setParam("store", this.storeType).URI();
1772
+ this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
1707
1773
  const res = await this.gateway.start(this._url);
1708
1774
  if (res.isErr()) {
1709
1775
  this.logger.Error().Result("gw-start", res).Msg("started-gateway");
1710
1776
  return res;
1711
1777
  }
1712
1778
  this._url = res.Ok();
1713
- const kb = await this.keybag();
1779
+ const kb = await this.keybag;
1714
1780
  const skRes = await kb.ensureKeyFromUrl(this._url, () => {
1715
- const idx = this._url.getParam("index");
1781
+ const idx = this._url.getParam("index" /* INDEX */);
1716
1782
  const storeKeyName = [this.name];
1717
1783
  if (idx) {
1718
1784
  storeKeyName.push(idx);
@@ -1745,8 +1811,8 @@ var BaseStoreImpl = class {
1745
1811
  };
1746
1812
  var MetaStoreImpl = class extends BaseStoreImpl {
1747
1813
  // remote: boolean;
1748
- constructor(sthis, name, url, opts) {
1749
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
1814
+ constructor(sthis, url, opts) {
1815
+ super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
1750
1816
  this.storeType = "meta";
1751
1817
  this.subscribers = /* @__PURE__ */ new Map();
1752
1818
  this.parents = [];
@@ -1817,13 +1883,21 @@ var MetaStoreImpl = class extends BaseStoreImpl {
1817
1883
  return Result5.Ok(void 0);
1818
1884
  }
1819
1885
  async destroy() {
1886
+ this.logger.Debug().Msg("destroy");
1820
1887
  return this.gateway.destroy(this.url());
1821
1888
  }
1822
1889
  };
1823
1890
  var DataStoreImpl = class extends BaseStoreImpl {
1824
1891
  // readonly tag: string = "car-base";
1825
- constructor(sthis, name, url, opts) {
1826
- super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
1892
+ constructor(sthis, url, opts) {
1893
+ super(
1894
+ sthis,
1895
+ url,
1896
+ {
1897
+ ...opts
1898
+ },
1899
+ ensureLogger(sthis, "DataStoreImpl")
1900
+ );
1827
1901
  this.storeType = "data";
1828
1902
  }
1829
1903
  async load(cid) {
@@ -1864,23 +1938,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
1864
1938
  return Result5.Ok(void 0);
1865
1939
  }
1866
1940
  destroy() {
1941
+ this.logger.Debug().Msg("destroy");
1867
1942
  return this.gateway.destroy(this.url());
1868
1943
  }
1869
1944
  };
1870
1945
  var WALStoreImpl = class extends BaseStoreImpl {
1871
- constructor(loader, url, opts) {
1872
- super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
1946
+ constructor(sthis, url, opts) {
1947
+ super(
1948
+ sthis,
1949
+ url,
1950
+ {
1951
+ ...opts
1952
+ },
1953
+ ensureLogger(sthis, "WALStoreImpl")
1954
+ );
1873
1955
  this.storeType = "wal";
1956
+ // readonly tag: string = "rwal-base";
1957
+ // readonly loader: Loadable;
1874
1958
  this._ready = new ResolveOnce3();
1875
1959
  this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
1876
1960
  this.processing = void 0;
1877
1961
  this.processQueue = new CommitQueue();
1878
- this.loader = loader;
1879
1962
  }
1880
1963
  async ready() {
1881
1964
  return this._ready.once(async () => {
1882
1965
  const walState = await this.load().catch((e) => {
1883
- this.logger.Error().Any("error", e).Msg("error loading wal");
1966
+ this.logger.Error().Err(e).Msg("error loading wal");
1884
1967
  return void 0;
1885
1968
  });
1886
1969
  if (!walState) {
@@ -1913,7 +1996,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1913
1996
  }
1914
1997
  async process() {
1915
1998
  await this.ready();
1916
- if (!this.loader.remoteCarStore) return;
1999
+ if (!this.loader?.remoteCarStore) return;
1917
2000
  await this.processQueue.enqueue(async () => {
1918
2001
  try {
1919
2002
  await this._doProcess();
@@ -1926,6 +2009,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1926
2009
  });
1927
2010
  }
1928
2011
  async _doProcess() {
2012
+ if (!this.loader) return;
1929
2013
  if (!this.loader.remoteCarStore) return;
1930
2014
  const operations = [...this.walState.operations];
1931
2015
  const noLoaderOps = [...this.walState.noLoaderOps];
@@ -1943,6 +2027,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1943
2027
  noLoaderOps,
1944
2028
  async (dbMeta) => {
1945
2029
  await retryableUpload(async () => {
2030
+ if (!this.loader) return;
1946
2031
  for (const cid of dbMeta.cars) {
1947
2032
  const car = await (await this.loader.carStore()).load(cid);
1948
2033
  if (!car) {
@@ -1962,6 +2047,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1962
2047
  operations,
1963
2048
  async (dbMeta) => {
1964
2049
  await retryableUpload(async () => {
2050
+ if (!this.loader) return;
1965
2051
  for (const cid of dbMeta.cars) {
1966
2052
  const car = await (await this.loader.carStore()).load(cid);
1967
2053
  if (!car) {
@@ -1981,6 +2067,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1981
2067
  fileOperations,
1982
2068
  async ({ cid: fileCid, public: publicFile }) => {
1983
2069
  await retryableUpload(async () => {
2070
+ if (!this.loader) return;
1984
2071
  const fileBlock = await (await this.loader.fileStore()).load(fileCid);
1985
2072
  if (!fileBlock) {
1986
2073
  throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
@@ -1994,6 +2081,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1994
2081
  if (operations.length) {
1995
2082
  const lastOp = operations[operations.length - 1];
1996
2083
  await retryableUpload(async () => {
2084
+ if (!this.loader) return;
1997
2085
  await this.loader.remoteMetaStore?.save(lastOp);
1998
2086
  }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
1999
2087
  }
@@ -2019,6 +2107,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
2019
2107
  }
2020
2108
  try {
2021
2109
  return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
2110
+ return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
2022
2111
  } catch (e) {
2023
2112
  throw this.logger.Error().Err(e).Msg("error parse").AsError();
2024
2113
  }
@@ -2045,72 +2134,92 @@ var WALStoreImpl = class extends BaseStoreImpl {
2045
2134
  return Result5.Ok(void 0);
2046
2135
  }
2047
2136
  destroy() {
2137
+ this.logger.Debug().Msg("destroy");
2048
2138
  return this.gateway.destroy(this.url());
2049
2139
  }
2050
2140
  };
2051
2141
 
2052
- // src/blockstore/store-factory.ts
2053
- function ensureIsIndex(url, isIndex) {
2054
- if (isIndex) {
2055
- return url.build().setParam("index", isIndex).URI();
2142
+ // src/blockstore/register-store-protocol.ts
2143
+ import { BuildURI, runtimeFn as runtimeFn2 } from "@adviser/cement";
2144
+
2145
+ // src/runtime/gateways/memory/gateway.ts
2146
+ import { Result as Result6 } from "@adviser/cement";
2147
+
2148
+ // src/runtime/gateways/memory/version.ts
2149
+ var MEMORY_VERSION = "v0.19-memory";
2150
+
2151
+ // src/runtime/gateways/memory/gateway.ts
2152
+ var MemoryGateway = class {
2153
+ constructor(memorys) {
2154
+ this.memorys = memorys;
2056
2155
  }
2057
- return url.build().delParam("index").URI();
2058
- }
2059
- function ensureName(name, url) {
2060
- if (!url.hasParam("name")) {
2061
- return url.build().setParam("name", name).URI();
2156
+ buildUrl(baseUrl, key) {
2157
+ return Promise.resolve(Result6.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
2062
2158
  }
2063
- return url;
2064
- }
2159
+ start(baseUrl) {
2160
+ return Promise.resolve(Result6.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
2161
+ }
2162
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2163
+ close(baseUrl) {
2164
+ return Promise.resolve(Result6.Ok(void 0));
2165
+ }
2166
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
2167
+ destroy(baseUrl) {
2168
+ this.memorys.clear();
2169
+ return Promise.resolve(Result6.Ok(void 0));
2170
+ }
2171
+ put(url, body) {
2172
+ this.memorys.set(url.toString(), body);
2173
+ return Promise.resolve(Result6.Ok(void 0));
2174
+ }
2175
+ // get could return a NotFoundError if the key is not found
2176
+ get(url) {
2177
+ const x = this.memorys.get(url.toString());
2178
+ if (x === void 0) {
2179
+ return Promise.resolve(Result6.Err(new NotFoundError("not found")));
2180
+ }
2181
+ return Promise.resolve(Result6.Ok(x));
2182
+ }
2183
+ delete(url) {
2184
+ this.memorys.delete(url.toString());
2185
+ return Promise.resolve(Result6.Ok(void 0));
2186
+ }
2187
+ };
2188
+ var MemoryTestGateway = class {
2189
+ constructor(memorys) {
2190
+ this.memorys = memorys;
2191
+ }
2192
+ async get(url, key) {
2193
+ return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
2194
+ }
2195
+ };
2196
+
2197
+ // src/blockstore/register-store-protocol.ts
2065
2198
  var storeFactory = /* @__PURE__ */ new Map();
2066
- function buildURL(optURL, loader) {
2067
- const storeOpts = loader.ebOpts.store;
2068
- const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
2069
- let obuUrl;
2070
- if (obuItem && obuItem.overrideBaseURL) {
2071
- obuUrl = URI5.from(obuItem.overrideBaseURL);
2072
- }
2073
- const ret = ensureIsIndex(
2074
- URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2075
- storeOpts.isIndex
2076
- );
2077
- return ret;
2078
- }
2079
- var onceGateway = new KeyedResolvOnce2();
2080
- async function getGatewayFromURL(url, sthis) {
2081
- return onceGateway.get(url.toString()).once(async () => {
2082
- const item = storeFactory.get(url.protocol);
2083
- if (item) {
2084
- const ret = {
2085
- gateway: await item.gateway(sthis),
2086
- test: await item.test(sthis)
2087
- };
2088
- const res = await ret.gateway.start(url);
2089
- if (res.isErr()) {
2090
- sthis.logger.Error().Result("start", res).Msg("start failed");
2091
- return void 0;
2092
- }
2093
- return ret;
2199
+ function getDefaultURI(sthis, protocol) {
2200
+ if (protocol) {
2201
+ if (!protocol.endsWith(":")) {
2202
+ protocol += ":";
2094
2203
  }
2095
- sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2096
- return void 0;
2097
- });
2204
+ const gfi = storeFactory.get(protocol);
2205
+ if (gfi) {
2206
+ return gfi.defaultURI(sthis);
2207
+ }
2208
+ }
2209
+ const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
2210
+ if (!found) {
2211
+ throw sthis.logger.Error().Msg("no default found").AsError();
2212
+ }
2213
+ return found.defaultURI(sthis);
2098
2214
  }
2099
2215
  function registerStoreProtocol(item) {
2100
2216
  let protocol = item.protocol;
2101
2217
  if (!protocol.endsWith(":")) {
2102
2218
  protocol += ":";
2103
2219
  }
2104
- if (storeFactory.has(protocol)) {
2105
- if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
2106
- throw new Error(`we need a logger here`);
2107
- return () => {
2108
- };
2109
- }
2110
- }
2111
- if (item.overrideBaseURL) {
2220
+ if (item.isDefault) {
2112
2221
  Array.from(storeFactory.values()).forEach((items) => {
2113
- items.overrideBaseURL = void 0;
2222
+ items.isDefault = false;
2114
2223
  });
2115
2224
  }
2116
2225
  storeFactory.set(protocol, item);
@@ -2118,134 +2227,183 @@ function registerStoreProtocol(item) {
2118
2227
  storeFactory.delete(protocol);
2119
2228
  };
2120
2229
  }
2121
- var onceDataStoreFactory = new KeyedResolvOnce2();
2122
- async function dataStoreFactory(loader) {
2123
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
2124
- const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
2125
- return onceDataStoreFactory.get(url.toString()).once(async () => {
2126
- const gateway = await getGatewayFromURL(url, sthis);
2127
- if (!gateway) {
2128
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2230
+ function getGatewayFactoryItem(protocol) {
2231
+ return storeFactory.get(protocol);
2232
+ }
2233
+ function fileGatewayFactoryItem() {
2234
+ return {
2235
+ protocol: "file:",
2236
+ isDefault: true,
2237
+ defaultURI: (sthis) => {
2238
+ return BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
2239
+ },
2240
+ gateway: async (sthis) => {
2241
+ const { FileGateway } = await import("./gateway-GK5QZ6KP.js");
2242
+ return new FileGateway(sthis);
2243
+ },
2244
+ test: async (sthis) => {
2245
+ const { FileTestStore } = await import("./gateway-GK5QZ6KP.js");
2246
+ return new FileTestStore(sthis);
2247
+ }
2248
+ };
2249
+ }
2250
+ if (runtimeFn2().isBrowser) {
2251
+ registerStoreProtocol({
2252
+ protocol: "indexdb:",
2253
+ isDefault: true,
2254
+ defaultURI: () => {
2255
+ return BuildURI.from("indexdb://").pathname("fp").URI();
2256
+ },
2257
+ gateway: async (logger) => {
2258
+ const { IndexDBGateway } = await import("./gateway-TQTGDRCN.js");
2259
+ return new IndexDBGateway(logger);
2260
+ },
2261
+ test: async (logger) => {
2262
+ const { IndexDBTestStore } = await import("./gateway-TQTGDRCN.js");
2263
+ return new IndexDBTestStore(logger);
2129
2264
  }
2130
- const store = new DataStoreImpl(sthis, loader.name, url, {
2131
- gateway: gateway.gateway,
2132
- keybag: () => getKeyBag(loader.sthis, {
2133
- ...loader.ebOpts.keyBag
2134
- })
2135
- });
2136
- return store;
2137
2265
  });
2266
+ } else {
2267
+ registerStoreProtocol(fileGatewayFactoryItem());
2138
2268
  }
2139
- var onceMetaStoreFactory = new KeyedResolvOnce2();
2140
- async function metaStoreFactory(loader) {
2141
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
2142
- const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
2143
- return onceMetaStoreFactory.get(url.toString()).once(async () => {
2144
- sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
2145
- const gateway = await getGatewayFromURL(url, sthis);
2146
- if (!gateway) {
2147
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2148
- }
2149
- const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
2150
- gateway: gateway.gateway,
2151
- keybag: () => getKeyBag(loader.sthis, {
2152
- ...loader.ebOpts.keyBag
2153
- })
2154
- });
2155
- return store;
2269
+ var memory = /* @__PURE__ */ new Map();
2270
+ registerStoreProtocol({
2271
+ protocol: "memory:",
2272
+ isDefault: false,
2273
+ defaultURI: () => {
2274
+ return BuildURI.from("memory://").pathname("ram").URI();
2275
+ },
2276
+ gateway: async () => {
2277
+ return new MemoryGateway(memory);
2278
+ },
2279
+ test: async () => {
2280
+ return new MemoryTestGateway(memory);
2281
+ }
2282
+ });
2283
+
2284
+ // src/blockstore/store-factory.ts
2285
+ var onceGateway = new KeyedResolvOnce2();
2286
+ var gatewayInstances = new KeyedResolvOnce2();
2287
+ async function getStartedGateway(sthis, url) {
2288
+ return onceGateway.get(url.toString()).once(async () => {
2289
+ const item = getGatewayFactoryItem(url.protocol);
2290
+ if (item) {
2291
+ const ret = {
2292
+ url,
2293
+ ...await gatewayInstances.get(url.protocol).once(async () => ({
2294
+ gateway: await item.gateway(sthis),
2295
+ test: await item.test(sthis)
2296
+ }))
2297
+ };
2298
+ const res = await ret.gateway.start(url);
2299
+ if (res.isErr()) {
2300
+ return Result7.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
2301
+ }
2302
+ ret.url = res.Ok();
2303
+ return Result7.Ok(ret);
2304
+ }
2305
+ return Result7.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
2156
2306
  });
2157
2307
  }
2158
- var onceRemoteWalFactory = new KeyedResolvOnce2();
2159
- async function remoteWalFactory(loader) {
2160
- const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
2161
- const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
2162
- return onceRemoteWalFactory.get(url.toString()).once(async () => {
2163
- const gateway = await getGatewayFromURL(url, sthis);
2164
- if (!gateway) {
2165
- throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2166
- }
2167
- sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
2168
- const store = new WALStoreImpl(loader, url, {
2169
- gateway: gateway.gateway,
2170
- keybag: () => getKeyBag(loader.sthis, {
2171
- ...loader.ebOpts.keyBag
2172
- })
2173
- });
2174
- return store;
2308
+ async function dataStoreFactory(sfi) {
2309
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
2310
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2311
+ if (rgateway.isErr()) {
2312
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2313
+ }
2314
+ const gateway = rgateway.Ok();
2315
+ const store = new DataStoreImpl(sfi.sthis, gateway.url, {
2316
+ gateway: gateway.gateway,
2317
+ keybag: sfi.keybag
2318
+ });
2319
+ return store;
2320
+ }
2321
+ async function metaStoreFactory(sfi) {
2322
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
2323
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2324
+ if (rgateway.isErr()) {
2325
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2326
+ }
2327
+ const gateway = rgateway.Ok();
2328
+ const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
2329
+ gateway: gateway.gateway,
2330
+ keybag: sfi.keybag
2331
+ });
2332
+ return store;
2333
+ }
2334
+ async function WALStoreFactory(sfi) {
2335
+ const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
2336
+ const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2337
+ if (rgateway.isErr()) {
2338
+ throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2339
+ }
2340
+ const gateway = rgateway.Ok();
2341
+ const store = new WALStoreImpl(sfi.sthis, gateway.url, {
2342
+ gateway: gateway.gateway,
2343
+ keybag: sfi.keybag
2175
2344
  });
2345
+ return store;
2176
2346
  }
2177
2347
  async function testStoreFactory(url, sthis) {
2178
- sthis = ensureSuperLog(sthis, "testStoreFactory");
2179
- const gateway = await getGatewayFromURL(url, sthis);
2180
- if (!gateway) {
2348
+ const rgateway = await getStartedGateway(sthis, url);
2349
+ if (!rgateway) {
2181
2350
  throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2182
2351
  }
2183
- return gateway.test;
2352
+ return rgateway.Ok().test;
2184
2353
  }
2185
- async function ensureStart(store, logger) {
2354
+ async function ensureStart(store) {
2186
2355
  const ret = await store.start();
2187
2356
  if (ret.isErr()) {
2188
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
2357
+ throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
2189
2358
  }
2190
- logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2359
+ store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2191
2360
  return store;
2192
2361
  }
2193
- function toStoreRuntime(opts, sthis) {
2194
- const logger = ensureLogger(sthis, "toStoreRuntime", {});
2362
+ function ensureStoreEnDeFile(ende) {
2363
+ ende = ende || {};
2195
2364
  return {
2196
- makeMetaStore: async (loader) => {
2197
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
2198
- return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
2199
- },
2200
- makeDataStore: async (loader) => {
2201
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
2202
- return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
2203
- },
2204
- makeWALStore: async (loader) => {
2205
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
2206
- return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
2207
- },
2208
- encodeFile: opts.encodeFile || encodeFile,
2209
- decodeFile: opts.decodeFile || decodeFile
2365
+ encodeFile: ende.encodeFile || encodeFile,
2366
+ decodeFile: ende.decodeFile || decodeFile
2367
+ };
2368
+ }
2369
+ function toStoreRuntime(sthis, endeOpts = {}) {
2370
+ return {
2371
+ makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
2372
+ // async (loader: Loadable) => {
2373
+ // logger
2374
+ // .Debug()
2375
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
2376
+ // .Msg("makeMetaStore");
2377
+ // return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
2378
+ // },
2379
+ makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
2380
+ // async (loader: Loadable) => {
2381
+ // logger
2382
+ // .Debug()
2383
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
2384
+ // .Msg("makeDataStore");
2385
+ // return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
2386
+ // },
2387
+ makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
2388
+ // async (loader: Loadable) => {
2389
+ // logger
2390
+ // .Debug()
2391
+ // .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
2392
+ // .Msg("makeRemoteWAL");
2393
+ // return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
2394
+ // },
2395
+ ...ensureStoreEnDeFile(endeOpts)
2210
2396
  };
2211
2397
  }
2212
- registerStoreProtocol({
2213
- protocol: "file:",
2214
- gateway: async (sthis) => {
2215
- const { FileGateway } = await import("./gateway-5FCWPX5W.js");
2216
- return new FileGateway(sthis);
2217
- },
2218
- test: async (sthis) => {
2219
- const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
2220
- return new FileTestStore(sthis);
2221
- }
2222
- });
2223
- registerStoreProtocol({
2224
- protocol: "indexdb:",
2225
- gateway: async (sthis) => {
2226
- const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
2227
- return new IndexDBGateway(sthis);
2228
- },
2229
- test: async (sthis) => {
2230
- const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
2231
- return new IndexDBTestStore(sthis);
2232
- }
2233
- });
2234
2398
 
2235
2399
  // src/blockstore/store-remote.ts
2236
2400
  async function RemoteDataStore(sthis, name, url, opts) {
2237
- const ds = new DataStoreImpl(sthis, name, url, opts);
2401
+ const ds = new DataStoreImpl(sthis, url, opts);
2238
2402
  await ds.start();
2239
2403
  return ds;
2240
2404
  }
2241
2405
  async function RemoteMetaStore(sthis, name, url, opts) {
2242
- const ms = new MetaStoreImpl(
2243
- sthis,
2244
- name,
2245
- url,
2246
- opts
2247
- /* , true*/
2248
- );
2406
+ const ms = new MetaStoreImpl(sthis, url, opts);
2249
2407
  await ms.start();
2250
2408
  return ms;
2251
2409
  }
@@ -2270,14 +2428,17 @@ var ConnectionBase = class {
2270
2428
  if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
2271
2429
  this.loader = loader;
2272
2430
  await this.onConnect();
2273
- const metaUrl = this.url.build().defParam("store", "meta").URI();
2274
- const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
2275
- if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2431
+ const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
2432
+ const rgateway = await getStartedGateway(loader.sthis, metaUrl);
2433
+ if (rgateway.isErr())
2434
+ throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2435
+ const name = metaUrl.toString();
2276
2436
  const dbName = metaUrl.getParam("name");
2277
- if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
2278
- const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
2437
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
2438
+ const gateway = rgateway.Ok();
2439
+ const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
2279
2440
  gateway: gateway.gateway,
2280
- keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
2441
+ keybag: await loader.keyBag(),
2281
2442
  loader
2282
2443
  });
2283
2444
  this.loader.remoteMetaStore = remote;
@@ -2290,14 +2451,15 @@ var ConnectionBase = class {
2290
2451
  async connectStorage_X({ loader }) {
2291
2452
  if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
2292
2453
  this.loader = loader;
2293
- const dataUrl = this.url.build().defParam("store", "data").URI();
2294
- const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
2295
- if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2454
+ const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
2455
+ const rgateway = await getStartedGateway(loader.sthis, dataUrl);
2456
+ if (rgateway.isErr())
2457
+ throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2296
2458
  const name = dataUrl.getParam("name");
2297
2459
  if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
2298
2460
  loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
2299
- gateway: gateway.gateway,
2300
- keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
2461
+ gateway: rgateway.Ok().gateway,
2462
+ keybag: await loader.keyBag()
2301
2463
  });
2302
2464
  loader.remoteFileStore = loader.remoteCarStore;
2303
2465
  }
@@ -2337,6 +2499,12 @@ var ConnectionBase = class {
2337
2499
  };
2338
2500
 
2339
2501
  // src/crdt-helpers.ts
2502
+ import { parse as parse3 } from "multiformats/link";
2503
+ import { sha256 as hasher5 } from "multiformats/hashes/sha2";
2504
+ import * as codec from "@ipld/dag-cbor";
2505
+ import { put, get, entries, root } from "@web3-storage/pail/crdt";
2506
+ import { EventFetcher, vis } from "@web3-storage/pail/clock";
2507
+ import * as Batch from "@web3-storage/pail/crdt/batch";
2340
2508
  function time(tag) {
2341
2509
  }
2342
2510
  function timeEnd(tag) {
@@ -2384,7 +2552,7 @@ async function writeDocContent(store, blocks, update, logger) {
2384
2552
  await processFiles(store, blocks, update.value, logger);
2385
2553
  value = { doc: update.value };
2386
2554
  }
2387
- const block = await encode({ value, hasher: hasher5, codec });
2555
+ const block = await encode3({ value, hasher: hasher5, codec });
2388
2556
  blocks.putSync(block.cid, block.bytes);
2389
2557
  return block.cid;
2390
2558
  }
@@ -2475,7 +2643,7 @@ function readFileset(blocks, files, isPublic = false) {
2475
2643
  async function getValueFromLink(blocks, link, logger) {
2476
2644
  const block = await blocks.get(link);
2477
2645
  if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
2478
- const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
2646
+ const { value } = await decode3({ bytes: block.bytes, hasher: hasher5, codec });
2479
2647
  const cvalue = {
2480
2648
  ...value,
2481
2649
  cid: link
@@ -2596,7 +2764,7 @@ async function doCompact(blockLog, head, logger) {
2596
2764
  async function getBlock(blocks, cidString) {
2597
2765
  const block = await blocks.get(parse3(cidString));
2598
2766
  if (!block) throw new Error(`Missing block ${cidString}`);
2599
- const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
2767
+ const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: hasher5 });
2600
2768
  return new Block({ cid, value, bytes: block.bytes });
2601
2769
  }
2602
2770
 
@@ -2654,7 +2822,8 @@ function makeProllyGetBlock(blocks) {
2654
2822
  return create({ cid, bytes, hasher: hasher6, codec: codec2 });
2655
2823
  };
2656
2824
  }
2657
- async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
2825
+ async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
2826
+ logger.Debug().Msg("enter bulkIndex");
2658
2827
  if (!indexEntries.length) return inIndex;
2659
2828
  if (!inIndex.root) {
2660
2829
  if (!inIndex.cid) {
@@ -2671,18 +2840,22 @@ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
2671
2840
  returnNode = node;
2672
2841
  }
2673
2842
  if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
2843
+ logger.Debug().Msg("exit !root bulkIndex");
2674
2844
  return { root: returnNode, cid: returnRootBlock.cid };
2675
2845
  } else {
2676
2846
  inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
2677
2847
  }
2678
2848
  }
2849
+ logger.Debug().Msg("pre bulk bulkIndex");
2679
2850
  const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
2680
2851
  if (root3) {
2852
+ logger.Debug().Msg("pre root put bulkIndex");
2681
2853
  for await (const block of newBlocks) {
2682
2854
  await tblocks.put(block.cid, block.bytes);
2683
2855
  }
2684
2856
  return { root: root3, cid: (await root3.block).cid };
2685
2857
  } else {
2858
+ logger.Debug().Msg("pre !root bulkIndex");
2686
2859
  return { root: void 0, cid: void 0 };
2687
2860
  }
2688
2861
  }
@@ -2722,17 +2895,17 @@ function encodeKey(key) {
2722
2895
  }
2723
2896
 
2724
2897
  // src/indexer.ts
2725
- function index(sthis, { _crdt }, name, mapFn, meta) {
2726
- if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2727
- if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
2728
- if (_crdt.indexers.has(name)) {
2729
- const idx = _crdt.indexers.get(name);
2898
+ function index(refDb, name, mapFn, meta) {
2899
+ if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2900
+ if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
2901
+ if (refDb.crdt.indexers.has(name)) {
2902
+ const idx = refDb.crdt.indexers.get(name);
2730
2903
  idx.applyMapFn(name, mapFn, meta);
2731
2904
  } else {
2732
- const idx = new Index(sthis, _crdt, name, mapFn, meta);
2733
- _crdt.indexers.set(name, idx);
2905
+ const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
2906
+ refDb.crdt.indexers.set(name, idx);
2734
2907
  }
2735
- return _crdt.indexers.get(name);
2908
+ return refDb.crdt.indexers.get(name);
2736
2909
  }
2737
2910
  var Index = class {
2738
2911
  constructor(sthis, crdt, name, mapFn, meta) {
@@ -2751,18 +2924,9 @@ var Index = class {
2751
2924
  return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
2752
2925
  });
2753
2926
  }
2754
- close() {
2755
- return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
2756
- });
2757
- }
2758
- destroy() {
2759
- return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
2760
- });
2761
- }
2762
2927
  applyMapFn(name, mapFn, meta) {
2763
2928
  if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2764
2929
  if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
2765
- this.name = name;
2766
2930
  try {
2767
2931
  if (meta) {
2768
2932
  if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
@@ -2810,9 +2974,13 @@ var Index = class {
2810
2974
  }
2811
2975
  }
2812
2976
  async query(opts = {}) {
2977
+ this.logger.Debug().Msg("enter query");
2813
2978
  await this.ready();
2979
+ this.logger.Debug().Msg("post ready query");
2814
2980
  await this._updateIndex();
2981
+ this.logger.Debug().Msg("post _updateIndex query");
2815
2982
  await this._hydrateIndex();
2983
+ this.logger.Debug().Msg("post _hydrateIndex query");
2816
2984
  if (!this.byKey.root) {
2817
2985
  return await applyQuery(this.crdt, { result: [] }, opts);
2818
2986
  }
@@ -2868,13 +3036,16 @@ var Index = class {
2868
3036
  }
2869
3037
  async _updateIndex() {
2870
3038
  await this.ready();
3039
+ this.logger.Debug().Msg("enter _updateIndex");
2871
3040
  if (this.initError) throw this.initError;
2872
3041
  if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
2873
3042
  let result, head;
2874
3043
  if (!this.indexHead || this.indexHead.length === 0) {
2875
3044
  ({ result, head } = await this.crdt.allDocs());
3045
+ this.logger.Debug().Msg("enter crdt.allDocs");
2876
3046
  } else {
2877
3047
  ({ result, head } = await this.crdt.changes(this.indexHead));
3048
+ this.logger.Debug().Msg("enter crdt.changes");
2878
3049
  }
2879
3050
  if (result.length === 0) {
2880
3051
  this.indexHead = head;
@@ -2907,9 +3078,22 @@ var Index = class {
2907
3078
  if (result.length === 0) {
2908
3079
  return indexerMeta;
2909
3080
  }
3081
+ this.logger.Debug().Msg("pre this.blockstore.transaction");
2910
3082
  const { meta } = await this.blockstore.transaction(async (tblocks) => {
2911
- this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
2912
- this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
3083
+ this.byId = await bulkIndex(
3084
+ this.logger,
3085
+ tblocks,
3086
+ this.byId,
3087
+ removeIdIndexEntries.concat(byIdIndexEntries),
3088
+ byIdOpts
3089
+ );
3090
+ this.byKey = await bulkIndex(
3091
+ this.logger,
3092
+ tblocks,
3093
+ this.byKey,
3094
+ staleKeyIndexEntries.concat(indexEntries),
3095
+ byKeyOpts
3096
+ );
2913
3097
  this.indexHead = head;
2914
3098
  if (this.byId.cid && this.byKey.cid) {
2915
3099
  const idxMeta = {
@@ -2921,8 +3105,10 @@ var Index = class {
2921
3105
  };
2922
3106
  indexerMeta.indexes?.set(this.name, idxMeta);
2923
3107
  }
3108
+ this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
2924
3109
  return indexerMeta;
2925
3110
  });
3111
+ this.logger.Debug().Msg("post this.blockstore.transaction");
2926
3112
  return meta;
2927
3113
  }
2928
3114
  };
@@ -3050,21 +3236,23 @@ var CRDTClock = class {
3050
3236
  throw this.logger.Error().Msg("missing blockstore").AsError();
3051
3237
  }
3052
3238
  await validateBlocks(this.logger, newHead, this.blockstore);
3053
- const { meta } = await this.blockstore.transaction(
3054
- async (tblocks) => {
3055
- const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3056
- const result = await root2(tblocks, advancedHead);
3057
- for (const { cid, bytes } of [
3058
- ...result.additions
3059
- // ...result.removals
3060
- ]) {
3061
- tblocks.putSync(cid, bytes);
3062
- }
3063
- return { head: advancedHead };
3064
- },
3065
- { noLoader, add: false }
3066
- );
3067
- this.setHead(meta.head);
3239
+ if (!this.transaction) {
3240
+ this.transaction = this.blockstore.openTransaction({ noLoader, add: false });
3241
+ }
3242
+ const tblocks = this.transaction;
3243
+ const advancedHead = await advanceBlocks(this.logger, newHead, tblocks, this.head);
3244
+ const result = await root2(tblocks, advancedHead);
3245
+ for (const { cid, bytes } of [
3246
+ ...result.additions
3247
+ // ...result.removals
3248
+ ]) {
3249
+ tblocks.putSync(cid, bytes);
3250
+ }
3251
+ if (!noLoader) {
3252
+ await this.blockstore.commitTransaction(tblocks, { head: advancedHead }, { add: false, noLoader });
3253
+ this.transaction = void 0;
3254
+ }
3255
+ this.setHead(advancedHead);
3068
3256
  }
3069
3257
  };
3070
3258
  function sortClockHead(clockHead) {
@@ -3096,15 +3284,13 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
3096
3284
 
3097
3285
  // src/crdt.ts
3098
3286
  var CRDT = class {
3099
- constructor(sthis, name, opts = {}) {
3287
+ constructor(sthis, opts) {
3100
3288
  this.indexers = /* @__PURE__ */ new Map();
3101
3289
  this.onceReady = new ResolveOnce5();
3102
3290
  this.sthis = sthis;
3103
- this.name = name;
3104
3291
  this.logger = ensureLogger(sthis, "CRDT");
3105
3292
  this.opts = opts;
3106
3293
  this.blockstore = blockstoreFactory(sthis, {
3107
- name,
3108
3294
  applyMeta: async (meta) => {
3109
3295
  const crdtMeta = meta;
3110
3296
  if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
@@ -3114,23 +3300,27 @@ var CRDT = class {
3114
3300
  await doCompact(blocks, this.clock.head, this.logger);
3115
3301
  return { head: this.clock.head };
3116
3302
  },
3117
- autoCompact: this.opts.autoCompact || 100,
3118
- store: { ...this.opts.store, isIndex: void 0 },
3119
- public: this.opts.public,
3120
- meta: this.opts.meta,
3121
- threshold: this.opts.threshold
3303
+ // autoCompact: this.opts.autoCompact || 100,
3304
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
3305
+ storeUrls: this.opts.storeUrls.data,
3306
+ keyBag: this.opts.keyBag,
3307
+ // public: this.opts.public,
3308
+ meta: this.opts.meta
3309
+ // threshold: this.opts.threshold,
3122
3310
  });
3123
3311
  this.indexBlockstore = blockstoreFactory(sthis, {
3124
- name,
3312
+ // name: opts.name,
3125
3313
  applyMeta: async (meta) => {
3126
3314
  const idxCarMeta = meta;
3127
3315
  if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
3128
- for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
3129
- index(this.sthis, { _crdt: this }, name2, void 0, idx);
3316
+ for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
3317
+ index({ crdt: this }, name, void 0, idx);
3130
3318
  }
3131
3319
  },
3132
- store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
3133
- public: this.opts.public
3320
+ storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
3321
+ storeUrls: this.opts.storeUrls.idx,
3322
+ keyBag: this.opts.keyBag
3323
+ // public: this.opts.public,
3134
3324
  });
3135
3325
  this.clock = new CRDTClock(this.blockstore);
3136
3326
  this.clock.onZoom(() => {
@@ -3212,51 +3402,164 @@ var CRDT = class {
3212
3402
  };
3213
3403
 
3214
3404
  // src/database.ts
3215
- var Database = class {
3216
- constructor(name, opts) {
3217
- this.opts = {};
3405
+ var databases = new KeyedResolvOnce3();
3406
+ function toSortedArray(set) {
3407
+ if (!set) return [];
3408
+ return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
3409
+ }
3410
+ function keyConfigOpts(sthis, name, opts) {
3411
+ return JSON.stringify(
3412
+ toSortedArray({
3413
+ name,
3414
+ stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
3415
+ })
3416
+ );
3417
+ }
3418
+ function isDatabase(db) {
3419
+ return db instanceof DatabaseImpl || db instanceof DatabaseShell;
3420
+ }
3421
+ function DatabaseFactory(name, opts) {
3422
+ const sthis = ensureSuperThis(opts);
3423
+ return new DatabaseShell(
3424
+ databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
3425
+ const db = new DatabaseImpl(sthis, {
3426
+ name,
3427
+ meta: opts?.meta,
3428
+ keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
3429
+ storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
3430
+ storeEnDe: {
3431
+ encodeFile,
3432
+ decodeFile,
3433
+ ...opts?.storeEnDe
3434
+ }
3435
+ });
3436
+ db.onClosed(() => {
3437
+ databases.unget(key);
3438
+ });
3439
+ return db;
3440
+ })
3441
+ );
3442
+ }
3443
+ var DatabaseShell = class {
3444
+ constructor(ref) {
3445
+ this.ref = ref;
3446
+ ref.addShell(this);
3447
+ }
3448
+ get id() {
3449
+ return this.ref.id;
3450
+ }
3451
+ get logger() {
3452
+ return this.ref.logger;
3453
+ }
3454
+ get sthis() {
3455
+ return this.ref.sthis;
3456
+ }
3457
+ get crdt() {
3458
+ return this.ref.crdt;
3459
+ }
3460
+ name() {
3461
+ return this.ref.name();
3462
+ }
3463
+ onClosed(fn) {
3464
+ return this.ref.onClosed(fn);
3465
+ }
3466
+ close() {
3467
+ return this.ref.shellClose(this);
3468
+ }
3469
+ destroy() {
3470
+ return this.ref.destroy();
3471
+ }
3472
+ ready() {
3473
+ return this.ref.ready();
3474
+ }
3475
+ get(id) {
3476
+ return this.ref.get(id);
3477
+ }
3478
+ put(doc) {
3479
+ return this.ref.put(doc);
3480
+ }
3481
+ del(id) {
3482
+ return this.ref.del(id);
3483
+ }
3484
+ changes(since, opts) {
3485
+ return this.ref.changes(since, opts);
3486
+ }
3487
+ allDocs(opts) {
3488
+ return this.ref.allDocs(opts);
3489
+ }
3490
+ allDocuments() {
3491
+ return this.ref.allDocuments();
3492
+ }
3493
+ subscribe(listener, updates) {
3494
+ return this.ref.subscribe(listener, updates);
3495
+ }
3496
+ query(field, opts) {
3497
+ return this.ref.query(field, opts);
3498
+ }
3499
+ compact() {
3500
+ return this.ref.compact();
3501
+ }
3502
+ };
3503
+ var DatabaseImpl = class {
3504
+ constructor(sthis, opts) {
3218
3505
  this._listening = false;
3219
3506
  this._listeners = /* @__PURE__ */ new Set();
3220
3507
  this._noupdate_listeners = /* @__PURE__ */ new Set();
3508
+ // readonly blockstore: BaseBlockstore;
3509
+ this.shells = /* @__PURE__ */ new Set();
3510
+ this._onClosedFns = /* @__PURE__ */ new Set();
3221
3511
  this._ready = new ResolveOnce6();
3222
- this.name = name;
3223
- this.opts = opts || this.opts;
3224
- this.sthis = ensureSuperThis(this.opts);
3512
+ this.opts = opts;
3513
+ this.sthis = sthis;
3514
+ this.id = sthis.timeOrderedNextId().str;
3225
3515
  this.logger = ensureLogger(this.sthis, "Database");
3226
- this._crdt = new CRDT(this.sthis, name, this.opts);
3227
- this.blockstore = this._crdt.blockstore;
3516
+ this.crdt = new CRDT(this.sthis, this.opts);
3228
3517
  this._writeQueue = writeQueue(async (updates) => {
3229
- return await this._crdt.bulk(updates);
3518
+ return await this.crdt.bulk(updates);
3230
3519
  });
3231
- this._crdt.clock.onTock(() => {
3520
+ this.crdt.clock.onTock(() => {
3232
3521
  this._no_update_notify();
3233
3522
  });
3234
3523
  }
3235
- static {
3236
- this.databases = /* @__PURE__ */ new Map();
3524
+ addShell(shell) {
3525
+ this.shells.add(shell);
3526
+ }
3527
+ onClosed(fn) {
3528
+ this._onClosedFns.add(fn);
3237
3529
  }
3238
3530
  async close() {
3239
- await this.ready();
3240
- await this._crdt.close();
3241
- await this.blockstore.close();
3531
+ throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
3532
+ }
3533
+ async shellClose(db) {
3534
+ if (!this.shells.has(db)) {
3535
+ throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
3536
+ }
3537
+ this.shells.delete(db);
3538
+ if (this.shells.size === 0) {
3539
+ await this.ready();
3540
+ await this.crdt.close();
3541
+ this._onClosedFns.forEach((fn) => fn());
3542
+ }
3242
3543
  }
3243
3544
  async destroy() {
3244
3545
  await this.ready();
3245
- await this._crdt.destroy();
3246
- await this.blockstore.destroy();
3546
+ await this.crdt.destroy();
3247
3547
  }
3248
3548
  async ready() {
3249
- return this._ready.once(async () => {
3549
+ const ret = await this._ready.once(async () => {
3250
3550
  await this.sthis.start();
3251
- await this._crdt.ready();
3252
- await this.blockstore.ready();
3551
+ await this.crdt.ready();
3253
3552
  });
3553
+ return ret;
3554
+ }
3555
+ name() {
3556
+ return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
3254
3557
  }
3255
3558
  async get(id) {
3256
- if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
3559
+ if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
3257
3560
  await this.ready();
3258
3561
  this.logger.Debug().Str("id", id).Msg("get");
3259
- const got = await this._crdt.get(id).catch((e) => {
3562
+ const got = await this.crdt.get(id).catch((e) => {
3260
3563
  throw new NotFoundError(`Not found: ${id} - ${e.message}`);
3261
3564
  });
3262
3565
  if (!got) throw new NotFoundError(`Not found: ${id}`);
@@ -3275,34 +3578,34 @@ var Database = class {
3275
3578
  _id: docId
3276
3579
  }
3277
3580
  });
3278
- return { id: docId, clock: result?.head, name: this.name };
3581
+ return { id: docId, clock: result?.head, name: this.name() };
3279
3582
  }
3280
3583
  async del(id) {
3281
3584
  await this.ready();
3282
3585
  this.logger.Debug().Str("id", id).Msg("del");
3283
3586
  const result = await this._writeQueue.push({ id, del: true });
3284
- return { id, clock: result?.head, name: this.name };
3587
+ return { id, clock: result?.head, name: this.name() };
3285
3588
  }
3286
3589
  async changes(since = [], opts = {}) {
3287
3590
  await this.ready();
3288
3591
  this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
3289
- const { result, head } = await this._crdt.changes(since, opts);
3592
+ const { result, head } = await this.crdt.changes(since, opts);
3290
3593
  const rows = result.map(({ id: key, value, del, clock }) => ({
3291
3594
  key,
3292
3595
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
3293
3596
  clock
3294
3597
  }));
3295
- return { rows, clock: head, name: this.name };
3598
+ return { rows, clock: head, name: this.name() };
3296
3599
  }
3297
3600
  async allDocs(opts = {}) {
3298
3601
  await this.ready();
3299
3602
  this.logger.Debug().Msg("allDocs");
3300
- const { result, head } = await this._crdt.allDocs();
3603
+ const { result, head } = await this.crdt.allDocs();
3301
3604
  const rows = result.map(({ id: key, value, del }) => ({
3302
3605
  key,
3303
3606
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
3304
3607
  }));
3305
- return { rows, clock: head, name: this.name };
3608
+ return { rows, clock: head, name: this.name() };
3306
3609
  }
3307
3610
  async allDocuments() {
3308
3611
  return this.allDocs();
@@ -3312,7 +3615,7 @@ var Database = class {
3312
3615
  if (updates) {
3313
3616
  if (!this._listening) {
3314
3617
  this._listening = true;
3315
- this._crdt.clock.onTick((updates2) => {
3618
+ this.crdt.clock.onTick((updates2) => {
3316
3619
  void this._notify(updates2);
3317
3620
  });
3318
3621
  }
@@ -3331,13 +3634,13 @@ var Database = class {
3331
3634
  async query(field, opts = {}) {
3332
3635
  await this.ready();
3333
3636
  this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
3334
- const _crdt = this._crdt;
3335
- const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
3637
+ const _crdt = this.crdt;
3638
+ const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
3336
3639
  return await idx.query(opts);
3337
3640
  }
3338
3641
  async compact() {
3339
3642
  await this.ready();
3340
- await this._crdt.compact();
3643
+ await this.crdt.compact();
3341
3644
  }
3342
3645
  async _notify(updates) {
3343
3646
  await this.ready();
@@ -3361,23 +3664,62 @@ var Database = class {
3361
3664
  }
3362
3665
  }
3363
3666
  };
3364
- function toSortedArray(set) {
3365
- if (!set) return [];
3366
- return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
3667
+ function defaultURI(sthis, curi, uri, store, ctx) {
3668
+ ctx = ctx || {};
3669
+ const ret = (curi ? URI8.from(curi) : uri).build().setParam("store" /* STORE */, store);
3670
+ if (!ret.hasParam("name" /* NAME */)) {
3671
+ const name = sthis.pathOps.basename(ret.URI().pathname);
3672
+ if (!name) {
3673
+ throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
3674
+ }
3675
+ ret.setParam("name" /* NAME */, name);
3676
+ }
3677
+ if (ctx.idx) {
3678
+ ret.defParam("index" /* INDEX */, "idx");
3679
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
3680
+ } else {
3681
+ ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
3682
+ }
3683
+ if (store === "data") {
3684
+ if (ctx.file) {
3685
+ } else {
3686
+ ret.defParam("suffix" /* SUFFIX */, ".car");
3687
+ }
3688
+ }
3689
+ return ret.URI();
3367
3690
  }
3368
- function fireproof(name, opts) {
3369
- const key = JSON.stringify(
3370
- toSortedArray({
3371
- name,
3372
- stores: toSortedArray(opts?.store?.stores)
3373
- })
3374
- );
3375
- let db = Database.databases.get(key);
3376
- if (!db) {
3377
- db = new Database(name, opts);
3378
- Database.databases.set(key, db);
3691
+ function toStoreURIRuntime(sthis, name, sopts) {
3692
+ sopts = sopts || {};
3693
+ if (!sopts.base) {
3694
+ const fp_env = sthis.env.get("FP_STORAGE_URL");
3695
+ if (fp_env) {
3696
+ sopts = { ...sopts, base: BuildURI2.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
3697
+ } else {
3698
+ sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
3699
+ }
3379
3700
  }
3380
- return db;
3701
+ const bbase = BuildURI2.from(sopts.base);
3702
+ if (name) {
3703
+ bbase.setParam("name" /* NAME */, name);
3704
+ }
3705
+ const base = bbase.URI();
3706
+ return {
3707
+ idx: {
3708
+ data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
3709
+ file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
3710
+ meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
3711
+ wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
3712
+ },
3713
+ data: {
3714
+ data: defaultURI(sthis, sopts.data?.data, base, "data"),
3715
+ file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
3716
+ meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
3717
+ wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
3718
+ }
3719
+ };
3720
+ }
3721
+ function fireproof(name, opts) {
3722
+ return DatabaseFactory(name, opts);
3381
3723
  }
3382
3724
  function makeName(fnString) {
3383
3725
  const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
@@ -3408,7 +3750,7 @@ __export(runtime_exports, {
3408
3750
  kb: () => key_bag_exports,
3409
3751
  kc: () => keyed_crypto_exports,
3410
3752
  mf: () => wait_pr_multiformats_exports,
3411
- runtimeFn: () => runtimeFn2,
3753
+ runtimeFn: () => runtimeFn3,
3412
3754
  toArrayBuffer: () => toArrayBuffer
3413
3755
  });
3414
3756
 
@@ -3423,23 +3765,24 @@ __export(wait_pr_multiformats_exports, {
3423
3765
  var codec_interface_exports = {};
3424
3766
 
3425
3767
  // src/runtime/index.ts
3426
- import { runtimeFn as runtimeFn2 } from "@adviser/cement";
3768
+ import { runtimeFn as runtimeFn3 } from "@adviser/cement";
3427
3769
 
3428
3770
  // src/version.ts
3429
3771
  var PACKAGE_VERSION = Object.keys({
3430
- "0.19.100": "xxxx"
3772
+ "0.19.101": "xxxx"
3431
3773
  })[0];
3432
3774
  export {
3433
3775
  CRDT,
3434
- Database,
3776
+ DatabaseFactory,
3777
+ DatabaseShell,
3435
3778
  Index,
3436
3779
  NotFoundError,
3437
3780
  PACKAGE_VERSION,
3781
+ PARAM,
3438
3782
  Result,
3439
3783
  UInt8ArrayEqual,
3440
3784
  blockstore_exports as blockstore,
3441
3785
  blockstore_exports as bs,
3442
- dataDir,
3443
3786
  ensureLogger,
3444
3787
  ensureSuperLog,
3445
3788
  ensureSuperThis,
@@ -3450,10 +3793,13 @@ export {
3450
3793
  getName,
3451
3794
  getStore,
3452
3795
  index,
3796
+ isDatabase,
3453
3797
  isFalsy,
3454
3798
  isNotFoundError,
3799
+ keyConfigOpts,
3455
3800
  runtime_exports as rt,
3456
3801
  runtime_exports as runtime,
3457
- throwFalsy
3802
+ throwFalsy,
3803
+ toStoreURIRuntime
3458
3804
  };
3459
3805
  //# sourceMappingURL=index.js.map