@fireproof/core 0.19.101 → 0.19.103

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/{chunk-3EB3ENHT.js → chunk-OFGPKRCM.js} +25 -54
  2. package/chunk-OFGPKRCM.js.map +1 -0
  3. package/chunk-WS3YRPIA.js +75 -0
  4. package/chunk-WS3YRPIA.js.map +1 -0
  5. package/{gateway-GK5QZ6KP.js → gateway-5FCWPX5W.js} +12 -13
  6. package/gateway-5FCWPX5W.js.map +1 -0
  7. package/{gateway-TQTGDRCN.js → gateway-H7UD6TNB.js} +8 -9
  8. package/gateway-H7UD6TNB.js.map +1 -0
  9. package/index.cjs +1571 -1992
  10. package/index.cjs.map +1 -1
  11. package/index.d.cts +117 -257
  12. package/index.d.ts +117 -257
  13. package/index.global.js +12280 -12741
  14. package/index.global.js.map +1 -1
  15. package/index.js +1463 -1790
  16. package/index.js.map +1 -1
  17. package/{key-bag-file-VOSSK46F.js → key-bag-file-WADZBHYG.js} +3 -4
  18. package/{key-bag-file-VOSSK46F.js.map → key-bag-file-WADZBHYG.js.map} +1 -1
  19. package/{key-bag-indexdb-AXTQOSMC.js → key-bag-indexdb-PGVAI3FJ.js} +3 -4
  20. package/{key-bag-indexdb-AXTQOSMC.js.map → key-bag-indexdb-PGVAI3FJ.js.map} +1 -1
  21. package/mem-filesystem-YPPJV7Q2.js +41 -0
  22. package/mem-filesystem-YPPJV7Q2.js.map +1 -0
  23. package/metafile-cjs.json +1 -1
  24. package/metafile-esm.json +1 -1
  25. package/metafile-iife.json +1 -1
  26. package/{node-filesystem-CFRXFSO7.js → node-filesystem-INX4ZTHE.js} +9 -6
  27. package/node-filesystem-INX4ZTHE.js.map +1 -0
  28. package/package.json +1 -1
  29. package/tests/blockstore/keyed-crypto.test.ts +227 -63
  30. package/tests/blockstore/loader.test.ts +11 -19
  31. package/tests/blockstore/store.test.ts +19 -23
  32. package/tests/blockstore/transaction.test.ts +12 -12
  33. package/tests/fireproof/all-gateway.test.ts +193 -201
  34. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.ts +316 -324
  35. package/tests/fireproof/config.test.ts +172 -0
  36. package/tests/fireproof/crdt.test.ts +16 -67
  37. package/tests/fireproof/database.test.ts +21 -183
  38. package/tests/fireproof/fireproof.test.ts +74 -83
  39. package/tests/fireproof/hello.test.ts +14 -18
  40. package/tests/fireproof/indexer.test.ts +43 -53
  41. package/tests/fireproof/utils.test.ts +6 -18
  42. package/tests/helpers.ts +9 -27
  43. package/tests/react/useFireproof.test.tsx +1 -1
  44. package/{utils-STA2C35G.js → utils-QO2HIWGI.js} +3 -4
  45. package/chunk-3EB3ENHT.js.map +0 -1
  46. package/chunk-HQ7D3PEU.js +0 -61
  47. package/chunk-HQ7D3PEU.js.map +0 -1
  48. package/chunk-PZ5AY32C.js +0 -10
  49. package/deno-filesystem-Q2IJ7YDR.js +0 -57
  50. package/deno-filesystem-Q2IJ7YDR.js.map +0 -1
  51. package/gateway-GK5QZ6KP.js.map +0 -1
  52. package/gateway-TQTGDRCN.js.map +0 -1
  53. package/key-bag-memory-LWE6ARPX.js +0 -29
  54. package/key-bag-memory-LWE6ARPX.js.map +0 -1
  55. package/node-filesystem-CFRXFSO7.js.map +0 -1
  56. package/tests/blockstore/keyed-crypto-indexdb-file.test.ts +0 -129
  57. package/tests/gateway/file/loader-config.test.ts +0 -303
  58. package/tests/gateway/indexdb/loader-config.test.ts +0 -75
  59. package/utils-STA2C35G.js.map +0 -1
  60. /package/tests/fireproof/{fireproof.fixture.ts → fireproof.test.fixture.ts} +0 -0
  61. /package/{chunk-PZ5AY32C.js.map → utils-QO2HIWGI.js.map} +0 -0
package/index.js CHANGED
@@ -1,6 +1,3 @@
1
- import {
2
- INDEXDB_VERSION
3
- } from "./chunk-PB4BKL4O.js";
4
1
  import {
5
2
  FILESTORE_VERSION
6
3
  } from "./chunk-7EWIAXTM.js";
@@ -9,30 +6,28 @@ import {
9
6
  getFileSystem,
10
7
  getPath,
11
8
  toArrayBuffer
12
- } from "./chunk-HQ7D3PEU.js";
9
+ } from "./chunk-WS3YRPIA.js";
10
+ import {
11
+ INDEXDB_VERSION
12
+ } from "./chunk-PB4BKL4O.js";
13
13
  import {
14
14
  NotFoundError,
15
- PARAM,
16
15
  Result,
17
16
  UInt8ArrayEqual,
17
+ __export,
18
+ dataDir,
18
19
  ensureLogger,
19
20
  ensureSuperLog,
20
21
  ensureSuperThis,
21
22
  exceptionWrapper,
22
- falsyToUndef,
23
23
  getKey,
24
24
  getName,
25
25
  getStore,
26
- isFalsy,
27
- isNotFoundError,
28
- throwFalsy
29
- } from "./chunk-3EB3ENHT.js";
30
- import {
31
- __export
32
- } from "./chunk-PZ5AY32C.js";
26
+ isNotFoundError
27
+ } from "./chunk-OFGPKRCM.js";
33
28
 
34
29
  // src/database.ts
35
- import { BuildURI as BuildURI2, KeyedResolvOnce as KeyedResolvOnce3, ResolveOnce as ResolveOnce6, URI as URI8 } from "@adviser/cement";
30
+ import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
36
31
 
37
32
  // src/write-queue.ts
38
33
  function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -77,6 +72,85 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
77
72
  // src/crdt.ts
78
73
  import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
79
74
 
75
+ // src/runtime/wait-pr-multiformats/block.ts
76
+ var block_exports = {};
77
+ __export(block_exports, {
78
+ Block: () => Block,
79
+ create: () => create,
80
+ createUnsafe: () => createUnsafe,
81
+ decode: () => decode,
82
+ encode: () => encode
83
+ });
84
+ import { bytes as binary, CID } from "multiformats";
85
+ import { Block as mfBlock } from "multiformats/block";
86
+ var Block = mfBlock;
87
+ async function decode({
88
+ bytes,
89
+ codec: codec3,
90
+ hasher: hasher7
91
+ }) {
92
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
93
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
94
+ const value = await Promise.resolve(codec3.decode(bytes));
95
+ const hash = await hasher7.digest(bytes);
96
+ const cid = CID.create(1, codec3.code, hash);
97
+ return new mfBlock({ value, bytes, cid });
98
+ }
99
+ async function encode({
100
+ value,
101
+ codec: codec3,
102
+ hasher: hasher7
103
+ }) {
104
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
105
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
106
+ const bytes = await Promise.resolve(codec3.encode(value));
107
+ const hash = await hasher7.digest(bytes);
108
+ const cid = CID.create(1, codec3.code, hash);
109
+ return new mfBlock({ value, bytes, cid });
110
+ }
111
+ async function create({
112
+ bytes,
113
+ cid,
114
+ hasher: hasher7,
115
+ codec: codec3
116
+ }) {
117
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
118
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
119
+ const value = await Promise.resolve(codec3.decode(bytes));
120
+ const hash = await hasher7.digest(bytes);
121
+ if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
122
+ throw new Error("CID hash does not match bytes");
123
+ }
124
+ return createUnsafe({
125
+ bytes,
126
+ cid,
127
+ value,
128
+ codec: codec3
129
+ });
130
+ }
131
+ async function createUnsafe({
132
+ bytes,
133
+ cid,
134
+ value: maybeValue,
135
+ codec: codec3
136
+ }) {
137
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
138
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
139
+ return new Block({
140
+ cid,
141
+ bytes,
142
+ value
143
+ });
144
+ }
145
+
146
+ // src/crdt-helpers.ts
147
+ import { parse as parse3 } from "multiformats/link";
148
+ import { sha256 as hasher5 } from "multiformats/hashes/sha2";
149
+ import * as codec from "@ipld/dag-cbor";
150
+ import { put, get, entries, root } from "@web3-storage/pail/crdt";
151
+ import { EventFetcher, vis } from "@web3-storage/pail/clock";
152
+ import * as Batch from "@web3-storage/pail/crdt/batch";
153
+
80
154
  // src/blockstore/index.ts
81
155
  var blockstore_exports = {};
82
156
  __export(blockstore_exports, {
@@ -88,11 +162,8 @@ __export(blockstore_exports, {
88
162
  FragmentGateway: () => FragmentGateway,
89
163
  Loader: () => Loader,
90
164
  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
91
- ensureStoreEnDeFile: () => ensureStoreEnDeFile,
92
- fileGatewayFactoryItem: () => fileGatewayFactoryItem,
93
- getDefaultURI: () => getDefaultURI,
94
- getGatewayFactoryItem: () => getGatewayFactoryItem,
95
- getStartedGateway: () => getStartedGateway,
165
+ ensureStart: () => ensureStart,
166
+ getGatewayFromURL: () => getGatewayFromURL,
96
167
  parseCarFile: () => parseCarFile,
97
168
  registerStoreProtocol: () => registerStoreProtocol,
98
169
  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
@@ -107,7 +178,7 @@ function toCIDBlock(block) {
107
178
  }
108
179
 
109
180
  // src/blockstore/store-factory.ts
110
- import { KeyedResolvOnce as KeyedResolvOnce2, Result as Result7 } from "@adviser/cement";
181
+ import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
111
182
 
112
183
  // src/runtime/files.ts
113
184
  var files_exports = {};
@@ -182,402 +253,312 @@ var UnixFSFileBuilder = class {
182
253
 
183
254
  // src/blockstore/store.ts
184
255
  import { format as format2, parse as parse2 } from "@ipld/dag-json";
185
- import { ResolveOnce as ResolveOnce3, Result as Result5, exception2Result } from "@adviser/cement";
256
+ import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result5 } from "@adviser/cement";
186
257
 
187
- // src/blockstore/commit-queue.ts
188
- import { Future } from "@adviser/cement";
189
- var CommitQueue = class {
190
- constructor() {
191
- this.queue = [];
192
- this.processing = false;
193
- this._waitIdleItems = /* @__PURE__ */ new Set();
258
+ // src/types.ts
259
+ function isFalsy(value) {
260
+ return value === false && value === null && value === void 0;
261
+ }
262
+ function throwFalsy(value) {
263
+ if (isFalsy(value)) {
264
+ throw new Error("value is Falsy");
194
265
  }
195
- waitIdle() {
196
- if (this.queue.length === 0 && !this.processing) {
197
- return Promise.resolve();
198
- }
199
- const fn = new Future();
200
- this._waitIdleItems.add(fn);
201
- return fn.asPromise();
266
+ return value;
267
+ }
268
+ function falsyToUndef(value) {
269
+ if (isFalsy(value)) {
270
+ return void 0;
202
271
  }
203
- async enqueue(fn) {
204
- return new Promise((resolve, reject) => {
205
- const queueFn = async () => {
206
- try {
207
- resolve(await fn());
208
- } catch (e) {
209
- reject(e);
210
- } finally {
211
- this.processing = false;
212
- this.processNext();
213
- }
214
- };
215
- this.queue.push(queueFn);
216
- if (!this.processing) {
217
- this.processNext();
218
- }
219
- });
272
+ return value;
273
+ }
274
+
275
+ // src/blockstore/loader.ts
276
+ import pLimit from "p-limit";
277
+ import { CarReader } from "@ipld/car";
278
+ import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
279
+
280
+ // src/blockstore/loader-helpers.ts
281
+ import { sha256 as hasher } from "multiformats/hashes/sha2";
282
+ import * as dagCodec from "@ipld/dag-cbor";
283
+ async function parseCarFile(reader, logger) {
284
+ const roots = await reader.getRoots();
285
+ const header = await reader.get(roots[0]);
286
+ if (!header) throw logger.Error().Msg("missing header block").AsError();
287
+ const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
288
+ const fpvalue = dec.value;
289
+ if (fpvalue && !fpvalue.fp) {
290
+ throw logger.Error().Msg("missing fp").AsError();
220
291
  }
221
- processNext() {
222
- if (this.queue.length > 0 && !this.processing) {
223
- this.processing = true;
224
- const queueFn = this.queue.shift();
225
- if (queueFn) {
226
- queueFn().finally(() => {
227
- });
228
- }
229
- }
230
- if (this.queue.length === 0 && !this.processing) {
231
- const toResolve = Array.from(this._waitIdleItems);
232
- this._waitIdleItems.clear();
233
- toResolve.map((fn) => fn.resolve());
292
+ return fpvalue.fp;
293
+ }
294
+
295
+ // src/blockstore/transaction.ts
296
+ import { MemoryBlockstore } from "@web3-storage/pail/block";
297
+ import { toCryptoRuntime } from "@adviser/cement";
298
+ var CarTransaction = class extends MemoryBlockstore {
299
+ constructor(parent, opts = { add: true, noLoader: false }) {
300
+ super();
301
+ if (opts.add) {
302
+ parent.transactions.add(this);
234
303
  }
304
+ this.parent = parent;
305
+ }
306
+ async get(cid) {
307
+ return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
308
+ }
309
+ async superGet(cid) {
310
+ return super.get(cid);
235
311
  }
236
312
  };
237
-
238
- // src/runtime/keyed-crypto.ts
239
- var keyed_crypto_exports = {};
240
- __export(keyed_crypto_exports, {
241
- BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
242
- keyedCryptoFactory: () => keyedCryptoFactory
243
- });
244
- import { base58btc } from "multiformats/bases/base58";
245
- import { sha256 as hasher } from "multiformats/hashes/sha2";
246
- import * as CBOR from "cborg";
247
- var generateIV = {
248
- random: {
313
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
314
+ const logger = ensureLogger(sthis, component, ctx);
315
+ const store = opts.store || {};
316
+ return {
249
317
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
250
- calc: async (ko, crypto, data) => {
251
- return crypto.randomBytes(ko.ivLength);
318
+ applyMeta: (meta, snap) => {
319
+ return Promise.resolve();
252
320
  },
253
321
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
254
- verify: async (ko, crypto, iv, data) => {
255
- return true;
256
- }
257
- },
258
- hash: {
259
- calc: async (ko, crypto, data) => {
260
- const hash = await hasher.digest(data);
261
- const hashBytes = new Uint8Array(hash.bytes);
262
- const hashArray = new Uint8Array(ko.ivLength);
263
- for (let i = 0; i < hashBytes.length; i++) {
264
- hashArray[i % ko.ivLength] ^= hashBytes[i];
265
- }
266
- return hashArray;
322
+ compact: async (blocks) => {
323
+ return {};
267
324
  },
268
- verify: async function(ko, crypto, iv, data) {
269
- return ko.url.getParam("ivVerify" /* IV_VERIFY */) !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
270
- }
325
+ autoCompact: 100,
326
+ public: false,
327
+ name: void 0,
328
+ threshold: 1e3 * 1e3,
329
+ ...opts,
330
+ logger,
331
+ keyBag: opts.keyBag || {},
332
+ crypto: toCryptoRuntime(opts.crypto),
333
+ store,
334
+ storeRuntime: toStoreRuntime(store, sthis)
335
+ };
336
+ }
337
+ function blockstoreFactory(sthis, opts) {
338
+ if (opts.name) {
339
+ return new EncryptedBlockstore(sthis, opts);
340
+ } else {
341
+ return new BaseBlockstore(opts);
271
342
  }
272
- };
273
- function getGenerateIVFn(url, opts) {
274
- const ivhash = opts.ivCalc || url.getParam("ivHash" /* IV_HASH */) || "hash";
275
- return generateIV[ivhash] || generateIV["hash"];
276
343
  }
277
- var BlockIvKeyIdCodec = class {
278
- constructor(ko, iv, opts) {
279
- this.code = 3147065;
280
- this.name = "Fireproof@encrypted-block:aes-gcm";
281
- this.ko = ko;
282
- this.iv = iv;
283
- this.opts = opts || {};
344
+ var BaseBlockstore = class {
345
+ constructor(ebOpts = {}) {
346
+ this.transactions = /* @__PURE__ */ new Set();
347
+ this.sthis = ensureSuperThis(ebOpts);
348
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
349
+ this.logger = this.ebOpts.logger;
284
350
  }
285
- async encode(data) {
286
- const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
287
- const { iv } = this.ko.algo(calcIv);
288
- const fprt = await this.ko.fingerPrint();
289
- const keyId = base58btc.decode(fprt);
290
- this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
291
- return CBOR.encode({
292
- iv,
293
- keyId,
294
- data: await this.ko._encrypt({ iv, bytes: data })
295
- });
351
+ // ready: Promise<void>;
352
+ ready() {
353
+ return Promise.resolve();
296
354
  }
297
- async decode(abytes) {
298
- let bytes;
299
- if (abytes instanceof Uint8Array) {
300
- bytes = abytes;
301
- } else {
302
- bytes = new Uint8Array(abytes);
303
- }
304
- const { iv, keyId, data } = CBOR.decode(bytes);
305
- const fprt = await this.ko.fingerPrint();
306
- this.ko.logger.Debug().Str("fp", base58btc.encode(keyId)).Msg("decode");
307
- if (base58btc.encode(keyId) !== fprt) {
308
- throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
309
- }
310
- const result = await this.ko._decrypt({ iv, bytes: data });
311
- if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
312
- throw this.ko.logger.Error().Msg("iv missmatch").AsError();
313
- }
314
- return result;
355
+ async close() {
315
356
  }
316
- };
317
- var keyedCrypto = class {
318
- constructor(url, key, cyopt, sthis) {
319
- this.ivLength = 12;
320
- this.isEncrypting = true;
321
- this.logger = ensureLogger(sthis, "keyedCrypto");
322
- this.crypto = cyopt;
323
- this.key = key;
324
- this.url = url;
325
- }
326
- fingerPrint() {
327
- return Promise.resolve(this.key.fingerPrint);
357
+ async destroy() {
328
358
  }
329
- codec(iv, opts) {
330
- return new BlockIvKeyIdCodec(this, iv, opts);
359
+ async compact() {
331
360
  }
332
- algo(iv) {
333
- return {
334
- name: "AES-GCM",
335
- iv: iv || this.crypto.randomBytes(this.ivLength),
336
- tagLength: 128
337
- };
361
+ async get(cid) {
362
+ if (!cid) throw this.logger.Error().Msg("required cid").AsError();
363
+ for (const f of this.transactions) {
364
+ const v = await f.superGet(cid);
365
+ if (v) return v;
366
+ }
338
367
  }
339
- async _decrypt(data) {
340
- this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
341
- return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
368
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
369
+ async put(cid, block) {
370
+ throw this.logger.Error().Msg("use a transaction to put").AsError();
342
371
  }
343
- async _encrypt(data) {
344
- this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
345
- const a = this.algo(data.iv);
346
- return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
372
+ // TransactionMeta
373
+ async transaction(fn, _opts) {
374
+ const t = new CarTransaction(this, _opts);
375
+ const done = await fn(t);
376
+ this.lastTxMeta = done;
377
+ return { t, meta: done };
347
378
  }
348
- };
349
- var nullCodec = class {
350
- constructor() {
351
- this.code = 0;
352
- this.name = "Fireproof@unencrypted-block";
379
+ openTransaction(opts = { add: true, noLoader: false }) {
380
+ return new CarTransaction(this, opts);
353
381
  }
354
- encode(data) {
355
- return data;
382
+ async commitTransaction(t, done, opts) {
383
+ if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
384
+ const cars = await this.loader?.commit(t, done, opts);
385
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
386
+ setTimeout(() => void this.compact(), 10);
387
+ }
388
+ if (cars) {
389
+ this.transactions.delete(t);
390
+ return { meta: done, cars, t };
391
+ }
392
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
356
393
  }
357
- decode(data) {
358
- return data;
394
+ async *entries() {
395
+ const seen = /* @__PURE__ */ new Set();
396
+ for (const t of this.transactions) {
397
+ for await (const blk of t.entries()) {
398
+ if (seen.has(blk.cid.toString())) continue;
399
+ seen.add(blk.cid.toString());
400
+ yield blk;
401
+ }
402
+ }
359
403
  }
360
404
  };
361
- var noCrypto = class {
362
- constructor(url, cyrt, sthis) {
363
- this.ivLength = 0;
364
- this.code = 0;
365
- this.name = "Fireproof@unencrypted-block";
366
- this.isEncrypting = false;
367
- this._fingerPrint = "noCrypto:" + Math.random();
368
- this.logger = ensureLogger(sthis, "noCrypto");
369
- this.crypto = cyrt;
370
- this.url = url;
371
- }
372
- fingerPrint() {
373
- return Promise.resolve(this._fingerPrint);
374
- }
375
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
376
- codec(iv) {
377
- return new nullCodec();
405
+ var EncryptedBlockstore = class extends BaseBlockstore {
406
+ constructor(sthis, ebOpts) {
407
+ super(ebOpts);
408
+ this.compacting = false;
409
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
410
+ const { name } = ebOpts;
411
+ if (!name) {
412
+ throw this.logger.Error().Msg("name required").AsError();
413
+ }
414
+ this.name = name;
415
+ this.loader = new Loader(this.name, ebOpts, sthis);
378
416
  }
379
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
380
- algo(iv) {
381
- return {
382
- name: "noCrypto",
383
- iv: new Uint8Array(),
384
- tagLength: 0
385
- };
417
+ ready() {
418
+ return this.loader.ready();
386
419
  }
387
- _decrypt() {
388
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
420
+ close() {
421
+ return this.loader.close();
389
422
  }
390
- _encrypt() {
391
- throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
423
+ destroy() {
424
+ return this.loader.destroy();
392
425
  }
393
- };
394
- async function keyedCryptoFactory(url, kb, sthis) {
395
- const storekey = url.getParam("storekey" /* STORE_KEY */);
396
- if (storekey && storekey !== "insecure") {
397
- let rkey = await kb.getNamedKey(storekey, true);
398
- if (rkey.isErr()) {
399
- try {
400
- rkey = await kb.toKeyWithFingerPrint(storekey);
401
- } catch (e) {
402
- throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
403
- }
426
+ async get(cid) {
427
+ const got = await super.get(cid);
428
+ if (got) return got;
429
+ if (!this.loader) {
430
+ return;
404
431
  }
405
- return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
406
- }
407
- return new noCrypto(url, kb.rt.crypto, sthis);
408
- }
409
-
410
- // src/blockstore/fragment-gateway.ts
411
- import { Result as Result2 } from "@adviser/cement";
412
- import { base58btc as base58btc2 } from "multiformats/bases/base58";
413
- import { encode as encode2, decode as decode2 } from "cborg";
414
- function getFragSize(url) {
415
- const fragSize = url.getParam("fragSize" /* FRAG_SIZE */);
416
- let ret = 0;
417
- if (fragSize) {
418
- ret = parseInt(fragSize);
419
- }
420
- if (isNaN(ret) || ret <= 0) {
421
- ret = 0;
432
+ return falsyToUndef(await this.loader.getBlock(cid));
422
433
  }
423
- return ret;
424
- }
425
- async function getFrags(url, innerGW, headerSize, logger) {
426
- const fragSize = getFragSize(url);
427
- if (!fragSize) {
428
- const res = await innerGW.get(url);
429
- if (res.isErr()) {
430
- return [res];
434
+ async transaction(fn, opts = { noLoader: false }) {
435
+ const { t, meta: done } = await super.transaction(fn);
436
+ const cars = await this.loader.commit(t, done, opts);
437
+ if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
438
+ setTimeout(() => void this.compact(), 10);
431
439
  }
432
- const data = res.unwrap();
433
- return [
434
- Result2.Ok({
435
- fid: new Uint8Array(0),
436
- ofs: 0,
437
- len: data.length,
438
- data
439
- })
440
- ];
441
- }
442
- const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
443
- if (firstRaw.isErr()) {
444
- return [firstRaw];
440
+ if (cars) {
441
+ this.transactions.delete(t);
442
+ return { meta: done, cars, t };
443
+ }
444
+ throw this.logger.Error().Msg("failed to commit car files").AsError();
445
445
  }
446
- const firstFragment = decode2(firstRaw.unwrap());
447
- const blockSize = firstFragment.data.length;
448
- const ops = [Promise.resolve(Result2.Ok(firstFragment))];
449
- const fidStr = base58btc2.encode(firstFragment.fid);
450
- const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, firstFragment.len.toString()).setParam("headerSize" /* FRAG_HEAD */, headerSize.toString());
451
- for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
452
- ops.push(
453
- (async (furl, ofs2) => {
454
- const raw2 = await innerGW.get(furl);
455
- if (raw2.isErr()) {
456
- return raw2;
457
- }
458
- const fragment = decode2(raw2.unwrap());
459
- if (base58btc2.encode(fragment.fid) !== fidStr) {
460
- return Result2.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
461
- }
462
- if (fragment.ofs !== ofs2) {
463
- return Result2.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
464
- }
465
- return Result2.Ok(fragment);
466
- })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
446
+ async getFile(car, cid) {
447
+ await this.ready();
448
+ if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
449
+ const reader = await this.loader.loadFileCar(
450
+ car
451
+ /*, isPublic */
467
452
  );
453
+ const block = await reader.get(cid);
454
+ if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
455
+ return block.bytes;
468
456
  }
469
- return Promise.all(ops);
470
- }
471
- var FragmentGateway = class {
472
- constructor(sthis, innerGW) {
473
- this.fidLength = 4;
474
- this.headerSize = 32;
475
- this.sthis = ensureSuperLog(sthis, "FragmentGateway");
476
- this.logger = this.sthis.logger;
477
- this.innerGW = innerGW;
457
+ async compact() {
458
+ await this.ready();
459
+ if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
460
+ if (this.loader.carLog.length < 2) return;
461
+ const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
462
+ if (!compactFn || this.compacting) return;
463
+ const blockLog = new CompactionFetcher(this);
464
+ this.compacting = true;
465
+ const meta = await compactFn(blockLog);
466
+ await this.loader?.commit(blockLog.loggedBlocks, meta, {
467
+ compact: true,
468
+ noLoader: true
469
+ });
470
+ this.compacting = false;
478
471
  }
479
- slicer(url, body) {
480
- const fragSize = getFragSize(url);
481
- if (!fragSize) {
482
- return [this.innerGW.put(url, body)];
472
+ async defaultCompact(blocks, logger) {
473
+ if (!this.loader) {
474
+ throw logger.Error().Msg("no loader").AsError();
483
475
  }
484
- const blocksize = fragSize - this.headerSize;
485
- if (blocksize <= 0) {
486
- throw this.logger.Error().Uint64("fragSize" /* FRAG_SIZE */, fragSize).Uint64("headerSize" /* FRAG_HEAD */, this.headerSize).Msg("Fragment size is too small").AsError();
476
+ if (!this.lastTxMeta) {
477
+ throw logger.Error().Msg("no lastTxMeta").AsError();
487
478
  }
488
- const ops = [];
489
- const fid = this.sthis.nextId(this.fidLength);
490
- const fragUrl = url.build().setParam("fid" /* FRAG_FID */, fid.str).setParam("len" /* FRAG_LEN */, body.length.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString());
491
- for (let ofs = 0; ofs < body.length; ofs += blocksize) {
492
- const block = encode2({
493
- fid: fid.bin,
494
- ofs,
495
- len: body.length,
496
- data: body.slice(ofs, ofs + blocksize)
497
- });
498
- if (block.length > fragSize) {
499
- throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
479
+ for await (const blk of this.loader.entries(false)) {
480
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
481
+ }
482
+ for (const t of this.transactions) {
483
+ for await (const blk of t.entries()) {
484
+ blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
500
485
  }
501
- ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
502
486
  }
503
- return ops;
504
- }
505
- buildUrl(baseUrl, key) {
506
- return this.innerGW.buildUrl(baseUrl, key);
487
+ return this.lastTxMeta;
507
488
  }
508
- async destroy(iurl) {
509
- return this.innerGW.destroy(iurl);
489
+ async *entries() {
490
+ for await (const blk of this.loader.entries()) {
491
+ yield blk;
492
+ }
510
493
  }
511
- async start(url) {
512
- this.headerSize = encode2({
513
- fid: this.sthis.nextId(this.fidLength).bin,
514
- ofs: 1024 * 1024,
515
- // 32bit
516
- len: 16 * 1024 * 1024,
517
- // 32bit
518
- data: new Uint8Array(1024)
519
- }).length - 1024;
520
- return this.innerGW.start(url);
521
- }
522
- async close(url) {
523
- return this.innerGW.close(url);
494
+ };
495
+ var CompactionFetcher = class {
496
+ constructor(blocks) {
497
+ this.blockstore = blocks;
498
+ this.loggedBlocks = new CarTransaction(blocks);
524
499
  }
525
- async put(url, body) {
526
- await Promise.all(this.slicer(url, body));
527
- return Result2.Ok(void 0);
500
+ async get(cid) {
501
+ const block = await this.blockstore.get(cid);
502
+ if (block) this.loggedBlocks.putSync(cid, block.bytes);
503
+ return falsyToUndef(block);
528
504
  }
529
- async get(url) {
530
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
531
- let buffer = void 0;
532
- for (const rfrag of rfrags) {
533
- if (rfrag.isErr()) {
534
- return Result2.Err(rfrag.Err());
535
- }
536
- const frag = rfrag.Ok();
537
- buffer = buffer || new Uint8Array(frag.len);
538
- buffer.set(frag.data, frag.ofs);
539
- }
540
- return Result2.Ok(buffer || new Uint8Array(0));
505
+ };
506
+
507
+ // src/blockstore/commit-queue.ts
508
+ import { Future } from "@adviser/cement";
509
+ var CommitQueue = class {
510
+ constructor() {
511
+ this.queue = [];
512
+ this.processing = false;
513
+ this._waitIdleItems = /* @__PURE__ */ new Set();
541
514
  }
542
- async subscribe(url, callback) {
543
- if (this.innerGW.subscribe) {
544
- return this.innerGW.subscribe(url, callback);
545
- } else {
546
- return Result2.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
515
+ waitIdle() {
516
+ if (this.queue.length === 0 && !this.processing) {
517
+ return Promise.resolve();
547
518
  }
519
+ const fn = new Future();
520
+ this._waitIdleItems.add(fn);
521
+ return fn.asPromise();
548
522
  }
549
- async delete(url) {
550
- const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
551
- for (const rfrag of rfrags) {
552
- if (rfrag.isErr()) {
553
- return Result2.Err(rfrag.Err());
523
+ async enqueue(fn) {
524
+ return new Promise((resolve, reject) => {
525
+ const queueFn = async () => {
526
+ try {
527
+ resolve(await fn());
528
+ } catch (e) {
529
+ reject(e);
530
+ } finally {
531
+ this.processing = false;
532
+ this.processNext();
533
+ }
534
+ };
535
+ this.queue.push(queueFn);
536
+ if (!this.processing) {
537
+ this.processNext();
554
538
  }
555
- const frag = rfrag.Ok();
556
- let fragUrl;
557
- if (rfrags.length > 1) {
558
- const fidStr = base58btc2.encode(frag.fid);
559
- fragUrl = url.build().setParam("fid" /* FRAG_FID */, fidStr).setParam("len" /* FRAG_LEN */, frag.len.toString()).setParam("headerSize" /* FRAG_HEAD */, this.headerSize.toString()).URI();
560
- } else {
561
- fragUrl = url;
539
+ });
540
+ }
541
+ processNext() {
542
+ if (this.queue.length > 0 && !this.processing) {
543
+ this.processing = true;
544
+ const queueFn = this.queue.shift();
545
+ if (queueFn) {
546
+ queueFn().finally(() => {
547
+ });
562
548
  }
563
- await this.innerGW.delete(fragUrl);
564
549
  }
565
- return Result2.Ok(void 0);
550
+ if (this.queue.length === 0 && !this.processing) {
551
+ const toResolve = Array.from(this._waitIdleItems);
552
+ this._waitIdleItems.clear();
553
+ toResolve.map((fn) => fn.resolve());
554
+ }
566
555
  }
567
556
  };
568
557
 
569
- // src/blockstore/meta-key-helper.ts
570
- import { format, parse } from "@ipld/dag-json";
571
- import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
572
- import { CID } from "multiformats";
573
- import { base64pad } from "multiformats/bases/base64";
574
- import { Result as Result4 } from "@adviser/cement";
575
-
576
558
  // src/runtime/key-bag.ts
577
559
  var key_bag_exports = {};
578
560
  __export(key_bag_exports, {
579
561
  KeyBag: () => KeyBag,
580
- defaultKeyBagOpts: () => defaultKeyBagOpts,
581
562
  getKeyBag: () => getKeyBag,
582
563
  registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
583
564
  });
@@ -585,20 +566,18 @@ import {
585
566
  KeyedResolvOnce,
586
567
  ResolveOnce,
587
568
  ResolveSeq,
588
- Result as Result3,
569
+ Result as Result2,
589
570
  runtimeFn,
590
- toCryptoRuntime,
591
- URI as URI2
571
+ toCryptoRuntime as toCryptoRuntime2,
572
+ URI
592
573
  } from "@adviser/cement";
593
- import { base58btc as base58btc3 } from "multiformats/bases/base58";
574
+ import { base58btc } from "multiformats/bases/base58";
594
575
  var KeyBag = class {
595
576
  constructor(rt) {
596
577
  this.rt = rt;
597
578
  this._warnOnce = new ResolveOnce();
598
579
  this._seq = new ResolveSeq();
599
- this.logger = ensureLogger(rt.sthis, "KeyBag", {
600
- id: rt.id()
601
- });
580
+ this.logger = ensureLogger(rt.sthis, "KeyBag");
602
581
  this.logger.Debug().Msg("KeyBag created");
603
582
  }
604
583
  async subtleKey(key) {
@@ -611,7 +590,7 @@ var KeyBag = class {
611
590
  return await this.rt.crypto.importKey(
612
591
  "raw",
613
592
  // raw or jwk
614
- base58btc3.decode(key),
593
+ base58btc.decode(key),
615
594
  // hexStringToUint8Array(key), // raw data
616
595
  "AES-GCM",
617
596
  extractable,
@@ -619,9 +598,9 @@ var KeyBag = class {
619
598
  );
620
599
  }
621
600
  async ensureKeyFromUrl(url, keyFactory) {
622
- const storeKey = url.getParam("storekey" /* STORE_KEY */);
601
+ const storeKey = url.getParam("storekey");
623
602
  if (storeKey === "insecure") {
624
- return Result3.Ok(url);
603
+ return Result2.Ok(url);
625
604
  }
626
605
  if (!storeKey) {
627
606
  const keyName = `@${keyFactory()}@`;
@@ -629,8 +608,8 @@ var KeyBag = class {
629
608
  if (ret.isErr()) {
630
609
  return ret;
631
610
  }
632
- const urb = url.build().setParam("storekey" /* STORE_KEY */, keyName);
633
- return Result3.Ok(urb.URI());
611
+ const urb = url.build().setParam("storekey", keyName);
612
+ return Result2.Ok(urb.URI());
634
613
  }
635
614
  if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
636
615
  const ret = await this.getNamedKey(storeKey);
@@ -638,15 +617,15 @@ var KeyBag = class {
638
617
  return ret;
639
618
  }
640
619
  }
641
- return Result3.Ok(url);
620
+ return Result2.Ok(url);
642
621
  }
643
622
  async toKeyWithFingerPrint(keyStr) {
644
- const material = base58btc3.decode(keyStr);
623
+ const material = base58btc.decode(keyStr);
645
624
  const key = await this.subtleKey(keyStr);
646
625
  const fpr = await this.rt.crypto.digestSHA256(material);
647
- return Result3.Ok({
626
+ return Result2.Ok({
648
627
  key,
649
- fingerPrint: base58btc3.encode(new Uint8Array(fpr))
628
+ fingerPrint: base58btc.encode(new Uint8Array(fpr))
650
629
  });
651
630
  }
652
631
  async setNamedKey(name, key) {
@@ -669,13 +648,13 @@ var KeyBag = class {
669
648
  return ret;
670
649
  }
671
650
  const named = ret.Ok();
672
- return Result3.Ok({
651
+ return Result2.Ok({
673
652
  ...named,
674
653
  extract: async () => {
675
654
  const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
676
655
  return {
677
656
  key: ext,
678
- keyStr: base58btc3.encode(ext)
657
+ keyStr: base58btc.encode(ext)
679
658
  };
680
659
  }
681
660
  });
@@ -692,9 +671,9 @@ var KeyBag = class {
692
671
  }
693
672
  if (failIfNotFound) {
694
673
  this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
695
- return Result3.Err(new Error(`Key not found: ${name}`));
674
+ return Result2.Err(new Error(`Key not found: ${name}`));
696
675
  }
697
- const ret = await this._setNamedKey(name, base58btc3.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
676
+ const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
698
677
  this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
699
678
  return ret;
700
679
  });
@@ -705,14 +684,14 @@ var keyBagProviderFactories = new Map(
705
684
  {
706
685
  protocol: "file:",
707
686
  factory: async (url, sthis) => {
708
- const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
687
+ const { KeyBagProviderFile } = await import("./key-bag-file-WADZBHYG.js");
709
688
  return new KeyBagProviderFile(url, sthis);
710
689
  }
711
690
  },
712
691
  {
713
692
  protocol: "indexdb:",
714
693
  factory: async (url, sthis) => {
715
- const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
694
+ const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-PGVAI3FJ.js");
716
695
  return new KeyBagProviderIndexDB(url, sthis);
717
696
  }
718
697
  }
@@ -726,63 +705,44 @@ function registerKeyBagProviderFactory(item) {
726
705
  });
727
706
  }
728
707
  function defaultKeyBagOpts(sthis, kbo) {
729
- kbo = kbo || {};
730
708
  if (kbo.keyRuntime) {
731
709
  return kbo.keyRuntime;
732
710
  }
733
711
  const logger = ensureLogger(sthis, "KeyBag");
734
712
  let url;
735
713
  if (kbo.url) {
736
- url = URI2.from(kbo.url);
714
+ url = URI.from(kbo.url);
737
715
  logger.Debug().Url(url).Msg("from opts");
738
716
  } else {
739
717
  let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
740
718
  if (runtimeFn().isBrowser) {
741
- url = URI2.from(bagFnameOrUrl || "indexdb://fp-keybag");
719
+ url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
742
720
  } else {
743
721
  if (!bagFnameOrUrl) {
744
722
  const home = sthis.env.get("HOME");
745
723
  bagFnameOrUrl = `${home}/.fireproof/keybag`;
746
- url = URI2.from(`file://${bagFnameOrUrl}`);
724
+ url = URI.from(`file://${bagFnameOrUrl}`);
747
725
  } else {
748
- url = URI2.from(bagFnameOrUrl);
726
+ url = URI.from(bagFnameOrUrl);
749
727
  }
750
728
  }
751
729
  logger.Debug().Url(url).Msg("from env");
752
730
  }
753
- let keyProviderFactory;
754
- switch (url.protocol) {
755
- case "file:":
756
- keyProviderFactory = async () => {
757
- const { KeyBagProviderFile } = await import("./key-bag-file-VOSSK46F.js");
758
- return new KeyBagProviderFile(url, sthis);
759
- };
760
- break;
761
- case "indexdb:":
762
- keyProviderFactory = async () => {
763
- const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-AXTQOSMC.js");
764
- return new KeyBagProviderIndexDB(url, sthis);
765
- };
766
- break;
767
- case "memory:":
768
- keyProviderFactory = async () => {
769
- const { KeyBagProviderMemory } = await import("./key-bag-memory-LWE6ARPX.js");
770
- return new KeyBagProviderMemory(url, sthis);
771
- };
772
- break;
773
- default:
774
- throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
731
+ const kitem = keyBagProviderFactories.get(url.protocol);
732
+ if (!kitem) {
733
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
775
734
  }
735
+ const getBag = async () => kitem.factory(url, sthis);
776
736
  if (url.hasParam("masterkey")) {
777
737
  throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
778
738
  }
779
739
  return {
780
740
  url,
781
- crypto: kbo.crypto || toCryptoRuntime({}),
741
+ crypto: kbo.crypto || toCryptoRuntime2({}),
782
742
  sthis,
783
743
  logger,
784
744
  keyLength: kbo.keyLength || 16,
785
- getBag: keyProviderFactory,
745
+ getBag,
786
746
  id: () => {
787
747
  return url.toString();
788
748
  }
@@ -795,940 +755,936 @@ async function getKeyBag(sthis, kbo = {}) {
795
755
  return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
796
756
  }
797
757
 
798
- // src/blockstore/meta-key-helper.ts
799
- async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
800
- const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
801
- if (!crdtEntries.length) {
802
- sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
803
- return [];
758
+ // src/blockstore/commitor.ts
759
+ import * as CBW from "@ipld/car/buffer-writer";
760
+ import { sha256 as hasher2 } from "multiformats/hashes/sha2";
761
+ import * as dagCodec2 from "@ipld/dag-cbor";
762
+ async function encodeCarFile(roots, t, codec3) {
763
+ let size = 0;
764
+ const headerSize = CBW.headerLength({ roots });
765
+ size += headerSize;
766
+ for (const { cid, bytes } of t.entries()) {
767
+ size += CBW.blockLength({ cid, bytes });
804
768
  }
805
- if (!crdtEntries.map) {
806
- sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
807
- return [];
769
+ const buffer = new Uint8Array(size);
770
+ const writer = CBW.createWriter(buffer, { headerSize });
771
+ for (const r of roots) {
772
+ writer.addRoot(r);
808
773
  }
809
- return Promise.all(
810
- crdtEntries.map(async (crdtEntry) => {
811
- const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
812
- const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
813
- return {
814
- eventCid: eventBlock.cid,
815
- parents: crdtEntry.parents,
816
- dbMeta
817
- };
818
- })
819
- );
774
+ for (const { cid, bytes } of t.entries()) {
775
+ writer.write({ cid, bytes });
776
+ }
777
+ writer.close();
778
+ return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
820
779
  }
821
- async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
822
- try {
823
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
824
- const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
825
- if (keyInfo.length) {
826
- const dbMeta = keyInfo[0].dbMeta;
827
- if (dbMeta.key) {
828
- const kb = await getKeyBag(sthis);
829
- const keyName = getStoreKeyName(uri);
830
- const res = await kb.setNamedKey(keyName, dbMeta.key);
831
- if (res.isErr()) {
832
- sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
833
- throw res.Err();
834
- }
835
- }
836
- sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
837
- return Result4.Ok(dbMeta);
838
- }
839
- sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
840
- return Result4.Ok(void 0);
841
- } catch (error) {
842
- sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
843
- return Result4.Err(error);
780
+ async function createCarFile(encoder, cid, t) {
781
+ return encodeCarFile([cid], t, encoder);
782
+ }
783
+ async function commitFiles(fileStore, walStore, t, done) {
784
+ const { files: roots } = makeFileCarHeader(done);
785
+ const cids = [];
786
+ const codec3 = (await fileStore.keyedCrypto()).codec();
787
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
788
+ for (const car of cars) {
789
+ const { cid, bytes } = car;
790
+ await fileStore.save({ cid, bytes });
791
+ await walStore.enqueueFile(
792
+ cid
793
+ /*, !!opts.public*/
794
+ );
795
+ cids.push(cid);
844
796
  }
797
+ return cids;
845
798
  }
846
- async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
847
- try {
848
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
849
- const keyName = getStoreKeyName(uri);
850
- const kb = await getKeyBag(sthis);
851
- const res = await kb.getNamedExtractableKey(keyName, true);
852
- if (res.isErr()) {
853
- sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
854
- throw res.Err();
799
+ function makeFileCarHeader(result) {
800
+ const files = [];
801
+ for (const [, meta] of Object.entries(result.files || {})) {
802
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
803
+ files.push(meta.cid);
855
804
  }
856
- const keyData = await res.Ok().extract();
857
- const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
858
- const { dbMeta, parents } = dbMetas[0];
859
- const parentLinks = parents.map((p) => CID.parse(p));
860
- dbMeta.key = keyData.keyStr;
861
- const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
862
- const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
863
- sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
864
- return Result4.Ok(encoded);
865
- } catch (error) {
866
- sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
867
- return Result4.Err(error);
868
805
  }
806
+ return { ...result, files };
869
807
  }
870
- function getStoreKeyName(url) {
871
- const storeKeyName = [url.getParam("localName") || url.getParam("name")];
872
- const idx = url.getParam("index");
873
- if (idx) {
874
- storeKeyName.push(idx);
875
- }
876
- storeKeyName.push("data");
877
- return `@${storeKeyName.join(":")}@`;
808
+ async function prepareCarFilesFiles(encoder, roots, t) {
809
+ return [await encodeCarFile(roots, t, encoder)];
878
810
  }
879
- async function createDbMetaEventBlock(sthis, dbMeta, parents) {
880
- const event = await EventBlock.create(
881
- {
882
- dbMeta: sthis.txt.encode(format(dbMeta))
883
- },
884
- parents
885
- );
886
- return event;
811
+ function makeCarHeader(meta, cars, compact = false) {
812
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
813
+ return { ...coreHeader, meta };
887
814
  }
888
- async function encodeEventsWithParents(sthis, events, parents) {
889
- const crdtEntries = events.map((event) => {
890
- const base64String = base64pad.encode(event.bytes);
891
- return {
892
- cid: event.cid.toString(),
893
- data: base64String,
894
- parents: parents.map((p) => p.toString())
895
- };
815
+ async function encodeCarHeader(fp) {
816
+ return await encode({
817
+ value: { fp },
818
+ hasher: hasher2,
819
+ codec: dagCodec2
896
820
  });
897
- return sthis.txt.encode(JSON.stringify(crdtEntries));
898
821
  }
899
-
900
- // src/blockstore/store.ts
901
- import pRetry from "p-retry";
902
- import pMap from "p-map";
903
-
904
- // src/blockstore/loader.ts
905
- import pLimit from "p-limit";
906
- import { CarReader } from "@ipld/car";
907
- import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
908
-
909
- // src/runtime/wait-pr-multiformats/block.ts
910
- var block_exports = {};
911
- __export(block_exports, {
912
- Block: () => Block,
913
- create: () => create,
914
- createUnsafe: () => createUnsafe,
915
- decode: () => decode3,
916
- encode: () => encode3
917
- });
918
- import { bytes as binary, CID as CID2 } from "multiformats";
919
- import { Block as mfBlock } from "multiformats/block";
920
- var Block = mfBlock;
921
- async function decode3({
922
- bytes,
923
- codec: codec3,
924
- hasher: hasher7
925
- }) {
926
- if (bytes == null) throw new Error('Missing required argument "bytes"');
927
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
928
- const value = await Promise.resolve(codec3.decode(bytes));
929
- const hash = await hasher7.digest(bytes);
930
- const cid = CID2.create(1, codec3.code, hash);
931
- return new mfBlock({ value, bytes, cid });
932
- }
933
- async function encode3({
934
- value,
935
- codec: codec3,
936
- hasher: hasher7
937
- }) {
938
- if (typeof value === "undefined") throw new Error('Missing required argument "value"');
939
- if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
940
- const bytes = await Promise.resolve(codec3.encode(value));
941
- const hash = await hasher7.digest(bytes);
942
- const cid = CID2.create(1, codec3.code, hash);
943
- return new mfBlock({ value, bytes, cid });
944
- }
945
- async function create({
946
- bytes,
947
- cid,
948
- hasher: hasher7,
949
- codec: codec3
950
- }) {
951
- if (bytes == null) throw new Error('Missing required argument "bytes"');
952
- if (hasher7 == null) throw new Error('Missing required argument "hasher"');
953
- const value = await Promise.resolve(codec3.decode(bytes));
954
- const hash = await hasher7.digest(bytes);
955
- if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
956
- throw new Error("CID hash does not match bytes");
822
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
823
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
824
+ const rootBlock = await encodeCarHeader(fp);
825
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
826
+ const cids = [];
827
+ for (const car of cars) {
828
+ const { cid, bytes } = car;
829
+ await params.carStore.save({ cid, bytes });
830
+ cids.push(cid);
957
831
  }
958
- return createUnsafe({
959
- bytes,
960
- cid,
961
- value,
962
- codec: codec3
963
- });
964
- }
965
- async function createUnsafe({
966
- bytes,
967
- cid,
968
- value: maybeValue,
969
- codec: codec3
970
- }) {
971
- const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
972
- if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
973
- return new Block({
974
- cid,
975
- bytes,
976
- value
977
- });
832
+ const newDbMeta = { cars: cids };
833
+ await params.WALStore.enqueue(newDbMeta, opts);
834
+ await params.metaStore.save(newDbMeta);
835
+ return { cgrp: cids, header: fp };
978
836
  }
979
-
980
- // src/blockstore/loader-helpers.ts
981
- import { sha256 as hasher2 } from "multiformats/hashes/sha2";
982
- import * as dagCodec from "@ipld/dag-cbor";
983
- async function parseCarFile(reader, logger) {
984
- const roots = await reader.getRoots();
985
- const header = await reader.get(roots[0]);
986
- if (!header) throw logger.Error().Msg("missing header block").AsError();
987
- const dec = await decode3({ bytes: header.bytes, hasher: hasher2, codec: dagCodec });
988
- const fpvalue = dec.value;
989
- if (fpvalue && !fpvalue.fp) {
990
- throw logger.Error().Msg("missing fp").AsError();
837
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
838
+ const carFiles = [];
839
+ threshold = threshold || 128e3 * 8;
840
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
841
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
842
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
843
+ let cidRootBlock = rootBlock;
844
+ for (const { cid, bytes } of t.entries()) {
845
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
846
+ if (newsize >= threshold) {
847
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
848
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
849
+ clonedt.putSync(cid, bytes);
850
+ cidRootBlock = { cid, bytes };
851
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
852
+ } else {
853
+ clonedt.putSync(cid, bytes);
854
+ }
991
855
  }
992
- return fpvalue.fp;
856
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
857
+ return carFiles;
993
858
  }
994
859
 
995
- // src/blockstore/transaction.ts
996
- import { MemoryBlockstore } from "@web3-storage/pail/block";
997
- import { toCryptoRuntime as toCryptoRuntime2 } from "@adviser/cement";
998
- var CarTransaction = class extends MemoryBlockstore {
999
- constructor(parent, opts = { add: true, noLoader: false }) {
1000
- super();
1001
- if (opts.add) {
1002
- parent.transactions.add(this);
1003
- }
1004
- this.parent = parent;
860
+ // src/blockstore/loader.ts
861
+ import { sha256 as hasher3 } from "multiformats/hashes/sha2";
862
+
863
+ // src/blockstore/task-manager.ts
864
+ var TaskManager = class {
865
+ constructor(sthis, callback) {
866
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
867
+ this.queue = [];
868
+ this.isProcessing = false;
869
+ this.logger = ensureLogger(sthis, "TaskManager");
870
+ this.callback = callback;
1005
871
  }
1006
- async get(cid) {
1007
- return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
872
+ async handleEvent(cid, parents, dbMeta) {
873
+ for (const parent of parents) {
874
+ this.eventsWeHandled.add(parent.toString());
875
+ }
876
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
877
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
878
+ void this.processQueue();
1008
879
  }
1009
- async superGet(cid) {
1010
- return super.get(cid);
880
+ async processQueue() {
881
+ if (this.isProcessing) return;
882
+ this.isProcessing = true;
883
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
884
+ const first = filteredQueue[0];
885
+ if (!first) {
886
+ return;
887
+ }
888
+ try {
889
+ await this.callback(first.dbMeta);
890
+ this.eventsWeHandled.add(first.cid);
891
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
892
+ } catch (err) {
893
+ if (first.retries++ > 3) {
894
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
895
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
896
+ }
897
+ await new Promise((resolve) => setTimeout(resolve, 50));
898
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
899
+ } finally {
900
+ this.isProcessing = false;
901
+ if (this.queue.length > 0) {
902
+ void this.processQueue();
903
+ }
904
+ }
1011
905
  }
1012
906
  };
1013
- function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
1014
- const logger = ensureLogger(sthis, component, ctx);
1015
- return {
1016
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1017
- applyMeta: (meta, snap) => {
1018
- return Promise.resolve();
1019
- },
1020
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1021
- compact: async (blocks) => {
1022
- return {};
1023
- },
1024
- autoCompact: 100,
1025
- public: false,
1026
- // name: undefined,
1027
- threshold: 1e3 * 1e3,
1028
- ...opts,
1029
- logger,
1030
- keyBag: opts.keyBag || {},
1031
- crypto: toCryptoRuntime2(opts.crypto),
1032
- storeUrls: opts.storeUrls,
1033
- // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
1034
- // store,
1035
- storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
1036
- };
907
+
908
+ // src/blockstore/loader.ts
909
+ function carLogIncludesGroup(list, cids) {
910
+ return list.some((arr) => {
911
+ return arr.toString() === cids.toString();
912
+ });
1037
913
  }
1038
- function blockstoreFactory(sthis, opts) {
1039
- return new EncryptedBlockstore(sthis, opts);
914
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
915
+ const byString = /* @__PURE__ */ new Map();
916
+ for (const cid of list) {
917
+ if (remove.has(cid.toString())) continue;
918
+ byString.set(cid.toString(), cid);
919
+ }
920
+ return [...byString.values()];
1040
921
  }
1041
- var BaseBlockstore = class {
1042
- constructor(ebOpts) {
1043
- this.transactions = /* @__PURE__ */ new Set();
1044
- this.sthis = ensureSuperThis(ebOpts);
1045
- this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
1046
- this.sthis = ensureSuperThis(ebOpts);
1047
- this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
922
+ var Loader = class {
923
+ constructor(name, ebOpts, sthis) {
924
+ this.commitQueue = new CommitQueue();
925
+ this.isCompacting = false;
926
+ this.carReaders = /* @__PURE__ */ new Map();
927
+ this.seenCompacted = /* @__PURE__ */ new Set();
928
+ this.processedCars = /* @__PURE__ */ new Set();
929
+ this.carLog = [];
930
+ this.getBlockCache = /* @__PURE__ */ new Map();
931
+ this.seenMeta = /* @__PURE__ */ new Set();
932
+ this.writeLimit = pLimit(1);
933
+ this.onceReady = new ResolveOnce2();
934
+ this.name = name;
935
+ this.sthis = sthis;
936
+ this.ebOpts = defaultedBlockstoreRuntime(
937
+ sthis,
938
+ {
939
+ ...ebOpts,
940
+ name
941
+ },
942
+ "Loader"
943
+ );
1048
944
  this.logger = this.ebOpts.logger;
945
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
946
+ await this.handleDbMetasFromStore([dbMeta]);
947
+ });
1049
948
  }
1050
- // readonly name?: string;
1051
- // ready: Promise<void>;
1052
- ready() {
1053
- return Promise.resolve();
1054
- }
1055
- async close() {
949
+ // readonly id = uuidv4();
950
+ async keyBag() {
951
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
1056
952
  }
1057
- async destroy() {
953
+ async carStore() {
954
+ return this.ebOpts.storeRuntime.makeDataStore(this);
1058
955
  }
1059
- async compact() {
956
+ async fileStore() {
957
+ return this.ebOpts.storeRuntime.makeDataStore(this);
1060
958
  }
1061
- async get(cid) {
1062
- if (!cid) throw this.logger.Error().Msg("required cid").AsError();
1063
- for (const f of this.transactions) {
1064
- const v = await f.superGet(cid);
1065
- if (v) return v;
1066
- }
959
+ async WALStore() {
960
+ return this.ebOpts.storeRuntime.makeWALStore(this);
1067
961
  }
1068
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
1069
- async put(cid, block) {
1070
- throw this.logger.Error().Msg("use a transaction to put").AsError();
962
+ async metaStore() {
963
+ return this.ebOpts.storeRuntime.makeMetaStore(this);
1071
964
  }
1072
- // TransactionMeta
1073
- async transaction(fn, _opts) {
1074
- this.logger.Debug().Msg("enter transaction");
1075
- const t = new CarTransaction(this, _opts);
1076
- this.logger.Debug().Msg("post CarTransaction");
1077
- const done = await fn(t);
1078
- this.logger.Debug().Msg("post fn");
1079
- this.lastTxMeta = done;
1080
- return { t, meta: done };
965
+ async ready() {
966
+ return this.onceReady.once(async () => {
967
+ const metas = await (await this.metaStore()).load();
968
+ if (this.ebOpts.meta) {
969
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
970
+ } else if (metas) {
971
+ await this.handleDbMetasFromStore(metas);
972
+ }
973
+ });
1081
974
  }
1082
- openTransaction(opts = { add: true, noLoader: false }) {
1083
- return new CarTransaction(this, opts);
975
+ async close() {
976
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
977
+ await Promise.all(toClose.map((store) => store.close()));
1084
978
  }
1085
- async commitTransaction(t, done, opts) {
1086
- if (!this.loader) throw this.logger.Error().Msg("loader required to commit").AsError();
1087
- const cars = await this.loader?.commit(t, done, opts);
1088
- if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1089
- setTimeout(() => void this.compact(), 10);
1090
- }
1091
- if (cars) {
1092
- this.transactions.delete(t);
1093
- return { meta: done, cars, t };
1094
- }
1095
- throw this.logger.Error().Msg("failed to commit car files").AsError();
979
+ async destroy() {
980
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
981
+ await Promise.all(toDestroy.map((store) => store.destroy()));
1096
982
  }
1097
- async *entries() {
1098
- const seen = /* @__PURE__ */ new Set();
1099
- for (const t of this.transactions) {
1100
- for await (const blk of t.entries()) {
1101
- if (seen.has(blk.cid.toString())) continue;
1102
- seen.add(blk.cid.toString());
1103
- yield blk;
1104
- }
983
+ // async snapToCar(carCid: AnyLink | string) {
984
+ // await this.ready
985
+ // if (typeof carCid === 'string') {
986
+ // carCid = CID.parse(carCid)
987
+ // }
988
+ // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
989
+ // this.carLog = [carCid, ...carHeader.cars]
990
+ // await this.getMoreReaders(carHeader.cars)
991
+ // await this._applyCarHeader(carHeader, true)
992
+ // }
993
+ async handleDbMetasFromStore(metas) {
994
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
995
+ for (const meta of metas) {
996
+ await this.writeLimit(async () => {
997
+ await this.mergeDbMetaIntoClock(meta);
998
+ });
1105
999
  }
1106
1000
  }
1107
- };
1108
- var EncryptedBlockstore = class extends BaseBlockstore {
1109
- constructor(sthis, ebOpts) {
1110
- super(ebOpts);
1111
- this.compacting = false;
1112
- this.logger = ensureLogger(this.sthis, "EncryptedBlockstore", {
1113
- this: 1
1114
- });
1115
- this.loader = new Loader(sthis, ebOpts);
1116
- }
1117
- ready() {
1118
- return this.loader.ready();
1119
- }
1120
- close() {
1121
- return this.loader.close();
1122
- }
1123
- destroy() {
1124
- return this.loader.destroy();
1125
- }
1126
- async get(cid) {
1127
- const got = await super.get(cid);
1128
- if (got) return got;
1129
- if (!this.loader) {
1001
+ async mergeDbMetaIntoClock(meta) {
1002
+ if (this.isCompacting) {
1003
+ throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1004
+ }
1005
+ if (this.seenMeta.has(meta.cars.toString())) return;
1006
+ this.seenMeta.add(meta.cars.toString());
1007
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
1130
1008
  return;
1131
1009
  }
1132
- return falsyToUndef(await this.loader.getBlock(cid));
1010
+ const carHeader = await this.loadCarHeaderFromMeta(meta);
1011
+ carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1012
+ await this.getMoreReaders(carHeader.cars.flat());
1013
+ this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1014
+ await this.ebOpts.applyMeta?.(carHeader.meta);
1133
1015
  }
1134
- async transaction(fn, opts = { noLoader: false }) {
1135
- this.logger.Debug().Msg("enter transaction");
1136
- const { t, meta: done } = await super.transaction(fn);
1137
- this.logger.Debug().Msg("post super.transaction");
1138
- const cars = await this.loader.commit(t, done, opts);
1139
- this.logger.Debug().Msg("post this.loader.commit");
1140
- if (this.ebOpts.autoCompact && this.loader.carLog.length > this.ebOpts.autoCompact) {
1141
- setTimeout(() => void this.compact(), 10);
1142
- }
1143
- if (cars) {
1144
- this.transactions.delete(t);
1145
- return { meta: done, cars, t };
1146
- }
1147
- throw this.logger.Error().Msg("failed to commit car files").AsError();
1016
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1017
+ // const { key } = meta;
1018
+ // if (key) {
1019
+ // await this.setKey(key);
1020
+ // }
1021
+ // }
1022
+ async loadCarHeaderFromMeta({ cars: cids }) {
1023
+ const reader = await this.loadCar(cids[0]);
1024
+ return await parseCarFile(reader, this.logger);
1148
1025
  }
1149
- async getFile(car, cid) {
1026
+ // async _getKey(): Promise<string | undefined> {
1027
+ // if (this.key) return this.key;
1028
+ // // generate a random key
1029
+ // if (!this.ebOpts.public) {
1030
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1031
+ // }
1032
+ // return this.key || undefined;
1033
+ // }
1034
+ async commitFiles(t, done) {
1150
1035
  await this.ready();
1151
- if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
1152
- const reader = await this.loader.loadFileCar(
1153
- car
1154
- /*, isPublic */
1155
- );
1156
- const block = await reader.get(cid);
1157
- if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
1158
- return block.bytes;
1036
+ const fstore = await this.fileStore();
1037
+ const wstore = await this.WALStore();
1038
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1159
1039
  }
1160
- async compact() {
1040
+ async loadFileCar(cid) {
1041
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1042
+ }
1043
+ async commit(t, done, opts = { noLoader: false, compact: false }) {
1161
1044
  await this.ready();
1162
- if (!this.loader) throw this.logger.Error().Msg("loader required to compact").AsError();
1163
- if (this.loader.carLog.length < 2) return;
1164
- const compactFn = this.ebOpts.compact || ((blocks) => this.defaultCompact(blocks, this.logger));
1165
- if (!compactFn || this.compacting) return;
1166
- const blockLog = new CompactionFetcher(this);
1167
- this.compacting = true;
1168
- const meta = await compactFn(blockLog);
1169
- await this.loader?.commit(blockLog.loggedBlocks, meta, {
1170
- compact: true,
1171
- noLoader: true
1045
+ const fstore = await this.fileStore();
1046
+ const params = {
1047
+ encoder: (await fstore.keyedCrypto()).codec(),
1048
+ carLog: this.carLog,
1049
+ carStore: fstore,
1050
+ WALStore: await this.WALStore(),
1051
+ metaStore: await this.metaStore(),
1052
+ threshold: this.ebOpts.threshold
1053
+ };
1054
+ return this.commitQueue.enqueue(async () => {
1055
+ await this.cacheTransaction(t);
1056
+ const ret = await commit(params, t, done, opts);
1057
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1058
+ return ret.cgrp;
1172
1059
  });
1173
- this.compacting = false;
1174
1060
  }
1175
- async defaultCompact(blocks, logger) {
1176
- if (!this.loader) {
1177
- throw logger.Error().Msg("no loader").AsError();
1178
- }
1179
- if (!this.lastTxMeta) {
1180
- throw logger.Error().Msg("no lastTxMeta").AsError();
1181
- }
1182
- for await (const blk of this.loader.entries(false)) {
1183
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1061
+ async updateCarLog(cids, fp, compact) {
1062
+ if (compact) {
1063
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
1064
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1065
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1066
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1067
+ } else {
1068
+ this.carLog.unshift(cids);
1184
1069
  }
1185
- for (const t of this.transactions) {
1186
- for await (const blk of t.entries()) {
1187
- blocks.loggedBlocks.putSync(blk.cid, blk.bytes);
1070
+ }
1071
+ async cacheTransaction(t) {
1072
+ for await (const block of t.entries()) {
1073
+ const sBlock = block.cid.toString();
1074
+ if (!this.getBlockCache.has(sBlock)) {
1075
+ this.getBlockCache.set(sBlock, block);
1188
1076
  }
1189
1077
  }
1190
- return this.lastTxMeta;
1191
1078
  }
1192
- async *entries() {
1193
- for await (const blk of this.loader.entries()) {
1194
- yield blk;
1079
+ async cacheCarReader(carCidStr, reader) {
1080
+ if (this.processedCars.has(carCidStr)) return;
1081
+ this.processedCars.add(carCidStr);
1082
+ for await (const block of reader.blocks()) {
1083
+ const sBlock = block.cid.toString();
1084
+ if (!this.getBlockCache.has(sBlock)) {
1085
+ this.getBlockCache.set(sBlock, block);
1086
+ }
1195
1087
  }
1196
1088
  }
1197
- };
1198
- var CompactionFetcher = class {
1199
- constructor(blocks) {
1200
- this.blockstore = blocks;
1201
- this.loggedBlocks = new CarTransaction(blocks);
1202
- }
1203
- async get(cid) {
1204
- const block = await this.blockstore.get(cid);
1205
- if (block) this.loggedBlocks.putSync(cid, block.bytes);
1206
- return falsyToUndef(block);
1207
- }
1208
- };
1209
-
1210
- // src/blockstore/commitor.ts
1211
- import * as CBW from "@ipld/car/buffer-writer";
1212
- import { sha256 as hasher3 } from "multiformats/hashes/sha2";
1213
- import * as dagCodec2 from "@ipld/dag-cbor";
1214
- async function encodeCarFile(roots, t, codec3) {
1215
- let size = 0;
1216
- const headerSize = CBW.headerLength({ roots });
1217
- size += headerSize;
1218
- for (const { cid, bytes } of t.entries()) {
1219
- size += CBW.blockLength({ cid, bytes });
1220
- }
1221
- const buffer = new Uint8Array(size);
1222
- const writer = CBW.createWriter(buffer, { headerSize });
1223
- for (const r of roots) {
1224
- writer.addRoot(r);
1225
- }
1226
- for (const { cid, bytes } of t.entries()) {
1227
- writer.write({ cid, bytes });
1228
- }
1229
- writer.close();
1230
- return await encode3({ value: writer.bytes, hasher: hasher3, codec: codec3 });
1231
- }
1232
- async function createCarFile(encoder, cid, t) {
1233
- return encodeCarFile([cid], t, encoder);
1234
- }
1235
- async function commitFiles(fileStore, walStore, t, done) {
1236
- const { files: roots } = makeFileCarHeader(done);
1237
- const cids = [];
1238
- const codec3 = (await fileStore.keyedCrypto()).codec();
1239
- const cars = await prepareCarFilesFiles(codec3, roots, t);
1240
- for (const car of cars) {
1241
- const { cid, bytes } = car;
1242
- await fileStore.save({ cid, bytes });
1243
- await walStore.enqueueFile(
1244
- cid
1245
- /*, !!opts.public*/
1246
- );
1247
- cids.push(cid);
1248
- }
1249
- return cids;
1250
- }
1251
- function makeFileCarHeader(result) {
1252
- const files = [];
1253
- for (const [, meta] of Object.entries(result.files || {})) {
1254
- if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1255
- files.push(meta.cid);
1089
+ async removeCidsForCompact(cid) {
1090
+ const carHeader = await this.loadCarHeaderFromMeta({
1091
+ cars: [cid]
1092
+ });
1093
+ for (const cids of carHeader.compact) {
1094
+ for (const cid2 of cids) {
1095
+ await (await this.carStore()).remove(cid2);
1096
+ }
1256
1097
  }
1257
1098
  }
1258
- return { ...result, files };
1259
- }
1260
- async function prepareCarFilesFiles(encoder, roots, t) {
1261
- return [await encodeCarFile(roots, t, encoder)];
1262
- }
1263
- function makeCarHeader(meta, cars, compact = false) {
1264
- const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1265
- return { ...coreHeader, meta };
1266
- }
1267
- async function encodeCarHeader(fp) {
1268
- return await encode3({
1269
- value: { fp },
1270
- hasher: hasher3,
1271
- codec: dagCodec2
1272
- });
1273
- }
1274
- async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
1275
- const fp = makeCarHeader(done, params.carLog, !!opts.compact);
1276
- const rootBlock = await encodeCarHeader(fp);
1277
- const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
1278
- const cids = [];
1279
- for (const car of cars) {
1280
- const { cid, bytes } = car;
1281
- await params.carStore.save({ cid, bytes });
1282
- cids.push(cid);
1283
- }
1284
- const newDbMeta = { cars: cids };
1285
- await params.WALStore.enqueue(newDbMeta, opts);
1286
- await params.metaStore.save(newDbMeta);
1287
- return { cgrp: cids, header: fp };
1288
- }
1289
- async function prepareCarFiles(encoder, threshold, rootBlock, t) {
1290
- const carFiles = [];
1291
- threshold = threshold || 128e3 * 8;
1292
- let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1293
- clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1294
- let newsize = CBW.blockLength(toCIDBlock(rootBlock));
1295
- let cidRootBlock = rootBlock;
1296
- for (const { cid, bytes } of t.entries()) {
1297
- newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
1298
- if (newsize >= threshold) {
1299
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1300
- clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
1301
- clonedt.putSync(cid, bytes);
1302
- cidRootBlock = { cid, bytes };
1303
- newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
1099
+ // async flushCars() {
1100
+ // await this.ready
1101
+ // // for each cid in car log, make a dbMeta
1102
+ // for (const cid of this.carLog) {
1103
+ // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1104
+ // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1105
+ // }
1106
+ // }
1107
+ async *entries(cache2 = true) {
1108
+ await this.ready();
1109
+ if (cache2) {
1110
+ for (const [, block] of this.getBlockCache) {
1111
+ yield block;
1112
+ }
1304
1113
  } else {
1305
- clonedt.putSync(cid, bytes);
1114
+ for (const [, block] of this.getBlockCache) {
1115
+ yield block;
1116
+ }
1117
+ for (const cids of this.carLog) {
1118
+ for (const cid of cids) {
1119
+ const reader = await this.loadCar(cid);
1120
+ if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1121
+ for await (const block of reader.blocks()) {
1122
+ const sCid = block.cid.toString();
1123
+ if (!this.getBlockCache.has(sCid)) {
1124
+ yield block;
1125
+ }
1126
+ }
1127
+ }
1128
+ }
1306
1129
  }
1307
1130
  }
1308
- carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
1309
- return carFiles;
1310
- }
1311
-
1312
- // src/blockstore/loader.ts
1313
- import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1314
-
1315
- // src/blockstore/task-manager.ts
1316
- var TaskManager = class {
1317
- constructor(sthis, callback) {
1318
- this.eventsWeHandled = /* @__PURE__ */ new Set();
1319
- this.queue = [];
1320
- this.isProcessing = false;
1321
- this.logger = ensureLogger(sthis, "TaskManager");
1322
- this.callback = callback;
1323
- }
1324
- async handleEvent(cid, parents, dbMeta) {
1325
- for (const parent of parents) {
1326
- this.eventsWeHandled.add(parent.toString());
1131
+ async getBlock(cid) {
1132
+ await this.ready();
1133
+ const sCid = cid.toString();
1134
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1135
+ const getCarCid = async (carCid) => {
1136
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1137
+ const reader = await this.loadCar(carCid);
1138
+ if (!reader) {
1139
+ throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1140
+ }
1141
+ await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1142
+ return;
1143
+ });
1144
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1145
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1146
+ };
1147
+ const getCompactCarCids = async (carCid) => {
1148
+ const reader = await this.loadCar(carCid);
1149
+ if (!reader) {
1150
+ throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1151
+ }
1152
+ const header = await parseCarFile(reader, this.logger);
1153
+ const compacts = header.compact;
1154
+ let got2;
1155
+ const batchSize2 = 5;
1156
+ for (let i = 0; i < compacts.length; i += batchSize2) {
1157
+ const promises = [];
1158
+ for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1159
+ for (const cid2 of compacts[j]) {
1160
+ promises.push(getCarCid(cid2));
1161
+ }
1162
+ }
1163
+ try {
1164
+ got2 = await Promise.any(promises);
1165
+ } catch {
1166
+ }
1167
+ if (got2) break;
1168
+ }
1169
+ if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1170
+ throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1171
+ };
1172
+ let got;
1173
+ const batchSize = 5;
1174
+ for (let i = 0; i < this.carLog.length; i += batchSize) {
1175
+ const batch = this.carLog.slice(i, i + batchSize);
1176
+ const promises = batch.flatMap((slice) => slice.map(getCarCid));
1177
+ try {
1178
+ got = await Promise.any(promises);
1179
+ } catch {
1180
+ }
1181
+ if (got) break;
1327
1182
  }
1328
- this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
1329
- this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
1330
- void this.processQueue();
1183
+ if (!got) {
1184
+ try {
1185
+ got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1186
+ } catch {
1187
+ }
1188
+ }
1189
+ return got;
1331
1190
  }
1332
- async processQueue() {
1333
- if (this.isProcessing) return;
1334
- this.isProcessing = true;
1335
- const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1336
- const first = filteredQueue[0];
1337
- if (!first) {
1338
- return;
1191
+ async loadCar(cid) {
1192
+ if (!this.carStore) {
1193
+ throw this.logger.Error().Msg("car store not initialized").AsError();
1339
1194
  }
1195
+ const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1196
+ return loaded;
1197
+ }
1198
+ async makeDecoderAndCarReader(cid, local, remote) {
1199
+ const cidsString = cid.toString();
1200
+ let loadedCar = void 0;
1201
+ let activeStore = local;
1340
1202
  try {
1341
- await this.callback(first.dbMeta);
1342
- this.eventsWeHandled.add(first.cid);
1343
- this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
1344
- } catch (err) {
1345
- if (first.retries++ > 3) {
1346
- this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
1347
- this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
1348
- }
1349
- await new Promise((resolve) => setTimeout(resolve, 50));
1350
- throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
1351
- } finally {
1352
- this.isProcessing = false;
1353
- if (this.queue.length > 0) {
1354
- void this.processQueue();
1203
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1204
+ loadedCar = await local.load(cid);
1205
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1206
+ } catch (e) {
1207
+ if (remote) {
1208
+ const remoteCar = await remote.load(cid);
1209
+ if (remoteCar) {
1210
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1211
+ await local.save(remoteCar);
1212
+ loadedCar = remoteCar;
1213
+ activeStore = remote;
1214
+ }
1215
+ } else {
1216
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1355
1217
  }
1356
1218
  }
1219
+ if (!loadedCar) {
1220
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
1221
+ }
1222
+ const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
1223
+ const rawReader = await CarReader.fromBytes(bytes.value);
1224
+ const readerP = Promise.resolve(rawReader);
1225
+ const cachedReaderP = readerP.then(async (reader) => {
1226
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
1227
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1228
+ return;
1229
+ });
1230
+ return reader;
1231
+ });
1232
+ this.carReaders.set(cidsString, cachedReaderP);
1233
+ return readerP;
1234
+ }
1235
+ //What if instead it returns an Array of CarHeader
1236
+ async storesLoadCar(cid, local, remote) {
1237
+ const cidsString = cid.toString();
1238
+ let dacr = this.carReaders.get(cidsString);
1239
+ if (!dacr) {
1240
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
1241
+ this.carReaders.set(cidsString, dacr);
1242
+ }
1243
+ return dacr;
1244
+ }
1245
+ async getMoreReaders(cids) {
1246
+ const limit = pLimit(5);
1247
+ const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1248
+ await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1357
1249
  }
1358
1250
  };
1359
1251
 
1360
- // src/blockstore/loader.ts
1361
- function carLogIncludesGroup(list, cids) {
1362
- return list.some((arr) => {
1363
- return arr.toString() === cids.toString();
1364
- });
1365
- }
1366
- function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
1367
- const byString = /* @__PURE__ */ new Map();
1368
- for (const cid of list) {
1369
- if (remove.has(cid.toString())) continue;
1370
- byString.set(cid.toString(), cid);
1252
+ // src/runtime/keyed-crypto.ts
1253
+ var keyed_crypto_exports = {};
1254
+ __export(keyed_crypto_exports, {
1255
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1256
+ keyedCryptoFactory: () => keyedCryptoFactory
1257
+ });
1258
+ import { base58btc as base58btc2 } from "multiformats/bases/base58";
1259
+ import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1260
+ import * as CBOR from "cborg";
1261
+ var generateIV = {
1262
+ random: {
1263
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1264
+ calc: async (ko, crypto, data) => {
1265
+ return crypto.randomBytes(ko.ivLength);
1266
+ },
1267
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1268
+ verify: async (ko, crypto, iv, data) => {
1269
+ return true;
1270
+ }
1271
+ },
1272
+ hash: {
1273
+ calc: async (ko, crypto, data) => {
1274
+ const hash = await hasher4.digest(data);
1275
+ const hashBytes = new Uint8Array(hash.bytes);
1276
+ const hashArray = new Uint8Array(ko.ivLength);
1277
+ for (let i = 0; i < hashBytes.length; i++) {
1278
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
1279
+ }
1280
+ return hashArray;
1281
+ },
1282
+ verify: async function(ko, crypto, iv, data) {
1283
+ return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1284
+ }
1371
1285
  }
1372
- return [...byString.values()];
1286
+ };
1287
+ function getGenerateIVFn(url, opts) {
1288
+ const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
1289
+ return generateIV[ivhash] || generateIV["hash"];
1373
1290
  }
1374
- var Loader = class {
1375
- constructor(sthis, ebOpts) {
1376
- this.commitQueue = new CommitQueue();
1377
- this.isCompacting = false;
1378
- this.carReaders = /* @__PURE__ */ new Map();
1379
- this.seenCompacted = /* @__PURE__ */ new Set();
1380
- this.processedCars = /* @__PURE__ */ new Set();
1381
- this.carLog = [];
1382
- this.getBlockCache = /* @__PURE__ */ new Map();
1383
- this.seenMeta = /* @__PURE__ */ new Set();
1384
- this.writeLimit = pLimit(1);
1385
- this._carStore = new ResolveOnce2();
1386
- this._fileStore = new ResolveOnce2();
1387
- this._WALStore = new ResolveOnce2();
1388
- this._metaStore = new ResolveOnce2();
1389
- this.onceReady = new ResolveOnce2();
1390
- this.sthis = sthis;
1391
- this.ebOpts = defaultedBlockstoreRuntime(
1392
- sthis,
1393
- {
1394
- ...ebOpts
1395
- // name,
1396
- },
1397
- "Loader"
1398
- );
1399
- this.logger = this.ebOpts.logger;
1400
- this.taskManager = new TaskManager(sthis, async (dbMeta) => {
1401
- await this.handleDbMetasFromStore([dbMeta]);
1291
+ var BlockIvKeyIdCodec = class {
1292
+ constructor(ko, iv, opts) {
1293
+ this.code = 3147065;
1294
+ this.name = "Fireproof@encrypted-block:aes-gcm";
1295
+ this.ko = ko;
1296
+ this.iv = iv;
1297
+ this.opts = opts || {};
1298
+ }
1299
+ async encode(data) {
1300
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1301
+ const { iv } = this.ko.algo(calcIv);
1302
+ const fprt = await this.ko.fingerPrint();
1303
+ const keyId = base58btc2.decode(fprt);
1304
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1305
+ return CBOR.encode({
1306
+ iv,
1307
+ keyId,
1308
+ data: await this.ko._encrypt({ iv, bytes: data })
1402
1309
  });
1403
1310
  }
1404
- async carStore() {
1405
- return this._carStore.once(
1406
- async () => this.ebOpts.storeRuntime.makeDataStore({
1407
- sthis: this.sthis,
1408
- url: this.ebOpts.storeUrls.data,
1409
- keybag: await this.keyBag()
1410
- })
1411
- );
1311
+ async decode(abytes) {
1312
+ let bytes;
1313
+ if (abytes instanceof Uint8Array) {
1314
+ bytes = abytes;
1315
+ } else {
1316
+ bytes = new Uint8Array(abytes);
1317
+ }
1318
+ const { iv, keyId, data } = CBOR.decode(bytes);
1319
+ const fprt = await this.ko.fingerPrint();
1320
+ this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
1321
+ if (base58btc2.encode(keyId) !== fprt) {
1322
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
1323
+ }
1324
+ const result = await this.ko._decrypt({ iv, bytes: data });
1325
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1326
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1327
+ }
1328
+ return result;
1329
+ }
1330
+ };
1331
+ var keyedCrypto = class {
1332
+ constructor(url, key, cyopt, sthis) {
1333
+ this.ivLength = 12;
1334
+ this.isEncrypting = true;
1335
+ this.logger = ensureLogger(sthis, "keyedCrypto");
1336
+ this.crypto = cyopt;
1337
+ this.key = key;
1338
+ this.url = url;
1339
+ }
1340
+ fingerPrint() {
1341
+ return Promise.resolve(this.key.fingerPrint);
1412
1342
  }
1413
- async fileStore() {
1414
- return this._fileStore.once(
1415
- async () => this.ebOpts.storeRuntime.makeDataStore({
1416
- sthis: this.sthis,
1417
- url: this.ebOpts.storeUrls.file,
1418
- keybag: await this.keyBag()
1419
- })
1420
- );
1343
+ codec(iv, opts) {
1344
+ return new BlockIvKeyIdCodec(this, iv, opts);
1421
1345
  }
1422
- async WALStore() {
1423
- return this._WALStore.once(
1424
- async () => this.ebOpts.storeRuntime.makeWALStore({
1425
- sthis: this.sthis,
1426
- url: this.ebOpts.storeUrls.wal,
1427
- keybag: await this.keyBag()
1428
- })
1429
- );
1346
+ algo(iv) {
1347
+ return {
1348
+ name: "AES-GCM",
1349
+ iv: iv || this.crypto.randomBytes(this.ivLength),
1350
+ tagLength: 128
1351
+ };
1430
1352
  }
1431
- async metaStore() {
1432
- return this._metaStore.once(
1433
- async () => this.ebOpts.storeRuntime.makeMetaStore({
1434
- sthis: this.sthis,
1435
- url: this.ebOpts.storeUrls.meta,
1436
- keybag: await this.keyBag()
1437
- })
1438
- );
1353
+ async _decrypt(data) {
1354
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
1355
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
1439
1356
  }
1440
- keyBag() {
1441
- return getKeyBag(this.sthis, this.ebOpts.keyBag);
1357
+ async _encrypt(data) {
1358
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
1359
+ const a = this.algo(data.iv);
1360
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
1442
1361
  }
1443
- async ready() {
1444
- return this.onceReady.once(async () => {
1445
- const metas = await (await this.metaStore()).load();
1446
- if (this.ebOpts.meta) {
1447
- await this.handleDbMetasFromStore([this.ebOpts.meta]);
1448
- } else if (metas) {
1449
- await this.handleDbMetasFromStore(metas);
1450
- }
1451
- });
1362
+ };
1363
+ var nullCodec = class {
1364
+ constructor() {
1365
+ this.code = 0;
1366
+ this.name = "Fireproof@unencrypted-block";
1452
1367
  }
1453
- async close() {
1454
- await this.commitQueue.waitIdle();
1455
- const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1456
- await Promise.all(toClose.map((store) => store.close()));
1368
+ encode(data) {
1369
+ return data;
1457
1370
  }
1458
- async destroy() {
1459
- const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
1460
- await Promise.all(toDestroy.map((store) => store.destroy()));
1371
+ decode(data) {
1372
+ return data;
1461
1373
  }
1462
- // async snapToCar(carCid: AnyLink | string) {
1463
- // await this.ready
1464
- // if (typeof carCid === 'string') {
1465
- // carCid = CID.parse(carCid)
1466
- // }
1467
- // const carHeader = await this.loadCarHeaderFromMeta({ car: carCid, key: this.key || null })
1468
- // this.carLog = [carCid, ...carHeader.cars]
1469
- // await this.getMoreReaders(carHeader.cars)
1470
- // await this._applyCarHeader(carHeader, true)
1471
- // }
1472
- async handleDbMetasFromStore(metas) {
1473
- this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
1474
- for (const meta of metas) {
1475
- await this.writeLimit(async () => {
1476
- await this.mergeDbMetaIntoClock(meta);
1477
- });
1478
- }
1374
+ };
1375
+ var noCrypto = class {
1376
+ constructor(url, cyrt, sthis) {
1377
+ this.ivLength = 0;
1378
+ this.code = 0;
1379
+ this.name = "Fireproof@unencrypted-block";
1380
+ this.isEncrypting = false;
1381
+ this._fingerPrint = "noCrypto:" + Math.random();
1382
+ this.logger = ensureLogger(sthis, "noCrypto");
1383
+ this.crypto = cyrt;
1384
+ this.url = url;
1479
1385
  }
1480
- async mergeDbMetaIntoClock(meta) {
1481
- if (this.isCompacting) {
1482
- throw this.logger.Error().Msg("cannot merge while compacting").AsError();
1483
- }
1484
- if (this.seenMeta.has(meta.cars.toString())) return;
1485
- this.seenMeta.add(meta.cars.toString());
1486
- if (carLogIncludesGroup(this.carLog, meta.cars)) {
1487
- return;
1488
- }
1489
- const carHeader = await this.loadCarHeaderFromMeta(meta);
1490
- carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1491
- await this.getMoreReaders(carHeader.cars.flat());
1492
- this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
1493
- await this.ebOpts.applyMeta?.(carHeader.meta);
1386
+ fingerPrint() {
1387
+ return Promise.resolve(this._fingerPrint);
1494
1388
  }
1495
- // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1496
- // const { key } = meta;
1497
- // if (key) {
1498
- // await this.setKey(key);
1499
- // }
1500
- // }
1501
- async loadCarHeaderFromMeta({ cars: cids }) {
1502
- const reader = await this.loadCar(cids[0]);
1503
- return await parseCarFile(reader, this.logger);
1389
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1390
+ codec(iv) {
1391
+ return new nullCodec();
1504
1392
  }
1505
- // async _getKey(): Promise<string | undefined> {
1506
- // if (this.key) return this.key;
1507
- // // generate a random key
1508
- // if (!this.ebOpts.public) {
1509
- // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1510
- // }
1511
- // return this.key || undefined;
1512
- // }
1513
- async commitFiles(t, done) {
1514
- await this.ready();
1515
- const fstore = await this.fileStore();
1516
- const wstore = await this.WALStore();
1517
- return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
1393
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1394
+ algo(iv) {
1395
+ return {
1396
+ name: "noCrypto",
1397
+ iv: new Uint8Array(),
1398
+ tagLength: 0
1399
+ };
1518
1400
  }
1519
- async loadFileCar(cid) {
1520
- return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
1401
+ _decrypt() {
1402
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1521
1403
  }
1522
- async commit(t, done, opts = { noLoader: false, compact: false }) {
1523
- await this.ready();
1524
- const carStore = await this.carStore();
1525
- const params = {
1526
- encoder: (await carStore.keyedCrypto()).codec(),
1527
- carLog: this.carLog,
1528
- carStore,
1529
- WALStore: await this.WALStore(),
1530
- metaStore: await this.metaStore(),
1531
- threshold: this.ebOpts.threshold
1532
- };
1533
- return this.commitQueue.enqueue(async () => {
1534
- await this.cacheTransaction(t);
1535
- const ret = await commit(params, t, done, opts);
1536
- await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1537
- return ret.cgrp;
1538
- });
1404
+ _encrypt() {
1405
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1539
1406
  }
1540
- async updateCarLog(cids, fp, compact) {
1541
- if (compact) {
1542
- const previousCompactCid = fp.compact[fp.compact.length - 1];
1543
- fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1544
- this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1545
- await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1546
- } else {
1547
- this.carLog.unshift(cids);
1407
+ };
1408
+ async function keyedCryptoFactory(url, kb, sthis) {
1409
+ const storekey = url.getParam("storekey");
1410
+ if (storekey && storekey !== "insecure") {
1411
+ let rkey = await kb.getNamedKey(storekey, true);
1412
+ if (rkey.isErr()) {
1413
+ try {
1414
+ rkey = await kb.toKeyWithFingerPrint(storekey);
1415
+ } catch (e) {
1416
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
1417
+ }
1548
1418
  }
1419
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
1549
1420
  }
1550
- async cacheTransaction(t) {
1551
- for await (const block of t.entries()) {
1552
- const sBlock = block.cid.toString();
1553
- if (!this.getBlockCache.has(sBlock)) {
1554
- this.getBlockCache.set(sBlock, block);
1555
- }
1421
+ return new noCrypto(url, kb.rt.crypto, sthis);
1422
+ }
1423
+
1424
+ // src/blockstore/fragment-gateway.ts
1425
+ import { Result as Result3 } from "@adviser/cement";
1426
+ import { base58btc as base58btc3 } from "multiformats/bases/base58";
1427
+ import { encode as encode3, decode as decode3 } from "cborg";
1428
+ function getFragSize(url) {
1429
+ const fragSize = url.getParam("fragSize");
1430
+ let ret = 0;
1431
+ if (fragSize) {
1432
+ ret = parseInt(fragSize);
1433
+ }
1434
+ if (isNaN(ret) || ret <= 0) {
1435
+ ret = 0;
1436
+ }
1437
+ return ret;
1438
+ }
1439
+ async function getFrags(url, innerGW, headerSize, logger) {
1440
+ const fragSize = getFragSize(url);
1441
+ if (!fragSize) {
1442
+ const res = await innerGW.get(url);
1443
+ if (res.isErr()) {
1444
+ return [res];
1556
1445
  }
1446
+ const data = res.unwrap();
1447
+ return [
1448
+ Result3.Ok({
1449
+ fid: new Uint8Array(0),
1450
+ ofs: 0,
1451
+ len: data.length,
1452
+ data
1453
+ })
1454
+ ];
1557
1455
  }
1558
- async cacheCarReader(carCidStr, reader) {
1559
- if (this.processedCars.has(carCidStr)) return;
1560
- this.processedCars.add(carCidStr);
1561
- for await (const block of reader.blocks()) {
1562
- const sBlock = block.cid.toString();
1563
- if (!this.getBlockCache.has(sBlock)) {
1564
- this.getBlockCache.set(sBlock, block);
1456
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
1457
+ if (firstRaw.isErr()) {
1458
+ return [firstRaw];
1459
+ }
1460
+ const firstFragment = decode3(firstRaw.unwrap());
1461
+ const blockSize = firstFragment.data.length;
1462
+ const ops = [Promise.resolve(Result3.Ok(firstFragment))];
1463
+ const fidStr = base58btc3.encode(firstFragment.fid);
1464
+ const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
1465
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
1466
+ ops.push(
1467
+ (async (furl, ofs2) => {
1468
+ const raw2 = await innerGW.get(furl);
1469
+ if (raw2.isErr()) {
1470
+ return raw2;
1471
+ }
1472
+ const fragment = decode3(raw2.unwrap());
1473
+ if (base58btc3.encode(fragment.fid) !== fidStr) {
1474
+ return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
1475
+ }
1476
+ if (fragment.ofs !== ofs2) {
1477
+ return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
1478
+ }
1479
+ return Result3.Ok(fragment);
1480
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
1481
+ );
1482
+ }
1483
+ return Promise.all(ops);
1484
+ }
1485
+ var FragmentGateway = class {
1486
+ constructor(sthis, innerGW) {
1487
+ this.fidLength = 4;
1488
+ this.headerSize = 32;
1489
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
1490
+ this.logger = this.sthis.logger;
1491
+ this.innerGW = innerGW;
1492
+ }
1493
+ slicer(url, body) {
1494
+ const fragSize = getFragSize(url);
1495
+ if (!fragSize) {
1496
+ return [this.innerGW.put(url, body)];
1497
+ }
1498
+ const blocksize = fragSize - this.headerSize;
1499
+ if (blocksize <= 0) {
1500
+ throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
1501
+ }
1502
+ const ops = [];
1503
+ const fid = this.sthis.nextId(this.fidLength);
1504
+ const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
1505
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
1506
+ const block = encode3({
1507
+ fid: fid.bin,
1508
+ ofs,
1509
+ len: body.length,
1510
+ data: body.slice(ofs, ofs + blocksize)
1511
+ });
1512
+ if (block.length > fragSize) {
1513
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
1565
1514
  }
1515
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
1566
1516
  }
1517
+ return ops;
1567
1518
  }
1568
- async removeCidsForCompact(cid) {
1569
- const carHeader = await this.loadCarHeaderFromMeta({
1570
- cars: [cid]
1571
- });
1572
- for (const cids of carHeader.compact) {
1573
- for (const cid2 of cids) {
1574
- await (await this.carStore()).remove(cid2);
1519
+ buildUrl(baseUrl, key) {
1520
+ return this.innerGW.buildUrl(baseUrl, key);
1521
+ }
1522
+ async destroy(iurl) {
1523
+ return this.innerGW.destroy(iurl);
1524
+ }
1525
+ async start(url) {
1526
+ this.headerSize = encode3({
1527
+ fid: this.sthis.nextId(this.fidLength).bin,
1528
+ ofs: 1024 * 1024,
1529
+ // 32bit
1530
+ len: 16 * 1024 * 1024,
1531
+ // 32bit
1532
+ data: new Uint8Array(1024)
1533
+ }).length - 1024;
1534
+ return this.innerGW.start(url);
1535
+ }
1536
+ async close(url) {
1537
+ return this.innerGW.close(url);
1538
+ }
1539
+ async put(url, body) {
1540
+ await Promise.all(this.slicer(url, body));
1541
+ return Result3.Ok(void 0);
1542
+ }
1543
+ async get(url) {
1544
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1545
+ let buffer = void 0;
1546
+ for (const rfrag of rfrags) {
1547
+ if (rfrag.isErr()) {
1548
+ return Result3.Err(rfrag.Err());
1575
1549
  }
1550
+ const frag = rfrag.Ok();
1551
+ buffer = buffer || new Uint8Array(frag.len);
1552
+ buffer.set(frag.data, frag.ofs);
1576
1553
  }
1554
+ return Result3.Ok(buffer || new Uint8Array(0));
1577
1555
  }
1578
- // async flushCars() {
1579
- // await this.ready
1580
- // // for each cid in car log, make a dbMeta
1581
- // for (const cid of this.carLog) {
1582
- // const dbMeta = { car: cid, key: this.key || null } as DbMeta
1583
- // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1584
- // }
1585
- // }
1586
- async *entries(cache2 = true) {
1587
- await this.ready();
1588
- if (cache2) {
1589
- for (const [, block] of this.getBlockCache) {
1590
- yield block;
1591
- }
1556
+ async subscribe(url, callback) {
1557
+ if (this.innerGW.subscribe) {
1558
+ return this.innerGW.subscribe(url, callback);
1592
1559
  } else {
1593
- for (const [, block] of this.getBlockCache) {
1594
- yield block;
1595
- }
1596
- for (const cids of this.carLog) {
1597
- for (const cid of cids) {
1598
- const reader = await this.loadCar(cid);
1599
- if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
1600
- for await (const block of reader.blocks()) {
1601
- const sCid = block.cid.toString();
1602
- if (!this.getBlockCache.has(sCid)) {
1603
- yield block;
1604
- }
1605
- }
1606
- }
1607
- }
1560
+ return Result3.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
1608
1561
  }
1609
1562
  }
1610
- async getBlock(cid) {
1611
- await this.ready();
1612
- const sCid = cid.toString();
1613
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1614
- const getCarCid = async (carCid) => {
1615
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1616
- const reader = await this.loadCar(carCid);
1617
- if (!reader) {
1618
- throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
1619
- }
1620
- await this.cacheCarReader(carCid.toString(), reader).catch(() => {
1621
- return;
1622
- });
1623
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1624
- throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
1625
- };
1626
- const getCompactCarCids = async (carCid) => {
1627
- const reader = await this.loadCar(carCid);
1628
- if (!reader) {
1629
- throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
1630
- }
1631
- const header = await parseCarFile(reader, this.logger);
1632
- const compacts = header.compact;
1633
- let got2;
1634
- const batchSize2 = 5;
1635
- for (let i = 0; i < compacts.length; i += batchSize2) {
1636
- const promises = [];
1637
- for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
1638
- for (const cid2 of compacts[j]) {
1639
- promises.push(getCarCid(cid2));
1640
- }
1641
- }
1642
- try {
1643
- got2 = await Promise.any(promises);
1644
- } catch {
1645
- }
1646
- if (got2) break;
1647
- }
1648
- if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
1649
- throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
1650
- };
1651
- let got;
1652
- const batchSize = 5;
1653
- for (let i = 0; i < this.carLog.length; i += batchSize) {
1654
- const batch = this.carLog.slice(i, i + batchSize);
1655
- const promises = batch.flatMap((slice) => slice.map(getCarCid));
1656
- try {
1657
- got = await Promise.any(promises);
1658
- } catch {
1659
- }
1660
- if (got) break;
1661
- }
1662
- if (!got) {
1663
- try {
1664
- got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
1665
- } catch {
1563
+ async delete(url) {
1564
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1565
+ for (const rfrag of rfrags) {
1566
+ if (rfrag.isErr()) {
1567
+ return Result3.Err(rfrag.Err());
1666
1568
  }
1569
+ const frag = rfrag.Ok();
1570
+ const fidStr = base58btc3.encode(frag.fid);
1571
+ const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
1572
+ await this.innerGW.delete(fragUrl);
1667
1573
  }
1668
- return got;
1574
+ return Result3.Ok(void 0);
1669
1575
  }
1670
- async loadCar(cid) {
1671
- if (!this.carStore) {
1672
- throw this.logger.Error().Msg("car store not initialized").AsError();
1673
- }
1674
- const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1675
- return loaded;
1576
+ };
1577
+
1578
+ // src/blockstore/meta-key-helper.ts
1579
+ import { format, parse } from "@ipld/dag-json";
1580
+ import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
1581
+ import { CID as CID2 } from "multiformats";
1582
+ import { base64pad } from "multiformats/bases/base64";
1583
+ import { Result as Result4 } from "@adviser/cement";
1584
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
1585
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
1586
+ if (!crdtEntries.length) {
1587
+ sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
1588
+ return [];
1676
1589
  }
1677
- async makeDecoderAndCarReader(cid, local, remote) {
1678
- const cidsString = cid.toString();
1679
- let loadedCar = void 0;
1680
- let activeStore = local;
1681
- try {
1682
- this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1683
- loadedCar = await local.load(cid);
1684
- this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1685
- } catch (e) {
1686
- if (remote) {
1687
- const remoteCar = await remote.load(cid);
1688
- if (remoteCar) {
1689
- this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1690
- await local.save(remoteCar);
1691
- loadedCar = remoteCar;
1692
- activeStore = remote;
1590
+ if (!crdtEntries.map) {
1591
+ sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
1592
+ return [];
1593
+ }
1594
+ return Promise.all(
1595
+ crdtEntries.map(async (crdtEntry) => {
1596
+ const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
1597
+ const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
1598
+ return {
1599
+ eventCid: eventBlock.cid,
1600
+ parents: crdtEntry.parents,
1601
+ dbMeta
1602
+ };
1603
+ })
1604
+ );
1605
+ }
1606
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
1607
+ try {
1608
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
1609
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
1610
+ if (keyInfo.length) {
1611
+ const dbMeta = keyInfo[0].dbMeta;
1612
+ if (dbMeta.key) {
1613
+ const kb = await getKeyBag(sthis);
1614
+ const keyName = getStoreKeyName(uri);
1615
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
1616
+ if (res.isErr()) {
1617
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
1618
+ throw res.Err();
1693
1619
  }
1694
- } else {
1695
- this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1696
1620
  }
1621
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
1622
+ return Result4.Ok(dbMeta);
1697
1623
  }
1698
- if (!loadedCar) {
1699
- throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
1700
- }
1701
- const bytes = await decode3({ bytes: loadedCar.bytes, hasher: hasher4, codec: (await activeStore.keyedCrypto()).codec() });
1702
- const rawReader = await CarReader.fromBytes(bytes.value);
1703
- const readerP = Promise.resolve(rawReader);
1704
- const cachedReaderP = readerP.then(async (reader) => {
1705
- await this.cacheCarReader(cidsString, reader).catch((e) => {
1706
- this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1707
- return;
1708
- });
1709
- return reader;
1710
- });
1711
- this.carReaders.set(cidsString, cachedReaderP);
1712
- return readerP;
1624
+ sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
1625
+ return Result4.Ok(void 0);
1626
+ } catch (error) {
1627
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
1628
+ return Result4.Err(error);
1713
1629
  }
1714
- //What if instead it returns an Array of CarHeader
1715
- async storesLoadCar(cid, local, remote) {
1716
- const cidsString = cid.toString();
1717
- let dacr = this.carReaders.get(cidsString);
1718
- if (!dacr) {
1719
- dacr = this.makeDecoderAndCarReader(cid, local, remote);
1720
- this.carReaders.set(cidsString, dacr);
1630
+ }
1631
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
1632
+ try {
1633
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
1634
+ const keyName = getStoreKeyName(uri);
1635
+ const kb = await getKeyBag(sthis);
1636
+ const res = await kb.getNamedExtractableKey(keyName, true);
1637
+ if (res.isErr()) {
1638
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
1639
+ throw res.Err();
1721
1640
  }
1722
- return dacr;
1723
- }
1724
- async getMoreReaders(cids) {
1725
- const limit = pLimit(5);
1726
- const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
1727
- await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
1641
+ const keyData = await res.Ok().extract();
1642
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
1643
+ const { dbMeta, parents } = dbMetas[0];
1644
+ const parentLinks = parents.map((p) => CID2.parse(p));
1645
+ dbMeta.key = keyData.keyStr;
1646
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
1647
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
1648
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
1649
+ return Result4.Ok(encoded);
1650
+ } catch (error) {
1651
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
1652
+ return Result4.Err(error);
1728
1653
  }
1729
- };
1654
+ }
1655
+ function getStoreKeyName(url) {
1656
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
1657
+ const idx = url.getParam("index");
1658
+ if (idx) {
1659
+ storeKeyName.push(idx);
1660
+ }
1661
+ storeKeyName.push("data");
1662
+ return `@${storeKeyName.join(":")}@`;
1663
+ }
1664
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
1665
+ const event = await EventBlock.create(
1666
+ {
1667
+ dbMeta: sthis.txt.encode(format(dbMeta))
1668
+ },
1669
+ parents
1670
+ );
1671
+ return event;
1672
+ }
1673
+ async function encodeEventsWithParents(sthis, events, parents) {
1674
+ const crdtEntries = events.map((event) => {
1675
+ const base64String = base64pad.encode(event.bytes);
1676
+ return {
1677
+ cid: event.cid.toString(),
1678
+ data: base64String,
1679
+ parents: parents.map((p) => p.toString())
1680
+ };
1681
+ });
1682
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
1683
+ }
1730
1684
 
1731
1685
  // src/blockstore/store.ts
1686
+ import pRetry from "p-retry";
1687
+ import pMap from "p-map";
1732
1688
  function guardVersion(url) {
1733
1689
  if (!url.hasParam("version")) {
1734
1690
  return Result5.Err(`missing version: ${url.toString()}`);
@@ -1736,21 +1692,16 @@ function guardVersion(url) {
1736
1692
  return Result5.Ok(url);
1737
1693
  }
1738
1694
  var BaseStoreImpl = class {
1739
- constructor(sthis, url, opts, logger) {
1695
+ constructor(name, url, opts, sthis, logger) {
1740
1696
  this._onStarted = [];
1741
1697
  this._onClosed = [];
1698
+ this.name = name;
1742
1699
  this._url = url;
1743
1700
  this.keybag = opts.keybag;
1744
- this.loader = opts.loader;
1745
1701
  this.sthis = sthis;
1746
- const name = this._url.getParam("name" /* NAME */);
1747
- if (!name) {
1748
- throw logger.Error().Str("url", this._url.toString()).Msg("missing name").AsError();
1749
- }
1750
- this.name = name;
1751
- this.logger = logger.With().Str("this", this.sthis.nextId().str).Ref("url", () => this._url.toString()).Logger();
1752
- this.realGateway = opts.gateway;
1702
+ this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
1753
1703
  this.gateway = new FragmentGateway(this.sthis, opts.gateway);
1704
+ this.loader = opts.loader;
1754
1705
  }
1755
1706
  url() {
1756
1707
  return this._url;
@@ -1765,20 +1716,20 @@ var BaseStoreImpl = class {
1765
1716
  return;
1766
1717
  }
1767
1718
  async keyedCrypto() {
1768
- return keyedCryptoFactory(this._url, this.keybag, this.sthis);
1719
+ return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
1769
1720
  }
1770
1721
  async start() {
1771
1722
  this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
1772
- this._url = this._url.build().setParam("store" /* STORE */, this.storeType).URI();
1723
+ this._url = this._url.build().setParam("store", this.storeType).URI();
1773
1724
  const res = await this.gateway.start(this._url);
1774
1725
  if (res.isErr()) {
1775
1726
  this.logger.Error().Result("gw-start", res).Msg("started-gateway");
1776
1727
  return res;
1777
1728
  }
1778
1729
  this._url = res.Ok();
1779
- const kb = await this.keybag;
1730
+ const kb = await this.keybag();
1780
1731
  const skRes = await kb.ensureKeyFromUrl(this._url, () => {
1781
- const idx = this._url.getParam("index" /* INDEX */);
1732
+ const idx = this._url.getParam("index");
1782
1733
  const storeKeyName = [this.name];
1783
1734
  if (idx) {
1784
1735
  storeKeyName.push(idx);
@@ -1811,8 +1762,8 @@ var BaseStoreImpl = class {
1811
1762
  };
1812
1763
  var MetaStoreImpl = class extends BaseStoreImpl {
1813
1764
  // remote: boolean;
1814
- constructor(sthis, url, opts) {
1815
- super(sthis, url, { ...opts }, ensureLogger(sthis, "MetaStoreImpl"));
1765
+ constructor(sthis, name, url, opts) {
1766
+ super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
1816
1767
  this.storeType = "meta";
1817
1768
  this.subscribers = /* @__PURE__ */ new Map();
1818
1769
  this.parents = [];
@@ -1883,21 +1834,13 @@ var MetaStoreImpl = class extends BaseStoreImpl {
1883
1834
  return Result5.Ok(void 0);
1884
1835
  }
1885
1836
  async destroy() {
1886
- this.logger.Debug().Msg("destroy");
1887
1837
  return this.gateway.destroy(this.url());
1888
1838
  }
1889
1839
  };
1890
1840
  var DataStoreImpl = class extends BaseStoreImpl {
1891
1841
  // readonly tag: string = "car-base";
1892
- constructor(sthis, url, opts) {
1893
- super(
1894
- sthis,
1895
- url,
1896
- {
1897
- ...opts
1898
- },
1899
- ensureLogger(sthis, "DataStoreImpl")
1900
- );
1842
+ constructor(sthis, name, url, opts) {
1843
+ super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
1901
1844
  this.storeType = "data";
1902
1845
  }
1903
1846
  async load(cid) {
@@ -1938,32 +1881,23 @@ var DataStoreImpl = class extends BaseStoreImpl {
1938
1881
  return Result5.Ok(void 0);
1939
1882
  }
1940
1883
  destroy() {
1941
- this.logger.Debug().Msg("destroy");
1942
1884
  return this.gateway.destroy(this.url());
1943
1885
  }
1944
1886
  };
1945
1887
  var WALStoreImpl = class extends BaseStoreImpl {
1946
- constructor(sthis, url, opts) {
1947
- super(
1948
- sthis,
1949
- url,
1950
- {
1951
- ...opts
1952
- },
1953
- ensureLogger(sthis, "WALStoreImpl")
1954
- );
1888
+ constructor(loader, url, opts) {
1889
+ super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
1955
1890
  this.storeType = "wal";
1956
- // readonly tag: string = "rwal-base";
1957
- // readonly loader: Loadable;
1958
1891
  this._ready = new ResolveOnce3();
1959
1892
  this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
1960
1893
  this.processing = void 0;
1961
1894
  this.processQueue = new CommitQueue();
1895
+ this.loader = loader;
1962
1896
  }
1963
1897
  async ready() {
1964
1898
  return this._ready.once(async () => {
1965
1899
  const walState = await this.load().catch((e) => {
1966
- this.logger.Error().Err(e).Msg("error loading wal");
1900
+ this.logger.Error().Any("error", e).Msg("error loading wal");
1967
1901
  return void 0;
1968
1902
  });
1969
1903
  if (!walState) {
@@ -1996,7 +1930,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
1996
1930
  }
1997
1931
  async process() {
1998
1932
  await this.ready();
1999
- if (!this.loader?.remoteCarStore) return;
1933
+ if (!this.loader.remoteCarStore) return;
2000
1934
  await this.processQueue.enqueue(async () => {
2001
1935
  try {
2002
1936
  await this._doProcess();
@@ -2009,7 +1943,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
2009
1943
  });
2010
1944
  }
2011
1945
  async _doProcess() {
2012
- if (!this.loader) return;
2013
1946
  if (!this.loader.remoteCarStore) return;
2014
1947
  const operations = [...this.walState.operations];
2015
1948
  const noLoaderOps = [...this.walState.noLoaderOps];
@@ -2027,7 +1960,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
2027
1960
  noLoaderOps,
2028
1961
  async (dbMeta) => {
2029
1962
  await retryableUpload(async () => {
2030
- if (!this.loader) return;
2031
1963
  for (const cid of dbMeta.cars) {
2032
1964
  const car = await (await this.loader.carStore()).load(cid);
2033
1965
  if (!car) {
@@ -2047,7 +1979,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
2047
1979
  operations,
2048
1980
  async (dbMeta) => {
2049
1981
  await retryableUpload(async () => {
2050
- if (!this.loader) return;
2051
1982
  for (const cid of dbMeta.cars) {
2052
1983
  const car = await (await this.loader.carStore()).load(cid);
2053
1984
  if (!car) {
@@ -2067,7 +1998,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
2067
1998
  fileOperations,
2068
1999
  async ({ cid: fileCid, public: publicFile }) => {
2069
2000
  await retryableUpload(async () => {
2070
- if (!this.loader) return;
2071
2001
  const fileBlock = await (await this.loader.fileStore()).load(fileCid);
2072
2002
  if (!fileBlock) {
2073
2003
  throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
@@ -2081,7 +2011,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
2081
2011
  if (operations.length) {
2082
2012
  const lastOp = operations[operations.length - 1];
2083
2013
  await retryableUpload(async () => {
2084
- if (!this.loader) return;
2085
2014
  await this.loader.remoteMetaStore?.save(lastOp);
2086
2015
  }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
2087
2016
  }
@@ -2107,7 +2036,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
2107
2036
  }
2108
2037
  try {
2109
2038
  return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
2110
- return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
2111
2039
  } catch (e) {
2112
2040
  throw this.logger.Error().Err(e).Msg("error parse").AsError();
2113
2041
  }
@@ -2134,92 +2062,72 @@ var WALStoreImpl = class extends BaseStoreImpl {
2134
2062
  return Result5.Ok(void 0);
2135
2063
  }
2136
2064
  destroy() {
2137
- this.logger.Debug().Msg("destroy");
2138
2065
  return this.gateway.destroy(this.url());
2139
2066
  }
2140
2067
  };
2141
2068
 
2142
- // src/blockstore/register-store-protocol.ts
2143
- import { BuildURI, runtimeFn as runtimeFn2 } from "@adviser/cement";
2144
-
2145
- // src/runtime/gateways/memory/gateway.ts
2146
- import { Result as Result6 } from "@adviser/cement";
2147
-
2148
- // src/runtime/gateways/memory/version.ts
2149
- var MEMORY_VERSION = "v0.19-memory";
2150
-
2151
- // src/runtime/gateways/memory/gateway.ts
2152
- var MemoryGateway = class {
2153
- constructor(memorys) {
2154
- this.memorys = memorys;
2155
- }
2156
- buildUrl(baseUrl, key) {
2157
- return Promise.resolve(Result6.Ok(baseUrl.build().setParam("key" /* KEY */, key).URI()));
2158
- }
2159
- start(baseUrl) {
2160
- return Promise.resolve(Result6.Ok(baseUrl.build().setParam("version" /* VERSION */, MEMORY_VERSION).URI()));
2161
- }
2162
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
2163
- close(baseUrl) {
2164
- return Promise.resolve(Result6.Ok(void 0));
2165
- }
2166
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
2167
- destroy(baseUrl) {
2168
- this.memorys.clear();
2169
- return Promise.resolve(Result6.Ok(void 0));
2170
- }
2171
- put(url, body) {
2172
- this.memorys.set(url.toString(), body);
2173
- return Promise.resolve(Result6.Ok(void 0));
2174
- }
2175
- // get could return a NotFoundError if the key is not found
2176
- get(url) {
2177
- const x = this.memorys.get(url.toString());
2178
- if (x === void 0) {
2179
- return Promise.resolve(Result6.Err(new NotFoundError("not found")));
2180
- }
2181
- return Promise.resolve(Result6.Ok(x));
2182
- }
2183
- delete(url) {
2184
- this.memorys.delete(url.toString());
2185
- return Promise.resolve(Result6.Ok(void 0));
2186
- }
2187
- };
2188
- var MemoryTestGateway = class {
2189
- constructor(memorys) {
2190
- this.memorys = memorys;
2069
+ // src/blockstore/store-factory.ts
2070
+ function ensureIsIndex(url, isIndex) {
2071
+ if (isIndex) {
2072
+ return url.build().setParam("index", isIndex).URI();
2191
2073
  }
2192
- async get(url, key) {
2193
- return this.memorys.get(url.build().setParam("key" /* KEY */, key).toString());
2074
+ return url.build().delParam("index").URI();
2075
+ }
2076
+ function ensureName(name, url) {
2077
+ if (!url.hasParam("name")) {
2078
+ return url.build().setParam("name", name).URI();
2194
2079
  }
2195
- };
2196
-
2197
- // src/blockstore/register-store-protocol.ts
2080
+ return url;
2081
+ }
2198
2082
  var storeFactory = /* @__PURE__ */ new Map();
2199
- function getDefaultURI(sthis, protocol) {
2200
- if (protocol) {
2201
- if (!protocol.endsWith(":")) {
2202
- protocol += ":";
2203
- }
2204
- const gfi = storeFactory.get(protocol);
2205
- if (gfi) {
2206
- return gfi.defaultURI(sthis);
2083
+ function buildURL(optURL, loader) {
2084
+ const storeOpts = loader.ebOpts.store;
2085
+ const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
2086
+ let obuUrl;
2087
+ if (obuItem && obuItem.overrideBaseURL) {
2088
+ obuUrl = URI5.from(obuItem.overrideBaseURL);
2089
+ }
2090
+ const ret = ensureIsIndex(
2091
+ URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2092
+ storeOpts.isIndex
2093
+ );
2094
+ return ret;
2095
+ }
2096
+ var onceGateway = new KeyedResolvOnce2();
2097
+ async function getGatewayFromURL(url, sthis) {
2098
+ return onceGateway.get(url.toString()).once(async () => {
2099
+ const item = storeFactory.get(url.protocol);
2100
+ if (item) {
2101
+ const ret = {
2102
+ gateway: await item.gateway(sthis),
2103
+ test: await item.test(sthis)
2104
+ };
2105
+ const res = await ret.gateway.start(url);
2106
+ if (res.isErr()) {
2107
+ sthis.logger.Error().Result("start", res).Msg("start failed");
2108
+ return void 0;
2109
+ }
2110
+ return ret;
2207
2111
  }
2208
- }
2209
- const found = Array.from(storeFactory.values()).find((item) => item.isDefault);
2210
- if (!found) {
2211
- throw sthis.logger.Error().Msg("no default found").AsError();
2212
- }
2213
- return found.defaultURI(sthis);
2112
+ sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2113
+ return void 0;
2114
+ });
2214
2115
  }
2215
2116
  function registerStoreProtocol(item) {
2216
2117
  let protocol = item.protocol;
2217
2118
  if (!protocol.endsWith(":")) {
2218
2119
  protocol += ":";
2219
2120
  }
2220
- if (item.isDefault) {
2121
+ if (storeFactory.has(protocol)) {
2122
+ if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
2123
+ throw new Error(`we need a logger here`);
2124
+ return () => {
2125
+ };
2126
+ }
2127
+ }
2128
+ if (item.overrideBaseURL) {
2221
2129
  Array.from(storeFactory.values()).forEach((items) => {
2222
- items.isDefault = false;
2130
+ items.overrideBaseURL = void 0;
2223
2131
  });
2224
2132
  }
2225
2133
  storeFactory.set(protocol, item);
@@ -2227,183 +2135,134 @@ function registerStoreProtocol(item) {
2227
2135
  storeFactory.delete(protocol);
2228
2136
  };
2229
2137
  }
2230
- function getGatewayFactoryItem(protocol) {
2231
- return storeFactory.get(protocol);
2232
- }
2233
- function fileGatewayFactoryItem() {
2234
- return {
2235
- protocol: "file:",
2236
- isDefault: true,
2237
- defaultURI: (sthis) => {
2238
- return BuildURI.from("file://").pathname(`${sthis.env.get("HOME")}/.fireproof/${FILESTORE_VERSION.replace(/-.*$/, "")}`).URI();
2239
- },
2240
- gateway: async (sthis) => {
2241
- const { FileGateway } = await import("./gateway-GK5QZ6KP.js");
2242
- return new FileGateway(sthis);
2243
- },
2244
- test: async (sthis) => {
2245
- const { FileTestStore } = await import("./gateway-GK5QZ6KP.js");
2246
- return new FileTestStore(sthis);
2247
- }
2248
- };
2249
- }
2250
- if (runtimeFn2().isBrowser) {
2251
- registerStoreProtocol({
2252
- protocol: "indexdb:",
2253
- isDefault: true,
2254
- defaultURI: () => {
2255
- return BuildURI.from("indexdb://").pathname("fp").URI();
2256
- },
2257
- gateway: async (logger) => {
2258
- const { IndexDBGateway } = await import("./gateway-TQTGDRCN.js");
2259
- return new IndexDBGateway(logger);
2260
- },
2261
- test: async (logger) => {
2262
- const { IndexDBTestStore } = await import("./gateway-TQTGDRCN.js");
2263
- return new IndexDBTestStore(logger);
2264
- }
2265
- });
2266
- } else {
2267
- registerStoreProtocol(fileGatewayFactoryItem());
2268
- }
2269
- var memory = /* @__PURE__ */ new Map();
2270
- registerStoreProtocol({
2271
- protocol: "memory:",
2272
- isDefault: false,
2273
- defaultURI: () => {
2274
- return BuildURI.from("memory://").pathname("ram").URI();
2275
- },
2276
- gateway: async () => {
2277
- return new MemoryGateway(memory);
2278
- },
2279
- test: async () => {
2280
- return new MemoryTestGateway(memory);
2281
- }
2282
- });
2283
-
2284
- // src/blockstore/store-factory.ts
2285
- var onceGateway = new KeyedResolvOnce2();
2286
- var gatewayInstances = new KeyedResolvOnce2();
2287
- async function getStartedGateway(sthis, url) {
2288
- return onceGateway.get(url.toString()).once(async () => {
2289
- const item = getGatewayFactoryItem(url.protocol);
2290
- if (item) {
2291
- const ret = {
2292
- url,
2293
- ...await gatewayInstances.get(url.protocol).once(async () => ({
2294
- gateway: await item.gateway(sthis),
2295
- test: await item.test(sthis)
2296
- }))
2297
- };
2298
- const res = await ret.gateway.start(url);
2299
- if (res.isErr()) {
2300
- return Result7.Err(sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
2301
- }
2302
- ret.url = res.Ok();
2303
- return Result7.Ok(ret);
2138
+ var onceDataStoreFactory = new KeyedResolvOnce2();
2139
+ async function dataStoreFactory(loader) {
2140
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
2141
+ const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
2142
+ return onceDataStoreFactory.get(url.toString()).once(async () => {
2143
+ const gateway = await getGatewayFromURL(url, sthis);
2144
+ if (!gateway) {
2145
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2304
2146
  }
2305
- return Result7.Err(sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
2306
- });
2307
- }
2308
- async function dataStoreFactory(sfi) {
2309
- const storeUrl = sfi.url.build().setParam("store" /* STORE */, "data").URI();
2310
- const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2311
- if (rgateway.isErr()) {
2312
- throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2313
- }
2314
- const gateway = rgateway.Ok();
2315
- const store = new DataStoreImpl(sfi.sthis, gateway.url, {
2316
- gateway: gateway.gateway,
2317
- keybag: sfi.keybag
2147
+ const store = new DataStoreImpl(sthis, loader.name, url, {
2148
+ gateway: gateway.gateway,
2149
+ keybag: () => getKeyBag(loader.sthis, {
2150
+ ...loader.ebOpts.keyBag
2151
+ })
2152
+ });
2153
+ return store;
2318
2154
  });
2319
- return store;
2320
2155
  }
2321
- async function metaStoreFactory(sfi) {
2322
- const storeUrl = sfi.url.build().setParam("store" /* STORE */, "meta").URI();
2323
- const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2324
- if (rgateway.isErr()) {
2325
- throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2326
- }
2327
- const gateway = rgateway.Ok();
2328
- const store = new MetaStoreImpl(sfi.sthis, gateway.url, {
2329
- gateway: gateway.gateway,
2330
- keybag: sfi.keybag
2156
+ var onceMetaStoreFactory = new KeyedResolvOnce2();
2157
+ async function metaStoreFactory(loader) {
2158
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
2159
+ const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
2160
+ return onceMetaStoreFactory.get(url.toString()).once(async () => {
2161
+ sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
2162
+ const gateway = await getGatewayFromURL(url, sthis);
2163
+ if (!gateway) {
2164
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2165
+ }
2166
+ const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
2167
+ gateway: gateway.gateway,
2168
+ keybag: () => getKeyBag(loader.sthis, {
2169
+ ...loader.ebOpts.keyBag
2170
+ })
2171
+ });
2172
+ return store;
2331
2173
  });
2332
- return store;
2333
2174
  }
2334
- async function WALStoreFactory(sfi) {
2335
- const storeUrl = sfi.url.build().setParam("store" /* STORE */, "wal").URI();
2336
- const rgateway = await getStartedGateway(sfi.sthis, storeUrl);
2337
- if (rgateway.isErr()) {
2338
- throw sfi.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
2339
- }
2340
- const gateway = rgateway.Ok();
2341
- const store = new WALStoreImpl(sfi.sthis, gateway.url, {
2342
- gateway: gateway.gateway,
2343
- keybag: sfi.keybag
2175
+ var onceRemoteWalFactory = new KeyedResolvOnce2();
2176
+ async function remoteWalFactory(loader) {
2177
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
2178
+ const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
2179
+ return onceRemoteWalFactory.get(url.toString()).once(async () => {
2180
+ const gateway = await getGatewayFromURL(url, sthis);
2181
+ if (!gateway) {
2182
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2183
+ }
2184
+ sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
2185
+ const store = new WALStoreImpl(loader, url, {
2186
+ gateway: gateway.gateway,
2187
+ keybag: () => getKeyBag(loader.sthis, {
2188
+ ...loader.ebOpts.keyBag
2189
+ })
2190
+ });
2191
+ return store;
2344
2192
  });
2345
- return store;
2346
2193
  }
2347
2194
  async function testStoreFactory(url, sthis) {
2348
- const rgateway = await getStartedGateway(sthis, url);
2349
- if (!rgateway) {
2195
+ sthis = ensureSuperLog(sthis, "testStoreFactory");
2196
+ const gateway = await getGatewayFromURL(url, sthis);
2197
+ if (!gateway) {
2350
2198
  throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2351
2199
  }
2352
- return rgateway.Ok().test;
2200
+ return gateway.test;
2353
2201
  }
2354
- async function ensureStart(store) {
2202
+ async function ensureStart(store, logger) {
2355
2203
  const ret = await store.start();
2356
2204
  if (ret.isErr()) {
2357
- throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
2205
+ throw logger.Error().Result("start", ret).Msg("start failed").AsError();
2358
2206
  }
2359
- store.logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2207
+ logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2360
2208
  return store;
2361
2209
  }
2362
- function ensureStoreEnDeFile(ende) {
2363
- ende = ende || {};
2364
- return {
2365
- encodeFile: ende.encodeFile || encodeFile,
2366
- decodeFile: ende.decodeFile || decodeFile
2367
- };
2368
- }
2369
- function toStoreRuntime(sthis, endeOpts = {}) {
2210
+ function toStoreRuntime(opts, sthis) {
2211
+ const logger = ensureLogger(sthis, "toStoreRuntime", {});
2370
2212
  return {
2371
- makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
2372
- // async (loader: Loadable) => {
2373
- // logger
2374
- // .Debug()
2375
- // .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
2376
- // .Msg("makeMetaStore");
2377
- // return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
2378
- // },
2379
- makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
2380
- // async (loader: Loadable) => {
2381
- // logger
2382
- // .Debug()
2383
- // .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
2384
- // .Msg("makeDataStore");
2385
- // return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
2386
- // },
2387
- makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
2388
- // async (loader: Loadable) => {
2389
- // logger
2390
- // .Debug()
2391
- // .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
2392
- // .Msg("makeRemoteWAL");
2393
- // return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
2394
- // },
2395
- ...ensureStoreEnDeFile(endeOpts)
2213
+ makeMetaStore: async (loader) => {
2214
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
2215
+ return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
2216
+ },
2217
+ makeDataStore: async (loader) => {
2218
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
2219
+ return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
2220
+ },
2221
+ makeWALStore: async (loader) => {
2222
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
2223
+ return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
2224
+ },
2225
+ encodeFile: opts.encodeFile || encodeFile,
2226
+ decodeFile: opts.decodeFile || decodeFile
2396
2227
  };
2397
2228
  }
2229
+ registerStoreProtocol({
2230
+ protocol: "file:",
2231
+ gateway: async (sthis) => {
2232
+ const { FileGateway } = await import("./gateway-5FCWPX5W.js");
2233
+ return new FileGateway(sthis);
2234
+ },
2235
+ test: async (sthis) => {
2236
+ const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
2237
+ return new FileTestStore(sthis);
2238
+ }
2239
+ });
2240
+ registerStoreProtocol({
2241
+ protocol: "indexdb:",
2242
+ gateway: async (sthis) => {
2243
+ const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
2244
+ return new IndexDBGateway(sthis);
2245
+ },
2246
+ test: async (sthis) => {
2247
+ const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
2248
+ return new IndexDBTestStore(sthis);
2249
+ }
2250
+ });
2398
2251
 
2399
2252
  // src/blockstore/store-remote.ts
2400
2253
  async function RemoteDataStore(sthis, name, url, opts) {
2401
- const ds = new DataStoreImpl(sthis, url, opts);
2254
+ const ds = new DataStoreImpl(sthis, name, url, opts);
2402
2255
  await ds.start();
2403
2256
  return ds;
2404
2257
  }
2405
2258
  async function RemoteMetaStore(sthis, name, url, opts) {
2406
- const ms = new MetaStoreImpl(sthis, url, opts);
2259
+ const ms = new MetaStoreImpl(
2260
+ sthis,
2261
+ name,
2262
+ url,
2263
+ opts
2264
+ /* , true*/
2265
+ );
2407
2266
  await ms.start();
2408
2267
  return ms;
2409
2268
  }
@@ -2428,17 +2287,14 @@ var ConnectionBase = class {
2428
2287
  if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
2429
2288
  this.loader = loader;
2430
2289
  await this.onConnect();
2431
- const metaUrl = this.url.build().defParam("store" /* STORE */, "meta").URI();
2432
- const rgateway = await getStartedGateway(loader.sthis, metaUrl);
2433
- if (rgateway.isErr())
2434
- throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2435
- const name = metaUrl.toString();
2290
+ const metaUrl = this.url.build().defParam("store", "meta").URI();
2291
+ const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
2292
+ if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2436
2293
  const dbName = metaUrl.getParam("name");
2437
- if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: dbName is required").AsError();
2438
- const gateway = rgateway.Ok();
2439
- const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
2294
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
2295
+ const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
2440
2296
  gateway: gateway.gateway,
2441
- keybag: await loader.keyBag(),
2297
+ keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
2442
2298
  loader
2443
2299
  });
2444
2300
  this.loader.remoteMetaStore = remote;
@@ -2451,15 +2307,14 @@ var ConnectionBase = class {
2451
2307
  async connectStorage_X({ loader }) {
2452
2308
  if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
2453
2309
  this.loader = loader;
2454
- const dataUrl = this.url.build().defParam("store" /* STORE */, "data").URI();
2455
- const rgateway = await getStartedGateway(loader.sthis, dataUrl);
2456
- if (rgateway.isErr())
2457
- throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2310
+ const dataUrl = this.url.build().defParam("store", "data").URI();
2311
+ const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
2312
+ if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2458
2313
  const name = dataUrl.getParam("name");
2459
2314
  if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
2460
2315
  loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
2461
- gateway: rgateway.Ok().gateway,
2462
- keybag: await loader.keyBag()
2316
+ gateway: gateway.gateway,
2317
+ keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
2463
2318
  });
2464
2319
  loader.remoteFileStore = loader.remoteCarStore;
2465
2320
  }
@@ -2499,12 +2354,6 @@ var ConnectionBase = class {
2499
2354
  };
2500
2355
 
2501
2356
  // src/crdt-helpers.ts
2502
- import { parse as parse3 } from "multiformats/link";
2503
- import { sha256 as hasher5 } from "multiformats/hashes/sha2";
2504
- import * as codec from "@ipld/dag-cbor";
2505
- import { put, get, entries, root } from "@web3-storage/pail/crdt";
2506
- import { EventFetcher, vis } from "@web3-storage/pail/clock";
2507
- import * as Batch from "@web3-storage/pail/crdt/batch";
2508
2357
  function time(tag) {
2509
2358
  }
2510
2359
  function timeEnd(tag) {
@@ -2552,7 +2401,7 @@ async function writeDocContent(store, blocks, update, logger) {
2552
2401
  await processFiles(store, blocks, update.value, logger);
2553
2402
  value = { doc: update.value };
2554
2403
  }
2555
- const block = await encode3({ value, hasher: hasher5, codec });
2404
+ const block = await encode({ value, hasher: hasher5, codec });
2556
2405
  blocks.putSync(block.cid, block.bytes);
2557
2406
  return block.cid;
2558
2407
  }
@@ -2643,7 +2492,7 @@ function readFileset(blocks, files, isPublic = false) {
2643
2492
  async function getValueFromLink(blocks, link, logger) {
2644
2493
  const block = await blocks.get(link);
2645
2494
  if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
2646
- const { value } = await decode3({ bytes: block.bytes, hasher: hasher5, codec });
2495
+ const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
2647
2496
  const cvalue = {
2648
2497
  ...value,
2649
2498
  cid: link
@@ -2764,7 +2613,7 @@ async function doCompact(blockLog, head, logger) {
2764
2613
  async function getBlock(blocks, cidString) {
2765
2614
  const block = await blocks.get(parse3(cidString));
2766
2615
  if (!block) throw new Error(`Missing block ${cidString}`);
2767
- const { cid, value } = await decode3({ bytes: block.bytes, codec, hasher: hasher5 });
2616
+ const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
2768
2617
  return new Block({ cid, value, bytes: block.bytes });
2769
2618
  }
2770
2619
 
@@ -2822,8 +2671,7 @@ function makeProllyGetBlock(blocks) {
2822
2671
  return create({ cid, bytes, hasher: hasher6, codec: codec2 });
2823
2672
  };
2824
2673
  }
2825
- async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
2826
- logger.Debug().Msg("enter bulkIndex");
2674
+ async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
2827
2675
  if (!indexEntries.length) return inIndex;
2828
2676
  if (!inIndex.root) {
2829
2677
  if (!inIndex.cid) {
@@ -2840,22 +2688,18 @@ async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
2840
2688
  returnNode = node;
2841
2689
  }
2842
2690
  if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
2843
- logger.Debug().Msg("exit !root bulkIndex");
2844
2691
  return { root: returnNode, cid: returnRootBlock.cid };
2845
2692
  } else {
2846
2693
  inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
2847
2694
  }
2848
2695
  }
2849
- logger.Debug().Msg("pre bulk bulkIndex");
2850
2696
  const { root: root3, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
2851
2697
  if (root3) {
2852
- logger.Debug().Msg("pre root put bulkIndex");
2853
2698
  for await (const block of newBlocks) {
2854
2699
  await tblocks.put(block.cid, block.bytes);
2855
2700
  }
2856
2701
  return { root: root3, cid: (await root3.block).cid };
2857
2702
  } else {
2858
- logger.Debug().Msg("pre !root bulkIndex");
2859
2703
  return { root: void 0, cid: void 0 };
2860
2704
  }
2861
2705
  }
@@ -2895,17 +2739,17 @@ function encodeKey(key) {
2895
2739
  }
2896
2740
 
2897
2741
  // src/indexer.ts
2898
- function index(refDb, name, mapFn, meta) {
2899
- if (mapFn && meta) throw refDb.crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2900
- if (mapFn && mapFn.constructor.name !== "Function") throw refDb.crdt.logger.Error().Msg("mapFn must be a function").AsError();
2901
- if (refDb.crdt.indexers.has(name)) {
2902
- const idx = refDb.crdt.indexers.get(name);
2742
+ function index(sthis, { _crdt }, name, mapFn, meta) {
2743
+ if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2744
+ if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
2745
+ if (_crdt.indexers.has(name)) {
2746
+ const idx = _crdt.indexers.get(name);
2903
2747
  idx.applyMapFn(name, mapFn, meta);
2904
2748
  } else {
2905
- const idx = new Index(refDb.crdt.sthis, refDb.crdt, name, mapFn, meta);
2906
- refDb.crdt.indexers.set(name, idx);
2749
+ const idx = new Index(sthis, _crdt, name, mapFn, meta);
2750
+ _crdt.indexers.set(name, idx);
2907
2751
  }
2908
- return refDb.crdt.indexers.get(name);
2752
+ return _crdt.indexers.get(name);
2909
2753
  }
2910
2754
  var Index = class {
2911
2755
  constructor(sthis, crdt, name, mapFn, meta) {
@@ -2924,9 +2768,18 @@ var Index = class {
2924
2768
  return Promise.all([this.blockstore.ready(), this.crdt.ready()]).then(() => {
2925
2769
  });
2926
2770
  }
2771
+ close() {
2772
+ return Promise.all([this.blockstore.close(), this.crdt.close()]).then(() => {
2773
+ });
2774
+ }
2775
+ destroy() {
2776
+ return Promise.all([this.blockstore.destroy(), this.crdt.destroy()]).then(() => {
2777
+ });
2778
+ }
2927
2779
  applyMapFn(name, mapFn, meta) {
2928
2780
  if (mapFn && meta) throw this.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2929
2781
  if (this.name && this.name !== name) throw this.logger.Error().Msg("cannot change name").AsError();
2782
+ this.name = name;
2930
2783
  try {
2931
2784
  if (meta) {
2932
2785
  if (this.indexHead && this.indexHead.map((c) => c.toString()).join() !== meta.head.map((c) => c.toString()).join()) {
@@ -2974,13 +2827,9 @@ var Index = class {
2974
2827
  }
2975
2828
  }
2976
2829
  async query(opts = {}) {
2977
- this.logger.Debug().Msg("enter query");
2978
2830
  await this.ready();
2979
- this.logger.Debug().Msg("post ready query");
2980
2831
  await this._updateIndex();
2981
- this.logger.Debug().Msg("post _updateIndex query");
2982
2832
  await this._hydrateIndex();
2983
- this.logger.Debug().Msg("post _hydrateIndex query");
2984
2833
  if (!this.byKey.root) {
2985
2834
  return await applyQuery(this.crdt, { result: [] }, opts);
2986
2835
  }
@@ -3036,16 +2885,13 @@ var Index = class {
3036
2885
  }
3037
2886
  async _updateIndex() {
3038
2887
  await this.ready();
3039
- this.logger.Debug().Msg("enter _updateIndex");
3040
2888
  if (this.initError) throw this.initError;
3041
2889
  if (!this.mapFn) throw this.logger.Error().Msg("No map function defined").AsError();
3042
2890
  let result, head;
3043
2891
  if (!this.indexHead || this.indexHead.length === 0) {
3044
2892
  ({ result, head } = await this.crdt.allDocs());
3045
- this.logger.Debug().Msg("enter crdt.allDocs");
3046
2893
  } else {
3047
2894
  ({ result, head } = await this.crdt.changes(this.indexHead));
3048
- this.logger.Debug().Msg("enter crdt.changes");
3049
2895
  }
3050
2896
  if (result.length === 0) {
3051
2897
  this.indexHead = head;
@@ -3078,22 +2924,9 @@ var Index = class {
3078
2924
  if (result.length === 0) {
3079
2925
  return indexerMeta;
3080
2926
  }
3081
- this.logger.Debug().Msg("pre this.blockstore.transaction");
3082
2927
  const { meta } = await this.blockstore.transaction(async (tblocks) => {
3083
- this.byId = await bulkIndex(
3084
- this.logger,
3085
- tblocks,
3086
- this.byId,
3087
- removeIdIndexEntries.concat(byIdIndexEntries),
3088
- byIdOpts
3089
- );
3090
- this.byKey = await bulkIndex(
3091
- this.logger,
3092
- tblocks,
3093
- this.byKey,
3094
- staleKeyIndexEntries.concat(indexEntries),
3095
- byKeyOpts
3096
- );
2928
+ this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
2929
+ this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
3097
2930
  this.indexHead = head;
3098
2931
  if (this.byId.cid && this.byKey.cid) {
3099
2932
  const idxMeta = {
@@ -3105,10 +2938,8 @@ var Index = class {
3105
2938
  };
3106
2939
  indexerMeta.indexes?.set(this.name, idxMeta);
3107
2940
  }
3108
- this.logger.Debug().Any("indexerMeta", new Array(indexerMeta.indexes?.entries())).Msg("exit this.blockstore.transaction fn");
3109
2941
  return indexerMeta;
3110
2942
  });
3111
- this.logger.Debug().Msg("post this.blockstore.transaction");
3112
2943
  return meta;
3113
2944
  }
3114
2945
  };
@@ -3284,13 +3115,15 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
3284
3115
 
3285
3116
  // src/crdt.ts
3286
3117
  var CRDT = class {
3287
- constructor(sthis, opts) {
3118
+ constructor(sthis, name, opts = {}) {
3288
3119
  this.indexers = /* @__PURE__ */ new Map();
3289
3120
  this.onceReady = new ResolveOnce5();
3290
3121
  this.sthis = sthis;
3122
+ this.name = name;
3291
3123
  this.logger = ensureLogger(sthis, "CRDT");
3292
3124
  this.opts = opts;
3293
3125
  this.blockstore = blockstoreFactory(sthis, {
3126
+ name,
3294
3127
  applyMeta: async (meta) => {
3295
3128
  const crdtMeta = meta;
3296
3129
  if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
@@ -3300,27 +3133,23 @@ var CRDT = class {
3300
3133
  await doCompact(blocks, this.clock.head, this.logger);
3301
3134
  return { head: this.clock.head };
3302
3135
  },
3303
- // autoCompact: this.opts.autoCompact || 100,
3304
- storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
3305
- storeUrls: this.opts.storeUrls.data,
3306
- keyBag: this.opts.keyBag,
3307
- // public: this.opts.public,
3308
- meta: this.opts.meta
3309
- // threshold: this.opts.threshold,
3136
+ autoCompact: this.opts.autoCompact || 100,
3137
+ store: { ...this.opts.store, isIndex: void 0 },
3138
+ public: this.opts.public,
3139
+ meta: this.opts.meta,
3140
+ threshold: this.opts.threshold
3310
3141
  });
3311
3142
  this.indexBlockstore = blockstoreFactory(sthis, {
3312
- // name: opts.name,
3143
+ name,
3313
3144
  applyMeta: async (meta) => {
3314
3145
  const idxCarMeta = meta;
3315
3146
  if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
3316
- for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
3317
- index({ crdt: this }, name, void 0, idx);
3147
+ for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
3148
+ index(this.sthis, { _crdt: this }, name2, void 0, idx);
3318
3149
  }
3319
3150
  },
3320
- storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
3321
- storeUrls: this.opts.storeUrls.idx,
3322
- keyBag: this.opts.keyBag
3323
- // public: this.opts.public,
3151
+ store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
3152
+ public: this.opts.public
3324
3153
  });
3325
3154
  this.clock = new CRDTClock(this.blockstore);
3326
3155
  this.clock.onZoom(() => {
@@ -3402,164 +3231,51 @@ var CRDT = class {
3402
3231
  };
3403
3232
 
3404
3233
  // src/database.ts
3405
- var databases = new KeyedResolvOnce3();
3406
- function toSortedArray(set) {
3407
- if (!set) return [];
3408
- return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
3409
- }
3410
- function keyConfigOpts(sthis, name, opts) {
3411
- return JSON.stringify(
3412
- toSortedArray({
3413
- name,
3414
- stores: toSortedArray(JSON.parse(JSON.stringify(toStoreURIRuntime(sthis, name, opts?.storeUrls))))
3415
- })
3416
- );
3417
- }
3418
- function isDatabase(db) {
3419
- return db instanceof DatabaseImpl || db instanceof DatabaseShell;
3420
- }
3421
- function DatabaseFactory(name, opts) {
3422
- const sthis = ensureSuperThis(opts);
3423
- return new DatabaseShell(
3424
- databases.get(keyConfigOpts(sthis, name, opts)).once((key) => {
3425
- const db = new DatabaseImpl(sthis, {
3426
- name,
3427
- meta: opts?.meta,
3428
- keyBag: defaultKeyBagOpts(sthis, opts?.keyBag),
3429
- storeUrls: toStoreURIRuntime(sthis, name, opts?.storeUrls),
3430
- storeEnDe: {
3431
- encodeFile,
3432
- decodeFile,
3433
- ...opts?.storeEnDe
3434
- }
3435
- });
3436
- db.onClosed(() => {
3437
- databases.unget(key);
3438
- });
3439
- return db;
3440
- })
3441
- );
3442
- }
3443
- var DatabaseShell = class {
3444
- constructor(ref) {
3445
- this.ref = ref;
3446
- ref.addShell(this);
3447
- }
3448
- get id() {
3449
- return this.ref.id;
3450
- }
3451
- get logger() {
3452
- return this.ref.logger;
3453
- }
3454
- get sthis() {
3455
- return this.ref.sthis;
3456
- }
3457
- get crdt() {
3458
- return this.ref.crdt;
3459
- }
3460
- name() {
3461
- return this.ref.name();
3462
- }
3463
- onClosed(fn) {
3464
- return this.ref.onClosed(fn);
3465
- }
3466
- close() {
3467
- return this.ref.shellClose(this);
3468
- }
3469
- destroy() {
3470
- return this.ref.destroy();
3471
- }
3472
- ready() {
3473
- return this.ref.ready();
3474
- }
3475
- get(id) {
3476
- return this.ref.get(id);
3477
- }
3478
- put(doc) {
3479
- return this.ref.put(doc);
3480
- }
3481
- del(id) {
3482
- return this.ref.del(id);
3483
- }
3484
- changes(since, opts) {
3485
- return this.ref.changes(since, opts);
3486
- }
3487
- allDocs(opts) {
3488
- return this.ref.allDocs(opts);
3489
- }
3490
- allDocuments() {
3491
- return this.ref.allDocuments();
3492
- }
3493
- subscribe(listener, updates) {
3494
- return this.ref.subscribe(listener, updates);
3495
- }
3496
- query(field, opts) {
3497
- return this.ref.query(field, opts);
3498
- }
3499
- compact() {
3500
- return this.ref.compact();
3501
- }
3502
- };
3503
- var DatabaseImpl = class {
3504
- constructor(sthis, opts) {
3234
+ var Database = class {
3235
+ constructor(name, opts) {
3236
+ this.opts = {};
3505
3237
  this._listening = false;
3506
3238
  this._listeners = /* @__PURE__ */ new Set();
3507
3239
  this._noupdate_listeners = /* @__PURE__ */ new Set();
3508
- // readonly blockstore: BaseBlockstore;
3509
- this.shells = /* @__PURE__ */ new Set();
3510
- this._onClosedFns = /* @__PURE__ */ new Set();
3511
3240
  this._ready = new ResolveOnce6();
3512
- this.opts = opts;
3513
- this.sthis = sthis;
3514
- this.id = sthis.timeOrderedNextId().str;
3241
+ this.name = name;
3242
+ this.opts = opts || this.opts;
3243
+ this.sthis = ensureSuperThis(this.opts);
3515
3244
  this.logger = ensureLogger(this.sthis, "Database");
3516
- this.crdt = new CRDT(this.sthis, this.opts);
3245
+ this._crdt = new CRDT(this.sthis, name, this.opts);
3246
+ this.blockstore = this._crdt.blockstore;
3517
3247
  this._writeQueue = writeQueue(async (updates) => {
3518
- return await this.crdt.bulk(updates);
3248
+ return await this._crdt.bulk(updates);
3519
3249
  });
3520
- this.crdt.clock.onTock(() => {
3250
+ this._crdt.clock.onTock(() => {
3521
3251
  this._no_update_notify();
3522
3252
  });
3523
3253
  }
3524
- addShell(shell) {
3525
- this.shells.add(shell);
3526
- }
3527
- onClosed(fn) {
3528
- this._onClosedFns.add(fn);
3254
+ static {
3255
+ this.databases = /* @__PURE__ */ new Map();
3529
3256
  }
3530
3257
  async close() {
3531
- throw this.logger.Error().Str("db", this.name()).Msg(`use shellClose`).AsError();
3532
- }
3533
- async shellClose(db) {
3534
- if (!this.shells.has(db)) {
3535
- throw this.logger.Error().Str("db", this.name()).Msg(`Database Shell mismatch`).AsError();
3536
- }
3537
- this.shells.delete(db);
3538
- if (this.shells.size === 0) {
3539
- await this.ready();
3540
- await this.crdt.close();
3541
- this._onClosedFns.forEach((fn) => fn());
3542
- }
3258
+ await this.ready();
3259
+ await this._crdt.close();
3260
+ await this.blockstore.close();
3543
3261
  }
3544
3262
  async destroy() {
3545
3263
  await this.ready();
3546
- await this.crdt.destroy();
3264
+ await this._crdt.destroy();
3265
+ await this.blockstore.destroy();
3547
3266
  }
3548
3267
  async ready() {
3549
- const ret = await this._ready.once(async () => {
3268
+ return this._ready.once(async () => {
3550
3269
  await this.sthis.start();
3551
- await this.crdt.ready();
3270
+ await this._crdt.ready();
3271
+ await this.blockstore.ready();
3552
3272
  });
3553
- return ret;
3554
- }
3555
- name() {
3556
- return this.opts.storeUrls.data.data.getParam("name" /* NAME */) || "default";
3557
3273
  }
3558
3274
  async get(id) {
3559
- if (!id) throw this.logger.Error().Str("db", this.name()).Msg(`Doc id is required`).AsError();
3275
+ if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
3560
3276
  await this.ready();
3561
3277
  this.logger.Debug().Str("id", id).Msg("get");
3562
- const got = await this.crdt.get(id).catch((e) => {
3278
+ const got = await this._crdt.get(id).catch((e) => {
3563
3279
  throw new NotFoundError(`Not found: ${id} - ${e.message}`);
3564
3280
  });
3565
3281
  if (!got) throw new NotFoundError(`Not found: ${id}`);
@@ -3578,34 +3294,34 @@ var DatabaseImpl = class {
3578
3294
  _id: docId
3579
3295
  }
3580
3296
  });
3581
- return { id: docId, clock: result?.head, name: this.name() };
3297
+ return { id: docId, clock: result?.head, name: this.name };
3582
3298
  }
3583
3299
  async del(id) {
3584
3300
  await this.ready();
3585
3301
  this.logger.Debug().Str("id", id).Msg("del");
3586
3302
  const result = await this._writeQueue.push({ id, del: true });
3587
- return { id, clock: result?.head, name: this.name() };
3303
+ return { id, clock: result?.head, name: this.name };
3588
3304
  }
3589
3305
  async changes(since = [], opts = {}) {
3590
3306
  await this.ready();
3591
3307
  this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
3592
- const { result, head } = await this.crdt.changes(since, opts);
3308
+ const { result, head } = await this._crdt.changes(since, opts);
3593
3309
  const rows = result.map(({ id: key, value, del, clock }) => ({
3594
3310
  key,
3595
3311
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value },
3596
3312
  clock
3597
3313
  }));
3598
- return { rows, clock: head, name: this.name() };
3314
+ return { rows, clock: head, name: this.name };
3599
3315
  }
3600
3316
  async allDocs(opts = {}) {
3601
3317
  await this.ready();
3602
3318
  this.logger.Debug().Msg("allDocs");
3603
- const { result, head } = await this.crdt.allDocs();
3319
+ const { result, head } = await this._crdt.allDocs();
3604
3320
  const rows = result.map(({ id: key, value, del }) => ({
3605
3321
  key,
3606
3322
  value: del ? { _id: key, _deleted: true } : { _id: key, ...value }
3607
3323
  }));
3608
- return { rows, clock: head, name: this.name() };
3324
+ return { rows, clock: head, name: this.name };
3609
3325
  }
3610
3326
  async allDocuments() {
3611
3327
  return this.allDocs();
@@ -3615,7 +3331,7 @@ var DatabaseImpl = class {
3615
3331
  if (updates) {
3616
3332
  if (!this._listening) {
3617
3333
  this._listening = true;
3618
- this.crdt.clock.onTick((updates2) => {
3334
+ this._crdt.clock.onTick((updates2) => {
3619
3335
  void this._notify(updates2);
3620
3336
  });
3621
3337
  }
@@ -3634,13 +3350,13 @@ var DatabaseImpl = class {
3634
3350
  async query(field, opts = {}) {
3635
3351
  await this.ready();
3636
3352
  this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
3637
- const _crdt = this.crdt;
3638
- const idx = typeof field === "string" ? index({ crdt: _crdt }, field) : index({ crdt: _crdt }, makeName(field.toString()), field);
3353
+ const _crdt = this._crdt;
3354
+ const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
3639
3355
  return await idx.query(opts);
3640
3356
  }
3641
3357
  async compact() {
3642
3358
  await this.ready();
3643
- await this.crdt.compact();
3359
+ await this._crdt.compact();
3644
3360
  }
3645
3361
  async _notify(updates) {
3646
3362
  await this.ready();
@@ -3664,62 +3380,23 @@ var DatabaseImpl = class {
3664
3380
  }
3665
3381
  }
3666
3382
  };
3667
- function defaultURI(sthis, curi, uri, store, ctx) {
3668
- ctx = ctx || {};
3669
- const ret = (curi ? URI8.from(curi) : uri).build().setParam("store" /* STORE */, store);
3670
- if (!ret.hasParam("name" /* NAME */)) {
3671
- const name = sthis.pathOps.basename(ret.URI().pathname);
3672
- if (!name) {
3673
- throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Database name is required").AsError();
3674
- }
3675
- ret.setParam("name" /* NAME */, name);
3676
- }
3677
- if (ctx.idx) {
3678
- ret.defParam("index" /* INDEX */, "idx");
3679
- ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}-idx@`);
3680
- } else {
3681
- ret.defParam("storekey" /* STORE_KEY */, `@${ret.getParam("name" /* NAME */)}-${store}@`);
3682
- }
3683
- if (store === "data") {
3684
- if (ctx.file) {
3685
- } else {
3686
- ret.defParam("suffix" /* SUFFIX */, ".car");
3687
- }
3688
- }
3689
- return ret.URI();
3690
- }
3691
- function toStoreURIRuntime(sthis, name, sopts) {
3692
- sopts = sopts || {};
3693
- if (!sopts.base) {
3694
- const fp_env = sthis.env.get("FP_STORAGE_URL");
3695
- if (fp_env) {
3696
- sopts = { ...sopts, base: BuildURI2.from(fp_env).setParam("urlGen" /* URL_GEN */, "fromEnv") };
3697
- } else {
3698
- sopts = { ...sopts, base: getDefaultURI(sthis).build().setParam("urlGen" /* URL_GEN */, "default") };
3699
- }
3700
- }
3701
- const bbase = BuildURI2.from(sopts.base);
3702
- if (name) {
3703
- bbase.setParam("name" /* NAME */, name);
3704
- }
3705
- const base = bbase.URI();
3706
- return {
3707
- idx: {
3708
- data: defaultURI(sthis, sopts.idx?.data, base, "data", { idx: true }),
3709
- file: defaultURI(sthis, sopts.idx?.data, base, "data", { file: true, idx: true }),
3710
- meta: defaultURI(sthis, sopts.idx?.meta, base, "meta", { idx: true }),
3711
- wal: defaultURI(sthis, sopts.idx?.wal, base, "wal", { idx: true })
3712
- },
3713
- data: {
3714
- data: defaultURI(sthis, sopts.data?.data, base, "data"),
3715
- file: defaultURI(sthis, sopts.data?.data, base, "data", { file: true }),
3716
- meta: defaultURI(sthis, sopts.data?.meta, base, "meta"),
3717
- wal: defaultURI(sthis, sopts.data?.wal, base, "wal")
3718
- }
3719
- };
3383
+ function toSortedArray(set) {
3384
+ if (!set) return [];
3385
+ return Object.entries(set).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => ({ [k]: v }));
3720
3386
  }
3721
3387
  function fireproof(name, opts) {
3722
- return DatabaseFactory(name, opts);
3388
+ const key = JSON.stringify(
3389
+ toSortedArray({
3390
+ name,
3391
+ stores: toSortedArray(opts?.store?.stores)
3392
+ })
3393
+ );
3394
+ let db = Database.databases.get(key);
3395
+ if (!db) {
3396
+ db = new Database(name, opts);
3397
+ Database.databases.set(key, db);
3398
+ }
3399
+ return db;
3723
3400
  }
3724
3401
  function makeName(fnString) {
3725
3402
  const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g;
@@ -3750,7 +3427,7 @@ __export(runtime_exports, {
3750
3427
  kb: () => key_bag_exports,
3751
3428
  kc: () => keyed_crypto_exports,
3752
3429
  mf: () => wait_pr_multiformats_exports,
3753
- runtimeFn: () => runtimeFn3,
3430
+ runtimeFn: () => runtimeFn2,
3754
3431
  toArrayBuffer: () => toArrayBuffer
3755
3432
  });
3756
3433
 
@@ -3765,24 +3442,23 @@ __export(wait_pr_multiformats_exports, {
3765
3442
  var codec_interface_exports = {};
3766
3443
 
3767
3444
  // src/runtime/index.ts
3768
- import { runtimeFn as runtimeFn3 } from "@adviser/cement";
3445
+ import { runtimeFn as runtimeFn2 } from "@adviser/cement";
3769
3446
 
3770
3447
  // src/version.ts
3771
3448
  var PACKAGE_VERSION = Object.keys({
3772
- "0.19.101": "xxxx"
3449
+ "0.19.103": "xxxx"
3773
3450
  })[0];
3774
3451
  export {
3775
3452
  CRDT,
3776
- DatabaseFactory,
3777
- DatabaseShell,
3453
+ Database,
3778
3454
  Index,
3779
3455
  NotFoundError,
3780
3456
  PACKAGE_VERSION,
3781
- PARAM,
3782
3457
  Result,
3783
3458
  UInt8ArrayEqual,
3784
3459
  blockstore_exports as blockstore,
3785
3460
  blockstore_exports as bs,
3461
+ dataDir,
3786
3462
  ensureLogger,
3787
3463
  ensureSuperLog,
3788
3464
  ensureSuperThis,
@@ -3793,13 +3469,10 @@ export {
3793
3469
  getName,
3794
3470
  getStore,
3795
3471
  index,
3796
- isDatabase,
3797
3472
  isFalsy,
3798
3473
  isNotFoundError,
3799
- keyConfigOpts,
3800
3474
  runtime_exports as rt,
3801
3475
  runtime_exports as runtime,
3802
- throwFalsy,
3803
- toStoreURIRuntime
3476
+ throwFalsy
3804
3477
  };
3805
3478
  //# sourceMappingURL=index.js.map