@fireproof/core 0.19.8-dev-global → 0.19.11-dev-dryrun

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +34 -0
  2. package/chunk-7EWIAXTM.js +7 -0
  3. package/chunk-7EWIAXTM.js.map +1 -0
  4. package/chunk-MAK4D54P.js +280 -0
  5. package/chunk-MAK4D54P.js.map +1 -0
  6. package/chunk-PB4BKL4O.js +7 -0
  7. package/chunk-PB4BKL4O.js.map +1 -0
  8. package/chunk-XINRLWR3.js +75 -0
  9. package/chunk-XINRLWR3.js.map +1 -0
  10. package/deno.json +20 -0
  11. package/{store-indexdb-WLRSICCB.js → gateway-7OM6OSYK.js} +49 -82
  12. package/gateway-7OM6OSYK.js.map +1 -0
  13. package/gateway-VWWKLHUI.js +144 -0
  14. package/gateway-VWWKLHUI.js.map +1 -0
  15. package/index.cjs +2351 -1825
  16. package/index.cjs.map +1 -1
  17. package/index.d.cts +659 -535
  18. package/index.d.ts +659 -535
  19. package/index.global.js +26315 -20709
  20. package/index.global.js.map +1 -1
  21. package/index.js +1700 -1058
  22. package/index.js.map +1 -1
  23. package/key-bag-file-DFMW6ZM6.js +54 -0
  24. package/key-bag-file-DFMW6ZM6.js.map +1 -0
  25. package/key-bag-indexdb-R2RWGSQ4.js +50 -0
  26. package/key-bag-indexdb-R2RWGSQ4.js.map +1 -0
  27. package/mem-filesystem-BZQZLUR6.js +41 -0
  28. package/mem-filesystem-BZQZLUR6.js.map +1 -0
  29. package/metafile-cjs.json +1 -1
  30. package/metafile-esm.json +1 -1
  31. package/metafile-iife.json +1 -1
  32. package/node-filesystem-7YZR3POJ.js +45 -0
  33. package/node-filesystem-7YZR3POJ.js.map +1 -0
  34. package/package.json +12 -8
  35. package/tests/blockstore/fragment-gateway.test.ts +107 -0
  36. package/tests/blockstore/keyed-crypto.test.ts +332 -0
  37. package/tests/blockstore/loader.test.ts +24 -19
  38. package/tests/blockstore/store.test.ts +51 -40
  39. package/tests/blockstore/transaction.test.ts +19 -15
  40. package/tests/fireproof/all-gateway.test.ts +395 -0
  41. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.car +0 -0
  42. package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.js +316 -0
  43. package/tests/fireproof/config.test.ts +94 -78
  44. package/tests/fireproof/convert_uint8.py +27 -0
  45. package/tests/fireproof/crdt.test.ts +34 -28
  46. package/tests/fireproof/database.test.ts +22 -14
  47. package/tests/fireproof/fireproof.test.fixture.ts +133 -0
  48. package/tests/fireproof/fireproof.test.ts +331 -219
  49. package/tests/fireproof/hello.test.ts +34 -18
  50. package/tests/fireproof/indexer.test.ts +34 -27
  51. package/tests/fireproof/utils.test.ts +65 -0
  52. package/tests/helpers.ts +28 -57
  53. package/utils-AISQB3PB.js +14 -0
  54. package/utils-AISQB3PB.js.map +1 -0
  55. package/chunk-BNL4PVBF.js +0 -314
  56. package/chunk-BNL4PVBF.js.map +0 -1
  57. package/chunk-JW2QT6BF.js +0 -184
  58. package/chunk-JW2QT6BF.js.map +0 -1
  59. package/node-sys-container-MIEX6ELJ.js +0 -29
  60. package/node-sys-container-MIEX6ELJ.js.map +0 -1
  61. package/store-file-VJ6BI4II.js +0 -191
  62. package/store-file-VJ6BI4II.js.map +0 -1
  63. package/store-indexdb-WLRSICCB.js.map +0 -1
package/index.js CHANGED
@@ -1,30 +1,33 @@
1
+ import {
2
+ FILESTORE_VERSION
3
+ } from "./chunk-7EWIAXTM.js";
4
+ import {
5
+ getFileName,
6
+ getFileSystem,
7
+ getPath,
8
+ toArrayBuffer
9
+ } from "./chunk-XINRLWR3.js";
10
+ import {
11
+ INDEXDB_VERSION
12
+ } from "./chunk-PB4BKL4O.js";
1
13
  import {
2
14
  NotFoundError,
3
15
  Result,
16
+ UInt8ArrayEqual,
17
+ __export,
4
18
  dataDir,
5
- decodeFile,
6
- encodeFile,
7
19
  ensureLogger,
8
- exception2Result,
20
+ ensureSuperLog,
21
+ ensureSuperThis,
9
22
  exceptionWrapper,
10
23
  getKey,
11
24
  getName,
12
25
  getStore,
13
- isNotFoundError,
14
- runtime_exports,
15
- toCryptoOpts
16
- } from "./chunk-BNL4PVBF.js";
17
- import {
18
- SysContainer,
19
- __export,
20
- falsyToUndef,
21
- isFalsy,
22
- throwFalsy
23
- } from "./chunk-JW2QT6BF.js";
26
+ isNotFoundError
27
+ } from "./chunk-MAK4D54P.js";
24
28
 
25
29
  // src/database.ts
26
- import { uuidv7 } from "uuidv7";
27
- import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
30
+ import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
28
31
 
29
32
  // src/write-queue.ts
30
33
  function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -67,13 +70,83 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
67
70
  }
68
71
 
69
72
  // src/crdt.ts
70
- import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
73
+ import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
74
+
75
+ // src/runtime/wait-pr-multiformats/block.ts
76
+ var block_exports = {};
77
+ __export(block_exports, {
78
+ Block: () => Block,
79
+ create: () => create,
80
+ createUnsafe: () => createUnsafe,
81
+ decode: () => decode,
82
+ encode: () => encode
83
+ });
84
+ import { bytes as binary, CID } from "multiformats";
85
+ import { Block as mfBlock } from "multiformats/block";
86
+ var Block = mfBlock;
87
+ async function decode({
88
+ bytes,
89
+ codec: codec3,
90
+ hasher: hasher7
91
+ }) {
92
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
93
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
94
+ const value = await Promise.resolve(codec3.decode(bytes));
95
+ const hash = await hasher7.digest(bytes);
96
+ const cid = CID.create(1, codec3.code, hash);
97
+ return new mfBlock({ value, bytes, cid });
98
+ }
99
+ async function encode({
100
+ value,
101
+ codec: codec3,
102
+ hasher: hasher7
103
+ }) {
104
+ if (typeof value === "undefined") throw new Error('Missing required argument "value"');
105
+ if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
106
+ const bytes = await Promise.resolve(codec3.encode(value));
107
+ const hash = await hasher7.digest(bytes);
108
+ const cid = CID.create(1, codec3.code, hash);
109
+ return new mfBlock({ value, bytes, cid });
110
+ }
111
+ async function create({
112
+ bytes,
113
+ cid,
114
+ hasher: hasher7,
115
+ codec: codec3
116
+ }) {
117
+ if (bytes == null) throw new Error('Missing required argument "bytes"');
118
+ if (hasher7 == null) throw new Error('Missing required argument "hasher"');
119
+ const value = await Promise.resolve(codec3.decode(bytes));
120
+ const hash = await hasher7.digest(bytes);
121
+ if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
122
+ throw new Error("CID hash does not match bytes");
123
+ }
124
+ return createUnsafe({
125
+ bytes,
126
+ cid,
127
+ value,
128
+ codec: codec3
129
+ });
130
+ }
131
+ async function createUnsafe({
132
+ bytes,
133
+ cid,
134
+ value: maybeValue,
135
+ codec: codec3
136
+ }) {
137
+ const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
138
+ if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
139
+ return new Block({
140
+ cid,
141
+ bytes,
142
+ value
143
+ });
144
+ }
71
145
 
72
146
  // src/crdt-helpers.ts
73
- import { encode as encode3, decode as decode3, Block as Block2 } from "multiformats/block";
74
- import { parse as parse2 } from "multiformats/link";
75
- import { sha256 as hasher2 } from "multiformats/hashes/sha2";
76
- import * as codec2 from "@ipld/dag-cbor";
147
+ import { parse as parse3 } from "multiformats/link";
148
+ import { sha256 as hasher5 } from "multiformats/hashes/sha2";
149
+ import * as codec from "@ipld/dag-cbor";
77
150
  import { put, get, entries, root } from "@web3-storage/pail/crdt";
78
151
  import { EventFetcher, vis } from "@web3-storage/pail/clock";
79
152
  import * as Batch from "@web3-storage/pail/crdt/batch";
@@ -84,512 +157,147 @@ __export(blockstore_exports, {
84
157
  BaseBlockstore: () => BaseBlockstore,
85
158
  CarTransaction: () => CarTransaction,
86
159
  CompactionFetcher: () => CompactionFetcher,
87
- ConnectREST: () => ConnectREST,
88
160
  ConnectionBase: () => ConnectionBase,
89
- DataStore: () => DataStore,
90
161
  EncryptedBlockstore: () => EncryptedBlockstore,
91
- Loadable: () => Loadable,
162
+ FragmentGateway: () => FragmentGateway,
92
163
  Loader: () => Loader,
93
- MetaStore: () => MetaStore,
94
- NotFoundError: () => NotFoundError,
95
- RemoteWAL: () => RemoteWAL,
96
- isNotFoundError: () => isNotFoundError,
164
+ addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
165
+ ensureStart: () => ensureStart,
166
+ getGatewayFromURL: () => getGatewayFromURL,
97
167
  parseCarFile: () => parseCarFile,
98
168
  registerStoreProtocol: () => registerStoreProtocol,
169
+ setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
99
170
  testStoreFactory: () => testStoreFactory,
100
- toStoreRuntime: () => toStoreRuntime,
101
- toURL: () => toURL
171
+ toCIDBlock: () => toCIDBlock,
172
+ toStoreRuntime: () => toStoreRuntime
102
173
  });
103
174
 
104
- // src/blockstore/connection-base.ts
105
- import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
106
- import { MemoryBlockstore } from "@web3-storage/pail/block";
175
+ // src/blockstore/types.ts
176
+ function toCIDBlock(block) {
177
+ return block;
178
+ }
107
179
 
108
- // src/blockstore/task-manager.ts
109
- var TaskManager = class {
110
- constructor(loader) {
111
- this.eventsWeHandled = /* @__PURE__ */ new Set();
112
- this.queue = [];
113
- this.isProcessing = false;
114
- this.loader = loader;
115
- this.logger = ensureLogger(loader.logger, "TaskManager");
116
- }
117
- async handleEvent(eventBlock) {
118
- const cid = eventBlock.cid.toString();
119
- const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
120
- for (const parent of parents) {
121
- this.eventsWeHandled.add(parent);
122
- }
123
- this.queue.push({ cid, eventBlock, retries: 0 });
124
- this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
125
- void this.processQueue();
126
- }
127
- async processQueue() {
128
- if (this.isProcessing) return;
129
- this.isProcessing = true;
130
- const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
131
- const first = filteredQueue[0];
132
- if (!first) {
133
- return;
134
- }
135
- try {
136
- this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
137
- this.eventsWeHandled.add(first.cid);
138
- this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
139
- } catch (err) {
140
- if (first.retries++ > 3) {
141
- this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
142
- this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
143
- }
144
- await new Promise((resolve) => setTimeout(resolve, 50));
145
- throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
146
- } finally {
147
- this.isProcessing = false;
148
- if (this.queue.length > 0) {
149
- void this.processQueue();
150
- }
151
- }
152
- }
153
- };
180
+ // src/blockstore/store-factory.ts
181
+ import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
154
182
 
155
- // src/blockstore/connection-base.ts
156
- var ConnectionBase = class {
157
- constructor(logger) {
158
- // readonly ready: Promise<unknown>;
159
- // todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
160
- this.eventBlocks = new MemoryBlockstore();
161
- this.parents = [];
162
- this.loaded = Promise.resolve();
163
- this.logger = ensureLogger(logger, "ConnectionBase");
164
- }
165
- async refresh() {
166
- await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load("main");
167
- await (await throwFalsy(this.loader).remoteWAL())._process();
168
- }
169
- connect({ loader }) {
170
- if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
171
- this.connectMeta({ loader });
172
- this.connectStorage({ loader });
173
- }
174
- connectMeta({ loader }) {
175
- if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
176
- this.loader = loader;
177
- this.taskManager = new TaskManager(loader);
178
- this.onConnect();
179
- this.logger.Warn().Msg("connectMeta: connecting to remote meta store is disabled");
180
- }
181
- async onConnect() {
182
- return;
183
- }
184
- connectStorage({ loader }) {
185
- if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
186
- this.loader = loader;
187
- this.logger.Warn().Msg("connectStorage: connecting to remote meta store is disabled");
188
- }
189
- async createEventBlock(bytes) {
190
- const data = {
191
- dbMeta: bytes
192
- };
193
- const event = await EventBlock.create(
194
- data,
195
- this.parents
183
+ // src/runtime/files.ts
184
+ var files_exports = {};
185
+ __export(files_exports, {
186
+ decodeFile: () => decodeFile,
187
+ encodeFile: () => encodeFile
188
+ });
189
+ import * as UnixFS from "@ipld/unixfs";
190
+ import * as raw from "multiformats/codecs/raw";
191
+ import { withMaxChunkSize } from "@ipld/unixfs/file/chunker/fixed";
192
+ import { withWidth } from "@ipld/unixfs/file/layout/balanced";
193
+ import { exporter } from "ipfs-unixfs-exporter";
194
+ var queuingStrategy = UnixFS.withCapacity();
195
+ var settings = UnixFS.configure({
196
+ fileChunkEncoder: raw,
197
+ smallFileEncoder: raw,
198
+ chunker: withMaxChunkSize(1024 * 1024),
199
+ fileLayout: withWidth(1024)
200
+ });
201
+ async function collect(collectable) {
202
+ const chunks = [];
203
+ await collectable.pipeTo(
204
+ new WritableStream({
205
+ write(chunk) {
206
+ chunks.push(chunk);
207
+ }
208
+ })
209
+ );
210
+ return chunks;
211
+ }
212
+ async function encodeFile(blob) {
213
+ const readable = createFileEncoderStream(blob);
214
+ const blocks = await collect(readable);
215
+ return { cid: blocks.at(-1).cid, blocks };
216
+ }
217
+ async function decodeFile(blocks, cid, meta) {
218
+ const entry = await exporter(cid.toString(), blocks, { length: meta.size });
219
+ const chunks = [];
220
+ for await (const chunk of entry.content()) {
221
+ chunks.push(chunk);
222
+ }
223
+ return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
224
+ }
225
+ function createFileEncoderStream(blob) {
226
+ const { readable, writable } = new TransformStream({}, queuingStrategy);
227
+ const unixfsWriter = UnixFS.createWriter({ writable, settings });
228
+ const fileBuilder = new UnixFSFileBuilder("", blob);
229
+ void (async () => {
230
+ await fileBuilder.finalize(unixfsWriter);
231
+ await unixfsWriter.close();
232
+ })();
233
+ return readable;
234
+ }
235
+ var UnixFSFileBuilder = class {
236
+ #file;
237
+ constructor(name, file) {
238
+ this.name = name;
239
+ this.#file = file;
240
+ }
241
+ async finalize(writer) {
242
+ const unixfsFileWriter = UnixFS.createFileWriter(writer);
243
+ await this.#file.stream().pipeTo(
244
+ new WritableStream({
245
+ async write(chunk) {
246
+ await unixfsFileWriter.write(chunk);
247
+ }
248
+ })
196
249
  );
197
- await this.eventBlocks.put(event.cid, event.bytes);
198
- return event;
199
- }
200
- async decodeEventBlock(bytes) {
201
- const event = await decodeEventBlock(bytes);
202
- return event;
203
- }
204
- // move this stuff to connect
205
- // async getDashboardURL(compact = true) {
206
- // const baseUrl = 'https://dashboard.fireproof.storage/'
207
- // if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
208
- // // if (compact) {
209
- // // await this.compact()
210
- // // }
211
- // const currents = await this.loader?.metaStore?.load()
212
- // if (!currents) throw new Error("Can't sync empty database: save data first")
213
- // if (currents.length > 1)
214
- // throw new Error("Can't sync database with split heads: make an update first")
215
- // const current = currents[0]
216
- // const params = {
217
- // car: current.car.toString()
218
- // }
219
- // if (current.key) {
220
- // // @ts-ignore
221
- // params.key = current.key.toString()
222
- // }
223
- // // @ts-ignore
224
- // if (this.name) {
225
- // // @ts-ignore
226
- // params.name = this.name
227
- // }
228
- // const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
229
- // console.log('Import to dashboard: ' + url.toString())
230
- // return url
231
- // }
232
- // openDashboard() {
233
- // void this.getDashboardURL().then(url => {
234
- // if (url) window.open(url.toString(), '_blank')
235
- // })
236
- // }
237
- };
238
-
239
- // src/blockstore/connect-rest.ts
240
- var ConnectREST = class extends ConnectionBase {
241
- constructor(base, logger) {
242
- super(ensureLogger(logger, "ConnectREST"));
243
- this.baseUrl = new URL(base);
244
- }
245
- async dataUpload(bytes, params) {
246
- const carCid = params.car.toString();
247
- const uploadURL = new URL(`/cars/${carCid}.car`, this.baseUrl);
248
- const done = await fetch(uploadURL, { method: "PUT", body: bytes });
249
- if (!done.ok) {
250
- throw this.logger.Error().Msg("failed to upload data " + done.statusText);
251
- }
252
- }
253
- async dataDownload(params) {
254
- const { car } = params;
255
- const fetchFromUrl = new URL(`/cars/${car.toString()}.car`, this.baseUrl);
256
- const response = await fetch(fetchFromUrl);
257
- if (!response.ok) {
258
- return void 0;
259
- }
260
- const bytes = new Uint8Array(await response.arrayBuffer());
261
- return bytes;
262
- }
263
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
264
- async metaUpload(bytes, params) {
265
- return void 0;
266
- }
267
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
268
- async metaDownload(params) {
269
- return [];
250
+ return await unixfsFileWriter.close();
270
251
  }
271
252
  };
272
253
 
273
- // src/blockstore/store-factory.ts
274
- import { KeyedResolvOnce } from "@adviser/cement";
275
-
276
254
  // src/blockstore/store.ts
277
255
  import pLimit2 from "p-limit";
278
256
  import { format, parse } from "@ipld/dag-json";
279
- import { ResolveOnce as ResolveOnce2, Result as Result2 } from "@adviser/cement";
257
+ import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result4 } from "@adviser/cement";
258
+
259
+ // src/types.ts
260
+ function isFalsy(value) {
261
+ return value === false && value === null && value === void 0;
262
+ }
263
+ function throwFalsy(value) {
264
+ if (isFalsy(value)) {
265
+ throw new Error("value is Falsy");
266
+ }
267
+ return value;
268
+ }
269
+ function falsyToUndef(value) {
270
+ if (isFalsy(value)) {
271
+ return void 0;
272
+ }
273
+ return value;
274
+ }
280
275
 
281
276
  // src/blockstore/loader.ts
282
277
  import pLimit from "p-limit";
283
278
  import { CarReader } from "@ipld/car";
284
- import { ResolveOnce } from "@adviser/cement";
285
-
286
- // src/blockstore/types.ts
287
- function toCIDBlock(block) {
288
- return block;
289
- }
279
+ import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
290
280
 
291
281
  // src/blockstore/loader-helpers.ts
292
- import { encode, decode } from "multiformats/block";
293
282
  import { sha256 as hasher } from "multiformats/hashes/sha2";
294
- import * as raw from "multiformats/codecs/raw";
295
- import * as CBW from "@ipld/car/buffer-writer";
296
- import * as codec from "@ipld/dag-cbor";
297
- async function encodeCarFile(roots, t) {
298
- let size = 0;
299
- const headerSize = CBW.headerLength({ roots });
300
- size += headerSize;
301
- for (const { cid, bytes } of t.entries()) {
302
- size += CBW.blockLength({ cid, bytes });
303
- }
304
- const buffer = new Uint8Array(size);
305
- const writer = CBW.createWriter(buffer, { headerSize });
306
- for (const r of roots) {
307
- writer.addRoot(r);
308
- }
309
- for (const { cid, bytes } of t.entries()) {
310
- writer.write({ cid, bytes });
311
- }
312
- writer.close();
313
- return await encode({ value: writer.bytes, hasher, codec: raw });
314
- }
315
- async function encodeCarHeader(fp) {
316
- return await encode({
317
- value: { fp },
318
- hasher,
319
- codec
320
- });
321
- }
283
+ import * as dagCodec from "@ipld/dag-cbor";
322
284
  async function parseCarFile(reader, logger) {
323
285
  const roots = await reader.getRoots();
324
286
  const header = await reader.get(roots[0]);
325
287
  if (!header) throw logger.Error().Msg("missing header block").AsError();
326
- const { value } = await decode({ bytes: header.bytes, hasher, codec });
327
- const fpvalue = value;
288
+ const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
289
+ const fpvalue = dec.value;
328
290
  if (fpvalue && !fpvalue.fp) {
329
291
  throw logger.Error().Msg("missing fp").AsError();
330
292
  }
331
293
  return fpvalue.fp;
332
294
  }
333
295
 
334
- // src/blockstore/encrypt-helpers.ts
335
- import { sha256 } from "multiformats/hashes/sha2";
336
- import { CID as CID2 } from "multiformats";
337
- import { encode as encode2, decode as decode2, create as mfCreate } from "multiformats/block";
338
- import * as dagcbor from "@ipld/dag-cbor";
339
- import { MemoryBlockstore as MemoryBlockstore2 } from "@web3-storage/pail/block";
340
- import { bf } from "prolly-trees/utils";
341
- import { nocache as cache } from "prolly-trees/cache";
342
- import { create, load } from "prolly-trees/cid-set";
343
-
344
- // src/blockstore/encrypt-codec.ts
345
- import { CID } from "multiformats";
346
- function makeCodec(ilogger, crypto, randomBytes) {
347
- const logger = ensureLogger(ilogger, "makeCodec");
348
- const enc32 = (value) => {
349
- value = +value;
350
- const buff = new Uint8Array(4);
351
- buff[3] = value >>> 24;
352
- buff[2] = value >>> 16;
353
- buff[1] = value >>> 8;
354
- buff[0] = value & 255;
355
- return buff;
356
- };
357
- const readUInt32LE = (buffer) => {
358
- const offset = buffer.byteLength - 4;
359
- return (buffer[offset] | buffer[offset + 1] << 8 | buffer[offset + 2] << 16) + buffer[offset + 3] * 16777216;
360
- };
361
- const concat = (buffers) => {
362
- const uint8Arrays = buffers.map((b) => b instanceof ArrayBuffer ? new Uint8Array(b) : b);
363
- const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0);
364
- const result = new Uint8Array(totalLength);
365
- let offset = 0;
366
- for (const arr of uint8Arrays) {
367
- result.set(arr, offset);
368
- offset += arr.length;
369
- }
370
- return result;
371
- };
372
- const encode4 = ({ iv, bytes }) => concat([iv, bytes]);
373
- const decode4 = (bytes) => {
374
- const iv = bytes.subarray(0, 12);
375
- bytes = bytes.slice(12);
376
- return { iv, bytes };
377
- };
378
- const code = 3145728 + 1337;
379
- async function subtleKey(key) {
380
- return await crypto.importKey(
381
- "raw",
382
- // raw or jwk
383
- key,
384
- // raw data
385
- "AES-GCM",
386
- false,
387
- // extractable
388
- ["encrypt", "decrypt"]
389
- );
390
- }
391
- const decrypt = async ({ key, value }) => {
392
- const { bytes: inBytes, iv } = value;
393
- const cryKey = await subtleKey(key);
394
- const deBytes = await crypto.decrypt(
395
- {
396
- name: "AES-GCM",
397
- iv,
398
- tagLength: 128
399
- },
400
- cryKey,
401
- inBytes
402
- );
403
- const bytes = new Uint8Array(deBytes);
404
- const len = readUInt32LE(bytes.subarray(0, 4));
405
- const cid = CID.decode(bytes.subarray(4, 4 + len));
406
- return { cid, bytes: bytes.subarray(4 + len) };
407
- };
408
- const encrypt = async ({ key, cid, bytes }) => {
409
- const len = enc32(cid.bytes.byteLength);
410
- const iv = randomBytes(12);
411
- const msg = concat([len, cid.bytes, bytes]);
412
- try {
413
- const cryKey = await subtleKey(key);
414
- const deBytes = await crypto.encrypt(
415
- {
416
- name: "AES-GCM",
417
- iv,
418
- tagLength: 128
419
- },
420
- cryKey,
421
- msg
422
- );
423
- bytes = new Uint8Array(deBytes);
424
- } catch (e) {
425
- throw logger.Error().Err(e).Msg("encrypt failed").AsError();
426
- }
427
- return { value: { bytes, iv } };
428
- };
429
- const cryptoFn = (key) => {
430
- return { encrypt: (opts) => encrypt({ ...opts, key }), decrypt: (opts) => decrypt({ ...opts, key }) };
431
- };
432
- const name = "jchris@encrypted-block:aes-gcm";
433
- return { encode: encode4, decode: decode4, code, name, encrypt, decrypt, crypto: cryptoFn };
434
- }
435
-
436
- // src/blockstore/encrypt-helpers.ts
437
- function carLogIncludesGroup(list, cidMatch) {
438
- return list.some((cid) => {
439
- return cid.toString() === cidMatch.toString();
440
- });
441
- }
442
- function makeEncDec(logger, crypto, randomBytes) {
443
- const codec4 = makeCodec(logger, crypto, randomBytes);
444
- const encrypt = async function* ({
445
- get: get2,
446
- cids,
447
- hasher: hasher4,
448
- key,
449
- cache: cache3,
450
- chunker: chunker2,
451
- root: root3
452
- }) {
453
- const set = /* @__PURE__ */ new Set();
454
- let eroot;
455
- if (!carLogIncludesGroup(cids, root3)) cids.push(root3);
456
- for (const cid of cids) {
457
- const unencrypted = await get2(cid);
458
- if (!unencrypted) throw logger.Error().Ref("cid", cid).Msg("missing cid block").AsError();
459
- const encrypted = await codec4.encrypt({ ...unencrypted, key });
460
- const block2 = await encode2({ ...encrypted, codec: codec4, hasher: hasher4 });
461
- yield block2;
462
- set.add(block2.cid.toString());
463
- if (unencrypted.cid.equals(root3)) eroot = block2.cid;
464
- }
465
- if (!eroot) throw logger.Error().Msg("cids does not include root").AsError();
466
- const list = [...set].map((s) => CID2.parse(s));
467
- let last;
468
- for await (const node of create({ list, get: get2, cache: cache3, chunker: chunker2, hasher: hasher4, codec: dagcbor })) {
469
- const block2 = await node.block;
470
- yield block2;
471
- last = block2;
472
- }
473
- if (!last) throw logger.Error().Msg("missing last block").AsError();
474
- const head = [eroot, last.cid];
475
- const block = await encode2({ value: head, codec: dagcbor, hasher: hasher4 });
476
- yield block;
477
- };
478
- const decrypt = async function* ({
479
- root: root3,
480
- get: get2,
481
- key,
482
- cache: cache3,
483
- chunker: chunker2,
484
- hasher: hasher4
485
- }) {
486
- const getWithDecode = async (cid) => get2(cid).then(async (block) => {
487
- if (!block) return;
488
- const decoded = await decode2({ ...block, codec: dagcbor, hasher: hasher4 });
489
- return decoded;
490
- });
491
- const getWithDecrypt = async (cid) => get2(cid).then(async (block) => {
492
- if (!block) return;
493
- const decoded = await decode2({ ...block, codec: codec4, hasher: hasher4 });
494
- return decoded;
495
- });
496
- const decodedRoot = await getWithDecode(root3);
497
- if (!decodedRoot) throw logger.Error().Msg("missing root").AsError();
498
- if (!decodedRoot.bytes) throw logger.Error().Msg("missing bytes").AsError();
499
- const {
500
- value: [eroot, tree]
501
- } = decodedRoot;
502
- const rootBlock = await get2(eroot);
503
- if (!rootBlock) throw logger.Error().Msg("missing root block").AsError();
504
- const cidset = await load({ cid: tree, get: getWithDecode, cache: cache3, chunker: chunker2, codec: codec4, hasher: hasher4 });
505
- const { result: nodes } = await cidset.getAllEntries();
506
- const unwrap = async (eblock) => {
507
- if (!eblock) throw logger.Error().Msg("missing block").AsError();
508
- if (!eblock.value) {
509
- eblock = await decode2({ ...eblock, codec: codec4, hasher: hasher4 });
510
- if (!eblock.value) throw logger.Error().Msg("missing value").AsError();
511
- }
512
- const { bytes, cid } = await codec4.decrypt({ ...eblock, key }).catch((e) => {
513
- throw e;
514
- });
515
- const block = await mfCreate({ cid, bytes, hasher: hasher4, codec: codec4 });
516
- return block;
517
- };
518
- const promises = [];
519
- for (const { cid } of nodes) {
520
- if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap));
521
- }
522
- yield* promises;
523
- yield unwrap(rootBlock);
524
- };
525
- return { encrypt, decrypt };
526
- }
527
- var chunker = bf(30);
528
- function hexStringToUint8Array(hexString) {
529
- const length = hexString.length;
530
- const uint8Array = new Uint8Array(length / 2);
531
- for (let i = 0; i < length; i += 2) {
532
- uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16);
533
- }
534
- return uint8Array;
535
- }
536
- async function encryptedEncodeCarFile(logger, crypto, key, rootCid, t) {
537
- const encryptionKey = hexStringToUint8Array(key);
538
- const encryptedBlocks = new MemoryBlockstore2();
539
- const cidsToEncrypt = [];
540
- for (const { cid, bytes } of t.entries()) {
541
- cidsToEncrypt.push(cid);
542
- const g = await t.get(cid);
543
- if (!g) throw logger.Error().Ref("cid", cid).Int("bytes", bytes.length).Msg("missing cid block").AsError();
544
- }
545
- let last = null;
546
- const { encrypt } = makeEncDec(logger, crypto, crypto.randomBytes);
547
- for await (const block of encrypt({
548
- cids: cidsToEncrypt,
549
- get: t.get.bind(t),
550
- key: encryptionKey,
551
- hasher: sha256,
552
- chunker,
553
- cache,
554
- root: rootCid
555
- })) {
556
- await encryptedBlocks.put(block.cid, block.bytes);
557
- last = block;
558
- }
559
- if (!last) throw logger.Error().Msg("no blocks encrypted").AsError();
560
- const encryptedCar = await encodeCarFile([last.cid], encryptedBlocks);
561
- return encryptedCar;
562
- }
563
- async function decodeEncryptedCar(logger, crypto, key, reader) {
564
- const roots = await reader.getRoots();
565
- const root3 = roots[0];
566
- return await decodeCarBlocks(logger, crypto, root3, reader.get.bind(reader), key);
567
- }
568
- async function decodeCarBlocks(logger, crypto, root3, get2, keyMaterial) {
569
- const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial);
570
- const decryptionKey = decryptionKeyUint8.buffer.slice(0, decryptionKeyUint8.byteLength);
571
- const decryptedBlocks = new MemoryBlockstore2();
572
- let last = null;
573
- const { decrypt } = makeEncDec(logger, crypto, crypto.randomBytes);
574
- for await (const block of decrypt({
575
- root: root3,
576
- get: get2,
577
- key: decryptionKey,
578
- hasher: sha256,
579
- chunker,
580
- cache
581
- })) {
582
- await decryptedBlocks.put(block.cid, block.bytes);
583
- last = block;
584
- }
585
- if (!last) throw logger.Error().Msg("no blocks decrypted").AsError();
586
- return { blocks: decryptedBlocks, root: last.cid };
587
- }
588
-
589
296
  // src/blockstore/transaction.ts
590
- import { MemoryBlockstore as MemoryBlockstore3 } from "@web3-storage/pail/block";
591
- var CarTransaction = class extends MemoryBlockstore3 {
592
- constructor(parent, opts = { add: true }) {
297
+ import { MemoryBlockstore } from "@web3-storage/pail/block";
298
+ import { toCryptoRuntime } from "@adviser/cement";
299
+ var CarTransaction = class extends MemoryBlockstore {
300
+ constructor(parent, opts = { add: true, noLoader: false }) {
593
301
  super();
594
302
  if (opts.add) {
595
303
  parent.transactions.add(this);
@@ -603,8 +311,8 @@ var CarTransaction = class extends MemoryBlockstore3 {
603
311
  return super.get(cid);
604
312
  }
605
313
  };
606
- function defaultedBlockstoreRuntime(opts, component, ctx) {
607
- const logger = ensureLogger(opts, component, ctx);
314
+ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
315
+ const logger = ensureLogger(sthis, component, ctx);
608
316
  const store = opts.store || {};
609
317
  return {
610
318
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -621,22 +329,24 @@ function defaultedBlockstoreRuntime(opts, component, ctx) {
621
329
  threshold: 1e3 * 1e3,
622
330
  ...opts,
623
331
  logger,
624
- crypto: toCryptoOpts(opts.crypto),
332
+ keyBag: opts.keyBag || {},
333
+ crypto: toCryptoRuntime(opts.crypto),
625
334
  store,
626
- storeRuntime: toStoreRuntime(store, logger)
335
+ storeRuntime: toStoreRuntime(store, sthis)
627
336
  };
628
337
  }
629
- var blockstoreFactory = function(opts) {
338
+ function blockstoreFactory(sthis, opts) {
630
339
  if (opts.name) {
631
- return new EncryptedBlockstore(opts);
340
+ return new EncryptedBlockstore(sthis, opts);
632
341
  } else {
633
342
  return new BaseBlockstore(opts);
634
343
  }
635
- };
344
+ }
636
345
  var BaseBlockstore = class {
637
346
  constructor(ebOpts = {}) {
638
347
  this.transactions = /* @__PURE__ */ new Set();
639
- this.ebOpts = defaultedBlockstoreRuntime(ebOpts, "BaseBlockstore");
348
+ this.sthis = ensureSuperThis(ebOpts);
349
+ this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
640
350
  this.logger = this.ebOpts.logger;
641
351
  }
642
352
  // ready: Promise<void>;
@@ -659,8 +369,8 @@ var BaseBlockstore = class {
659
369
  throw this.logger.Error().Msg("use a transaction to put").AsError();
660
370
  }
661
371
  // TransactionMeta
662
- async transaction(fn, _opts = {}) {
663
- const t = new CarTransaction(this);
372
+ async transaction(fn, _opts) {
373
+ const t = new CarTransaction(this, _opts);
664
374
  const done = await fn(t);
665
375
  this.lastTxMeta = done;
666
376
  return { t, meta: done };
@@ -677,16 +387,16 @@ var BaseBlockstore = class {
677
387
  }
678
388
  };
679
389
  var EncryptedBlockstore = class extends BaseBlockstore {
680
- constructor(ebOpts) {
390
+ constructor(sthis, ebOpts) {
681
391
  super(ebOpts);
682
392
  this.compacting = false;
683
- this.logger = ensureLogger(ebOpts, "EncryptedBlockstore");
393
+ this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
684
394
  const { name } = ebOpts;
685
395
  if (!name) {
686
396
  throw this.logger.Error().Msg("name required").AsError();
687
397
  }
688
398
  this.name = name;
689
- this.loader = new Loader(this.name, ebOpts);
399
+ this.loader = new Loader(this.name, ebOpts, sthis);
690
400
  }
691
401
  ready() {
692
402
  return this.loader.ready();
@@ -717,10 +427,13 @@ var EncryptedBlockstore = class extends BaseBlockstore {
717
427
  }
718
428
  throw this.logger.Error().Msg("failed to commit car files").AsError();
719
429
  }
720
- async getFile(car, cid, isPublic = false) {
430
+ async getFile(car, cid) {
721
431
  await this.ready();
722
432
  if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
723
- const reader = await this.loader.loadFileCar(car, isPublic);
433
+ const reader = await this.loader.loadFileCar(
434
+ car
435
+ /*, isPublic */
436
+ );
724
437
  const block = await reader.get(cid);
725
438
  if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
726
439
  return block.bytes;
@@ -776,10 +489,20 @@ var CompactionFetcher = class {
776
489
  };
777
490
 
778
491
  // src/blockstore/commit-queue.ts
492
+ import { Future } from "@adviser/cement";
779
493
  var CommitQueue = class {
780
494
  constructor() {
781
495
  this.queue = [];
782
496
  this.processing = false;
497
+ this._waitIdleItems = /* @__PURE__ */ new Set();
498
+ }
499
+ waitIdle() {
500
+ if (this.queue.length === 0 && !this.processing) {
501
+ return Promise.resolve();
502
+ }
503
+ const fn = new Future();
504
+ this._waitIdleItems.add(fn);
505
+ return fn.asPromise();
783
506
  }
784
507
  async enqueue(fn) {
785
508
  return new Promise((resolve, reject) => {
@@ -804,38 +527,384 @@ var CommitQueue = class {
804
527
  this.processing = true;
805
528
  const queueFn = this.queue.shift();
806
529
  if (queueFn) {
807
- queueFn();
530
+ queueFn().finally(() => {
531
+ });
808
532
  }
809
533
  }
534
+ if (this.queue.length === 0 && !this.processing) {
535
+ const toResolve = Array.from(this._waitIdleItems);
536
+ this._waitIdleItems.clear();
537
+ toResolve.map((fn) => fn.resolve());
538
+ }
810
539
  }
811
540
  };
812
541
 
813
- // src/blockstore/loader.ts
814
- import * as CBW2 from "@ipld/car/buffer-writer";
815
- function carLogIncludesGroup2(list, cids) {
816
- return list.some((arr) => {
817
- return arr.toString() === cids.toString();
818
- });
819
- }
820
- function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
821
- const byString = /* @__PURE__ */ new Map();
822
- for (const cid of list) {
823
- if (remove.has(cid.toString())) continue;
824
- byString.set(cid.toString(), cid);
542
+ // src/runtime/key-bag.ts
543
+ var key_bag_exports = {};
544
+ __export(key_bag_exports, {
545
+ KeyBag: () => KeyBag,
546
+ getKeyBag: () => getKeyBag,
547
+ registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
548
+ });
549
+ import {
550
+ KeyedResolvOnce,
551
+ ResolveOnce,
552
+ ResolveSeq,
553
+ Result as Result2,
554
+ runtimeFn,
555
+ toCryptoRuntime as toCryptoRuntime2,
556
+ URI
557
+ } from "@adviser/cement";
558
+ import { base58btc } from "multiformats/bases/base58";
559
+ var KeyBag = class {
560
+ constructor(rt) {
561
+ this.rt = rt;
562
+ this._warnOnce = new ResolveOnce();
563
+ this._seq = new ResolveSeq();
564
+ this.logger = ensureLogger(rt.sthis, "KeyBag");
565
+ this.logger.Debug().Msg("KeyBag created");
566
+ }
567
+ async subtleKey(key) {
568
+ const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
569
+ if (extractable) {
570
+ this._warnOnce.once(
571
+ () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
572
+ );
573
+ }
574
+ return await this.rt.crypto.importKey(
575
+ "raw",
576
+ // raw or jwk
577
+ base58btc.decode(key),
578
+ // hexStringToUint8Array(key), // raw data
579
+ "AES-GCM",
580
+ extractable,
581
+ ["encrypt", "decrypt"]
582
+ );
825
583
  }
826
- return [...byString.values()];
827
- }
828
- function toHexString(byteArray) {
829
- return Array.from(byteArray).map((byte) => byte.toString(16).padStart(2, "0")).join("");
830
- }
831
- var Loadable = class {
832
- constructor() {
833
- this.name = "";
834
- this.carLog = new Array();
584
+ async ensureKeyFromUrl(url, keyFactory) {
585
+ const storeKey = url.getParam("storekey");
586
+ if (storeKey === "insecure") {
587
+ return Result2.Ok(url);
588
+ }
589
+ if (!storeKey) {
590
+ const keyName = `@${keyFactory()}@`;
591
+ const ret = await this.getNamedKey(keyName);
592
+ if (ret.isErr()) {
593
+ return ret;
594
+ }
595
+ const urb = url.build().setParam("storekey", keyName);
596
+ return Result2.Ok(urb.URI());
597
+ }
598
+ if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
599
+ const ret = await this.getNamedKey(storeKey);
600
+ if (ret.isErr()) {
601
+ return ret;
602
+ }
603
+ }
604
+ return Result2.Ok(url);
605
+ }
606
+ async toKeyWithFingerPrint(keyStr) {
607
+ const material = base58btc.decode(keyStr);
608
+ const key = await this.subtleKey(keyStr);
609
+ const fpr = await this.rt.crypto.digestSHA256(material);
610
+ return Result2.Ok({
611
+ key,
612
+ fingerPrint: base58btc.encode(new Uint8Array(fpr))
613
+ });
614
+ }
615
+ async setNamedKey(name, key) {
616
+ return this._seq.add(() => this._setNamedKey(name, key));
617
+ }
618
+ // avoid deadlock
619
+ async _setNamedKey(name, key) {
620
+ const item = {
621
+ name,
622
+ key
623
+ };
624
+ const bag = await this.rt.getBag();
625
+ this.logger.Debug().Str("name", name).Msg("setNamedKey");
626
+ await bag.set(name, item);
627
+ return await this.toKeyWithFingerPrint(item.key);
628
+ }
629
+ async getNamedExtractableKey(name, failIfNotFound = false) {
630
+ const ret = await this.getNamedKey(name, failIfNotFound);
631
+ if (ret.isErr()) {
632
+ return ret;
633
+ }
634
+ const named = ret.Ok();
635
+ return Result2.Ok({
636
+ ...named,
637
+ extract: async () => {
638
+ const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
639
+ return {
640
+ key: ext,
641
+ keyStr: base58btc.encode(ext)
642
+ };
643
+ }
644
+ });
645
+ }
646
+ async getNamedKey(name, failIfNotFound = false) {
647
+ const id = this.rt.sthis.nextId(4).str;
648
+ return this._seq.add(async () => {
649
+ const bag = await this.rt.getBag();
650
+ const named = await bag.get(name);
651
+ if (named) {
652
+ const fpr = await this.toKeyWithFingerPrint(named.key);
653
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
654
+ return fpr;
655
+ }
656
+ if (failIfNotFound) {
657
+ this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
658
+ return Result2.Err(new Error(`Key not found: ${name}`));
659
+ }
660
+ const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
661
+ this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
662
+ return ret;
663
+ });
664
+ }
665
+ };
666
+ var keyBagProviderFactories = new Map(
667
+ [
668
+ {
669
+ protocol: "file:",
670
+ factory: async (url, sthis) => {
671
+ const { KeyBagProviderFile } = await import("./key-bag-file-DFMW6ZM6.js");
672
+ return new KeyBagProviderFile(url, sthis);
673
+ }
674
+ },
675
+ {
676
+ protocol: "indexdb:",
677
+ factory: async (url, sthis) => {
678
+ const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-R2RWGSQ4.js");
679
+ return new KeyBagProviderIndexDB(url, sthis);
680
+ }
681
+ }
682
+ ].map((i) => [i.protocol, i])
683
+ );
684
+ function registerKeyBagProviderFactory(item) {
685
+ const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
686
+ keyBagProviderFactories.set(protocol, {
687
+ ...item,
688
+ protocol
689
+ });
690
+ }
691
+ function defaultKeyBagOpts(sthis, kbo) {
692
+ if (kbo.keyRuntime) {
693
+ return kbo.keyRuntime;
694
+ }
695
+ const logger = ensureLogger(sthis, "KeyBag");
696
+ let url;
697
+ if (kbo.url) {
698
+ url = URI.from(kbo.url);
699
+ logger.Debug().Url(url).Msg("from opts");
700
+ } else {
701
+ let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
702
+ if (runtimeFn().isBrowser) {
703
+ url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
704
+ } else {
705
+ if (!bagFnameOrUrl) {
706
+ const home = sthis.env.get("HOME");
707
+ bagFnameOrUrl = `${home}/.fireproof/keybag`;
708
+ url = URI.from(`file://${bagFnameOrUrl}`);
709
+ } else {
710
+ url = URI.from(bagFnameOrUrl);
711
+ }
712
+ }
713
+ logger.Debug().Url(url).Msg("from env");
714
+ }
715
+ const kitem = keyBagProviderFactories.get(url.protocol);
716
+ if (!kitem) {
717
+ throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
718
+ }
719
+ const getBag = async () => kitem.factory(url, sthis);
720
+ if (url.hasParam("masterkey")) {
721
+ throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
722
+ }
723
+ return {
724
+ url,
725
+ crypto: kbo.crypto || toCryptoRuntime2({}),
726
+ sthis,
727
+ logger,
728
+ keyLength: kbo.keyLength || 16,
729
+ getBag,
730
+ id: () => {
731
+ return url.toString();
732
+ }
733
+ };
734
+ }
735
+ var _keyBags = new KeyedResolvOnce();
736
+ async function getKeyBag(sthis, kbo = {}) {
737
+ await sthis.start();
738
+ const rt = defaultKeyBagOpts(sthis, kbo);
739
+ return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
740
+ }
741
+
742
+ // src/blockstore/commitor.ts
743
+ import * as CBW from "@ipld/car/buffer-writer";
744
+ import { sha256 as hasher2 } from "multiformats/hashes/sha2";
745
+ import * as dagCodec2 from "@ipld/dag-cbor";
746
+ async function encodeCarFile(roots, t, codec3) {
747
+ let size = 0;
748
+ const headerSize = CBW.headerLength({ roots });
749
+ size += headerSize;
750
+ for (const { cid, bytes } of t.entries()) {
751
+ size += CBW.blockLength({ cid, bytes });
752
+ }
753
+ const buffer = new Uint8Array(size);
754
+ const writer = CBW.createWriter(buffer, { headerSize });
755
+ for (const r of roots) {
756
+ writer.addRoot(r);
757
+ }
758
+ for (const { cid, bytes } of t.entries()) {
759
+ writer.write({ cid, bytes });
760
+ }
761
+ writer.close();
762
+ return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
763
+ }
764
+ async function createCarFile(encoder, cid, t) {
765
+ return encodeCarFile([cid], t, encoder);
766
+ }
767
+ async function commitFiles(fileStore, walStore, t, done) {
768
+ const { files: roots } = makeFileCarHeader(done);
769
+ const cids = [];
770
+ const codec3 = (await fileStore.keyedCrypto()).codec();
771
+ const cars = await prepareCarFilesFiles(codec3, roots, t);
772
+ for (const car of cars) {
773
+ const { cid, bytes } = car;
774
+ await fileStore.save({ cid, bytes });
775
+ await walStore.enqueueFile(
776
+ cid
777
+ /*, !!opts.public*/
778
+ );
779
+ cids.push(cid);
780
+ }
781
+ return cids;
782
+ }
783
+ function makeFileCarHeader(result) {
784
+ const files = [];
785
+ for (const [, meta] of Object.entries(result.files || {})) {
786
+ if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
787
+ files.push(meta.cid);
788
+ }
789
+ }
790
+ return { ...result, files };
791
+ }
792
+ async function prepareCarFilesFiles(encoder, roots, t) {
793
+ return [await encodeCarFile(roots, t, encoder)];
794
+ }
795
+ function makeCarHeader(meta, cars, compact = false) {
796
+ const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
797
+ return { ...coreHeader, meta };
798
+ }
799
+ async function encodeCarHeader(fp) {
800
+ return await encode({
801
+ value: { fp },
802
+ hasher: hasher2,
803
+ codec: dagCodec2
804
+ });
805
+ }
806
+ async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
807
+ const fp = makeCarHeader(done, params.carLog, !!opts.compact);
808
+ const rootBlock = await encodeCarHeader(fp);
809
+ const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
810
+ const cids = [];
811
+ for (const car of cars) {
812
+ const { cid, bytes } = car;
813
+ await params.carStore.save({ cid, bytes });
814
+ cids.push(cid);
815
+ }
816
+ const newDbMeta = { cars: cids };
817
+ await params.WALStore.enqueue(newDbMeta, opts);
818
+ await params.metaStore.save(newDbMeta);
819
+ return { cgrp: cids, header: fp };
820
+ }
821
+ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
822
+ const carFiles = [];
823
+ threshold = threshold || 1e3 * 1e3;
824
+ let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
825
+ clonedt.putSync(rootBlock.cid, rootBlock.bytes);
826
+ let newsize = CBW.blockLength(toCIDBlock(rootBlock));
827
+ let cidRootBlock = rootBlock;
828
+ for (const { cid, bytes } of t.entries()) {
829
+ newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
830
+ if (newsize >= threshold) {
831
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
832
+ clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
833
+ clonedt.putSync(cid, bytes);
834
+ cidRootBlock = { cid, bytes };
835
+ newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
836
+ } else {
837
+ clonedt.putSync(cid, bytes);
838
+ }
839
+ }
840
+ carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
841
+ return carFiles;
842
+ }
843
+
844
+ // src/blockstore/loader.ts
845
+ import { sha256 as hasher3 } from "multiformats/hashes/sha2";
846
+
847
+ // src/blockstore/task-manager.ts
848
+ var TaskManager = class {
849
+ constructor(sthis, callback) {
850
+ this.eventsWeHandled = /* @__PURE__ */ new Set();
851
+ this.queue = [];
852
+ this.isProcessing = false;
853
+ this.logger = ensureLogger(sthis, "TaskManager");
854
+ this.callback = callback;
855
+ }
856
+ async handleEvent(cid, parents, dbMeta) {
857
+ for (const parent of parents) {
858
+ this.eventsWeHandled.add(parent.toString());
859
+ }
860
+ this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
861
+ this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
862
+ void this.processQueue();
863
+ }
864
+ async processQueue() {
865
+ if (this.isProcessing) return;
866
+ this.isProcessing = true;
867
+ const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
868
+ const first = filteredQueue[0];
869
+ if (!first) {
870
+ return;
871
+ }
872
+ try {
873
+ await this.callback(first.dbMeta);
874
+ this.eventsWeHandled.add(first.cid);
875
+ this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
876
+ } catch (err) {
877
+ if (first.retries++ > 3) {
878
+ this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
879
+ this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
880
+ }
881
+ await new Promise((resolve) => setTimeout(resolve, 50));
882
+ throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
883
+ } finally {
884
+ this.isProcessing = false;
885
+ if (this.queue.length > 0) {
886
+ void this.processQueue();
887
+ }
888
+ }
835
889
  }
836
890
  };
891
+
892
+ // src/blockstore/loader.ts
893
+ function carLogIncludesGroup(list, cids) {
894
+ return list.some((arr) => {
895
+ return arr.toString() === cids.toString();
896
+ });
897
+ }
898
+ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
899
+ const byString = /* @__PURE__ */ new Map();
900
+ for (const cid of list) {
901
+ if (remove.has(cid.toString())) continue;
902
+ byString.set(cid.toString(), cid);
903
+ }
904
+ return [...byString.values()];
905
+ }
837
906
  var Loader = class {
838
- constructor(name, ebOpts) {
907
+ constructor(name, ebOpts, sthis) {
839
908
  this.commitQueue = new CommitQueue();
840
909
  this.isCompacting = false;
841
910
  this.carReaders = /* @__PURE__ */ new Map();
@@ -845,9 +914,11 @@ var Loader = class {
845
914
  this.getBlockCache = /* @__PURE__ */ new Map();
846
915
  this.seenMeta = /* @__PURE__ */ new Set();
847
916
  this.writeLimit = pLimit(1);
848
- this.onceReady = new ResolveOnce();
917
+ this.onceReady = new ResolveOnce2();
849
918
  this.name = name;
919
+ this.sthis = sthis;
850
920
  this.ebOpts = defaultedBlockstoreRuntime(
921
+ sthis,
851
922
  {
852
923
  ...ebOpts,
853
924
  name
@@ -855,34 +926,42 @@ var Loader = class {
855
926
  "Loader"
856
927
  );
857
928
  this.logger = this.ebOpts.logger;
929
+ this.taskManager = new TaskManager(sthis, async (dbMeta) => {
930
+ await this.handleDbMetasFromStore([dbMeta]);
931
+ });
858
932
  }
859
933
  // readonly id = uuidv4();
934
+ async keyBag() {
935
+ return getKeyBag(this.sthis, this.ebOpts.keyBag);
936
+ }
860
937
  async carStore() {
861
938
  return this.ebOpts.storeRuntime.makeDataStore(this);
862
939
  }
863
940
  async fileStore() {
864
941
  return this.ebOpts.storeRuntime.makeDataStore(this);
865
942
  }
866
- async remoteWAL() {
867
- return this.ebOpts.storeRuntime.makeRemoteWAL(this);
943
+ async WALStore() {
944
+ return this.ebOpts.storeRuntime.makeWALStore(this);
868
945
  }
869
946
  async metaStore() {
870
947
  return this.ebOpts.storeRuntime.makeMetaStore(this);
871
948
  }
872
949
  async ready() {
873
950
  return this.onceReady.once(async () => {
874
- const metas = this.ebOpts.meta ? [this.ebOpts.meta] : await (await this.metaStore()).load("main");
875
- if (metas) {
951
+ const metas = await (await this.metaStore()).load();
952
+ if (this.ebOpts.meta) {
953
+ await this.handleDbMetasFromStore([this.ebOpts.meta]);
954
+ } else if (metas) {
876
955
  await this.handleDbMetasFromStore(metas);
877
956
  }
878
957
  });
879
958
  }
880
959
  async close() {
881
- const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.remoteWAL()]);
960
+ const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
882
961
  await Promise.all(toClose.map((store) => store.close()));
883
962
  }
884
963
  async destroy() {
885
- const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.remoteWAL()]);
964
+ const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
886
965
  await Promise.all(toDestroy.map((store) => store.destroy()));
887
966
  }
888
967
  // async snapToCar(carCid: AnyLink | string) {
@@ -896,6 +975,7 @@ var Loader = class {
896
975
  // await this._applyCarHeader(carHeader, true)
897
976
  // }
898
977
  async handleDbMetasFromStore(metas) {
978
+ this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
899
979
  for (const meta of metas) {
900
980
  await this.writeLimit(async () => {
901
981
  await this.mergeDbMetaIntoClock(meta);
@@ -908,10 +988,7 @@ var Loader = class {
908
988
  }
909
989
  if (this.seenMeta.has(meta.cars.toString())) return;
910
990
  this.seenMeta.add(meta.cars.toString());
911
- if (meta.key) {
912
- await this.setKey(meta.key);
913
- }
914
- if (carLogIncludesGroup2(this.carLog, meta.cars)) {
991
+ if (carLogIncludesGroup(this.carLog, meta.cars)) {
915
992
  return;
916
993
  }
917
994
  const carHeader = await this.loadCarHeaderFromMeta(meta);
@@ -920,45 +997,59 @@ var Loader = class {
920
997
  this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
921
998
  await this.ebOpts.applyMeta?.(carHeader.meta);
922
999
  }
923
- async ingestKeyFromMeta(meta) {
924
- const { key } = meta;
925
- if (key) {
926
- await this.setKey(key);
927
- }
928
- }
1000
+ // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
1001
+ // const { key } = meta;
1002
+ // if (key) {
1003
+ // await this.setKey(key);
1004
+ // }
1005
+ // }
929
1006
  async loadCarHeaderFromMeta({ cars: cids }) {
930
1007
  const reader = await this.loadCar(cids[0]);
931
1008
  return await parseCarFile(reader, this.logger);
932
1009
  }
933
- async _getKey() {
934
- if (this.key) return this.key;
935
- if (!this.ebOpts.public) {
936
- await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
937
- }
938
- return this.key || void 0;
939
- }
940
- async commitFiles(t, done, opts = { noLoader: false, compact: false }) {
941
- return this.commitQueue.enqueue(() => this._commitInternalFiles(t, done, opts));
942
- }
943
- // can these skip the queue? or have a file queue?
944
- async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
1010
+ // async _getKey(): Promise<string | undefined> {
1011
+ // if (this.key) return this.key;
1012
+ // // generate a random key
1013
+ // if (!this.ebOpts.public) {
1014
+ // await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
1015
+ // }
1016
+ // return this.key || undefined;
1017
+ // }
1018
+ async commitFiles(t, done) {
945
1019
  await this.ready();
946
- const { files: roots } = this.makeFileCarHeader(done);
947
- const cids = [];
948
- const cars = await this.prepareCarFilesFiles(roots, t, !!opts.public);
949
- for (const car of cars) {
950
- const { cid, bytes } = car;
951
- await (await this.fileStore()).save({ cid, bytes });
952
- await (await this.remoteWAL()).enqueueFile(cid, !!opts.public);
953
- cids.push(cid);
954
- }
955
- return cids;
1020
+ const fstore = await this.fileStore();
1021
+ const wstore = await this.WALStore();
1022
+ return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
956
1023
  }
957
- async loadFileCar(cid, isPublic = false) {
958
- return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore, isPublic);
1024
+ async loadFileCar(cid) {
1025
+ return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
959
1026
  }
960
1027
  async commit(t, done, opts = { noLoader: false, compact: false }) {
961
- return this.commitQueue.enqueue(() => this._commitInternal(t, done, opts));
1028
+ await this.ready();
1029
+ const fstore = await this.fileStore();
1030
+ const params = {
1031
+ encoder: (await fstore.keyedCrypto()).codec(),
1032
+ carLog: this.carLog,
1033
+ carStore: fstore,
1034
+ WALStore: await this.WALStore(),
1035
+ metaStore: await this.metaStore()
1036
+ };
1037
+ return this.commitQueue.enqueue(async () => {
1038
+ await this.cacheTransaction(t);
1039
+ const ret = await commit(params, t, done, opts);
1040
+ await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
1041
+ return ret.cgrp;
1042
+ });
1043
+ }
1044
+ async updateCarLog(cids, fp, compact) {
1045
+ if (compact) {
1046
+ const previousCompactCid = fp.compact[fp.compact.length - 1];
1047
+ fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1048
+ this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1049
+ await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
1050
+ } else {
1051
+ this.carLog.unshift(cids);
1052
+ }
962
1053
  }
963
1054
  async cacheTransaction(t) {
964
1055
  for await (const block of t.entries()) {
@@ -978,79 +1069,6 @@ var Loader = class {
978
1069
  }
979
1070
  }
980
1071
  }
981
- async _commitInternal(t, done, opts = { noLoader: false, compact: false }) {
982
- await this.ready();
983
- const fp = this.makeCarHeader(done, this.carLog, !!opts.compact);
984
- const rootBlock = await encodeCarHeader(fp);
985
- const cars = await this.prepareCarFiles(rootBlock, t, !!opts.public);
986
- const cids = [];
987
- for (const car of cars) {
988
- const { cid, bytes } = car;
989
- await (await this.carStore()).save({ cid, bytes });
990
- cids.push(cid);
991
- }
992
- await this.cacheTransaction(t);
993
- const newDbMeta = { cars: cids, key: this.key || null };
994
- await (await this.remoteWAL()).enqueue(newDbMeta, opts);
995
- await (await this.metaStore()).save(newDbMeta);
996
- await this.updateCarLog(cids, fp, !!opts.compact);
997
- return cids;
998
- }
999
- async prepareCarFilesFiles(roots, t, isPublic) {
1000
- const theKey = isPublic ? null : await this._getKey();
1001
- const car = theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, roots[0], t) : await encodeCarFile(roots, t);
1002
- return [car];
1003
- }
1004
- async prepareCarFiles(rootBlock, t, isPublic) {
1005
- const theKey = isPublic ? void 0 : await this._getKey();
1006
- const carFiles = [];
1007
- const threshold = this.ebOpts.threshold || 1e3 * 1e3;
1008
- let clonedt = new CarTransaction(t.parent, { add: false });
1009
- clonedt.putSync(rootBlock.cid, rootBlock.bytes);
1010
- let newsize = CBW2.blockLength(toCIDBlock(rootBlock));
1011
- let cidRootBlock = rootBlock;
1012
- for (const { cid, bytes } of t.entries()) {
1013
- newsize += CBW2.blockLength(toCIDBlock({ cid, bytes }));
1014
- if (newsize >= threshold) {
1015
- carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
1016
- clonedt = new CarTransaction(t.parent, { add: false });
1017
- clonedt.putSync(cid, bytes);
1018
- cidRootBlock = { cid, bytes };
1019
- newsize = CBW2.blockLength(toCIDBlock({ cid, bytes }));
1020
- } else {
1021
- clonedt.putSync(cid, bytes);
1022
- }
1023
- }
1024
- carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
1025
- return carFiles;
1026
- }
1027
- async createCarFile(theKey, cid, t) {
1028
- try {
1029
- return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, cid, t) : await encodeCarFile([cid], t);
1030
- } catch (e) {
1031
- console.error("error creating car file", e);
1032
- throw e;
1033
- }
1034
- }
1035
- makeFileCarHeader(result) {
1036
- const files = [];
1037
- for (const [, meta] of Object.entries(result.files || {})) {
1038
- if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
1039
- files.push(meta.cid);
1040
- }
1041
- }
1042
- return { ...result, files };
1043
- }
1044
- async updateCarLog(cids, fp, compact) {
1045
- if (compact) {
1046
- const previousCompactCid = fp.compact[fp.compact.length - 1];
1047
- fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
1048
- this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
1049
- await this.removeCidsForCompact(previousCompactCid[0]);
1050
- } else {
1051
- this.carLog.unshift(cids);
1052
- }
1053
- }
1054
1072
  async removeCidsForCompact(cid) {
1055
1073
  const carHeader = await this.loadCarHeaderFromMeta({
1056
1074
  cars: [cid]
@@ -1069,9 +1087,9 @@ var Loader = class {
1069
1087
  // await this.remoteWAL!.enqueue(dbMeta, { public: false })
1070
1088
  // }
1071
1089
  // }
1072
- async *entries(cache3 = true) {
1090
+ async *entries(cache2 = true) {
1073
1091
  await this.ready();
1074
- if (cache3) {
1092
+ if (cache2) {
1075
1093
  for (const [, block] of this.getBlockCache) {
1076
1094
  yield block;
1077
1095
  }
@@ -1153,10 +1171,6 @@ var Loader = class {
1153
1171
  }
1154
1172
  return got;
1155
1173
  }
1156
- makeCarHeader(meta, cars, compact = false) {
1157
- const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
1158
- return { ...coreHeader, meta };
1159
- }
1160
1174
  async loadCar(cid) {
1161
1175
  if (!this.carStore) {
1162
1176
  throw this.logger.Error().Msg("car store not initialized").AsError();
@@ -1164,72 +1178,52 @@ var Loader = class {
1164
1178
  const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
1165
1179
  return loaded;
1166
1180
  }
1167
- //What if instead it returns an Array of CarHeader
1168
- async storesLoadCar(cid, local, remote, publicFiles) {
1181
+ async makeDecoderAndCarReader(cid, local, remote) {
1169
1182
  const cidsString = cid.toString();
1170
- if (!this.carReaders.has(cidsString)) {
1171
- this.carReaders.set(
1172
- cidsString,
1173
- (async () => {
1174
- let loadedCar = void 0;
1175
- try {
1176
- this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1177
- loadedCar = await local.load(cid);
1178
- this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1179
- } catch (e) {
1180
- if (remote) {
1181
- const remoteCar = await remote.load(cid);
1182
- if (remoteCar) {
1183
- this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1184
- await local.save(remoteCar);
1185
- loadedCar = remoteCar;
1186
- }
1187
- } else {
1188
- this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1189
- }
1190
- }
1191
- if (!loadedCar) {
1192
- throw this.logger.Error().Url(local.url).Str("cid", cidsString).Msg("missing car files").AsError();
1193
- }
1194
- const rawReader = await CarReader.fromBytes(loadedCar.bytes);
1195
- const readerP = publicFiles ? Promise.resolve(rawReader) : this.ensureDecryptedReader(rawReader);
1196
- const cachedReaderP = readerP.then(async (reader) => {
1197
- await this.cacheCarReader(cidsString, reader).catch(() => {
1198
- return;
1199
- });
1200
- return reader;
1201
- });
1202
- this.carReaders.set(cidsString, cachedReaderP);
1203
- return readerP;
1204
- })().catch((e) => {
1205
- this.carReaders.delete(cidsString);
1206
- throw e;
1207
- })
1208
- );
1183
+ let loadedCar = void 0;
1184
+ let activeStore = local;
1185
+ try {
1186
+ this.logger.Debug().Str("cid", cidsString).Msg("loading car");
1187
+ loadedCar = await local.load(cid);
1188
+ this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
1189
+ } catch (e) {
1190
+ if (remote) {
1191
+ const remoteCar = await remote.load(cid);
1192
+ if (remoteCar) {
1193
+ this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
1194
+ await local.save(remoteCar);
1195
+ loadedCar = remoteCar;
1196
+ activeStore = remote;
1197
+ }
1198
+ } else {
1199
+ this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
1200
+ }
1209
1201
  }
1210
- return this.carReaders.get(cidsString);
1211
- }
1212
- async ensureDecryptedReader(reader) {
1213
- const theKey = await this._getKey();
1214
- if (this.ebOpts.public || !(theKey && this.ebOpts.crypto)) {
1215
- return reader;
1202
+ if (!loadedCar) {
1203
+ throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
1216
1204
  }
1217
- const { blocks, root: root3 } = await decodeEncryptedCar(this.logger, this.ebOpts.crypto, theKey, reader);
1218
- return {
1219
- getRoots: () => [root3],
1220
- get: blocks.get.bind(blocks),
1221
- blocks: blocks.entries.bind(blocks)
1222
- };
1205
+ const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
1206
+ const rawReader = await CarReader.fromBytes(bytes.value);
1207
+ const readerP = Promise.resolve(rawReader);
1208
+ const cachedReaderP = readerP.then(async (reader) => {
1209
+ await this.cacheCarReader(cidsString, reader).catch((e) => {
1210
+ this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
1211
+ return;
1212
+ });
1213
+ return reader;
1214
+ });
1215
+ this.carReaders.set(cidsString, cachedReaderP);
1216
+ return readerP;
1223
1217
  }
1224
- async setKey(key) {
1225
- if (this.key && this.key !== key)
1226
- throw this.logger.Error().Str("this.key", this.key).Str("key", key).Msg("setting key").AsError();
1227
- this.key = key;
1228
- const encoder = new TextEncoder();
1229
- const data = encoder.encode(key);
1230
- const hashBuffer = await this.ebOpts.crypto.digestSHA256(data);
1231
- const hashArray = Array.from(new Uint8Array(hashBuffer));
1232
- this.keyId = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
1218
+ //What if instead it returns an Array of CarHeader
1219
+ async storesLoadCar(cid, local, remote) {
1220
+ const cidsString = cid.toString();
1221
+ let dacr = this.carReaders.get(cidsString);
1222
+ if (!dacr) {
1223
+ dacr = this.makeDecoderAndCarReader(cid, local, remote);
1224
+ this.carReaders.set(cidsString, dacr);
1225
+ }
1226
+ return dacr;
1233
1227
  }
1234
1228
  async getMoreReaders(cids) {
1235
1229
  const limit = pLimit(5);
@@ -1238,127 +1232,441 @@ var Loader = class {
1238
1232
  }
1239
1233
  };
1240
1234
 
1241
- // src/blockstore/store.ts
1242
- function guardVersion(url) {
1243
- if (!url.searchParams.has("version")) {
1244
- return Result2.Err(`missing version: ${url.toString()}`);
1235
+ // src/runtime/keyed-crypto.ts
1236
+ var keyed_crypto_exports = {};
1237
+ __export(keyed_crypto_exports, {
1238
+ BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
1239
+ keyedCryptoFactory: () => keyedCryptoFactory
1240
+ });
1241
+ import { base58btc as base58btc2 } from "multiformats/bases/base58";
1242
+ import { sha256 as hasher4 } from "multiformats/hashes/sha2";
1243
+ import * as CBOR from "cborg";
1244
+ var generateIV = {
1245
+ random: {
1246
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1247
+ calc: async (ko, crypto, data) => {
1248
+ return crypto.randomBytes(ko.ivLength);
1249
+ },
1250
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1251
+ verify: async (ko, crypto, iv, data) => {
1252
+ return true;
1253
+ }
1254
+ },
1255
+ hash: {
1256
+ calc: async (ko, crypto, data) => {
1257
+ const hash = await hasher4.digest(data);
1258
+ const hashBytes = new Uint8Array(hash.bytes);
1259
+ const hashArray = new Uint8Array(ko.ivLength);
1260
+ for (let i = 0; i < hashBytes.length; i++) {
1261
+ hashArray[i % ko.ivLength] ^= hashBytes[i];
1262
+ }
1263
+ return hashArray;
1264
+ },
1265
+ verify: async function(ko, crypto, iv, data) {
1266
+ return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
1267
+ }
1245
1268
  }
1246
- return Result2.Ok(url);
1247
- }
1248
- var VersionedStore = class {
1249
- constructor(name, url, logger) {
1250
- this._onStarted = [];
1251
- this._onClosed = [];
1252
- this.name = name;
1269
+ };
1270
+ function getGenerateIVFn(url, opts) {
1271
+ const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
1272
+ return generateIV[ivhash] || generateIV["hash"];
1273
+ }
1274
+ var BlockIvKeyIdCodec = class {
1275
+ constructor(ko, iv, opts) {
1276
+ this.code = 3147065;
1277
+ this.name = "Fireproof@encrypted-block:aes-gcm";
1278
+ this.ko = ko;
1279
+ this.iv = iv;
1280
+ this.opts = opts || {};
1281
+ }
1282
+ async encode(data) {
1283
+ const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
1284
+ const { iv } = this.ko.algo(calcIv);
1285
+ const fprt = await this.ko.fingerPrint();
1286
+ const keyId = base58btc2.decode(fprt);
1287
+ this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
1288
+ return CBOR.encode({
1289
+ iv,
1290
+ keyId,
1291
+ data: await this.ko._encrypt({ iv, bytes: data })
1292
+ });
1293
+ }
1294
+ async decode(abytes) {
1295
+ let bytes;
1296
+ if (abytes instanceof Uint8Array) {
1297
+ bytes = abytes;
1298
+ } else {
1299
+ bytes = new Uint8Array(abytes);
1300
+ }
1301
+ const { iv, keyId, data } = CBOR.decode(bytes);
1302
+ const fprt = await this.ko.fingerPrint();
1303
+ this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
1304
+ if (base58btc2.encode(keyId) !== fprt) {
1305
+ throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
1306
+ }
1307
+ const result = await this.ko._decrypt({ iv, bytes: data });
1308
+ if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
1309
+ throw this.ko.logger.Error().Msg("iv missmatch").AsError();
1310
+ }
1311
+ return result;
1312
+ }
1313
+ };
1314
+ var keyedCrypto = class {
1315
+ constructor(url, key, cyopt, sthis) {
1316
+ this.ivLength = 12;
1317
+ this.isEncrypting = true;
1318
+ this.logger = ensureLogger(sthis, "keyedCrypto");
1319
+ this.crypto = cyopt;
1320
+ this.key = key;
1253
1321
  this.url = url;
1254
- this.logger = logger;
1255
1322
  }
1256
- onStarted(fn) {
1257
- this._onStarted.push(fn);
1323
+ fingerPrint() {
1324
+ return Promise.resolve(this.key.fingerPrint);
1258
1325
  }
1259
- onClosed(fn) {
1260
- this._onClosed.push(fn);
1326
+ codec(iv, opts) {
1327
+ return new BlockIvKeyIdCodec(this, iv, opts);
1328
+ }
1329
+ algo(iv) {
1330
+ return {
1331
+ name: "AES-GCM",
1332
+ iv: iv || this.crypto.randomBytes(this.ivLength),
1333
+ tagLength: 128
1334
+ };
1335
+ }
1336
+ async _decrypt(data) {
1337
+ this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
1338
+ return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
1339
+ }
1340
+ async _encrypt(data) {
1341
+ this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
1342
+ const a = this.algo(data.iv);
1343
+ return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
1261
1344
  }
1262
1345
  };
1263
- var textEncoder = new TextEncoder();
1264
- var textDecoder = new TextDecoder();
1265
- var MetaStore = class extends VersionedStore {
1266
- constructor(name, url, logger, gateway) {
1267
- super(name, url, ensureLogger(logger, "MetaStore", {}));
1268
- this.tag = "header-base";
1269
- this.gateway = gateway;
1270
- }
1271
- makeHeader({ cars, key }) {
1272
- const toEncode = { cars };
1273
- if (key) toEncode.key = key;
1274
- return format(toEncode);
1275
- }
1276
- parseHeader(headerData) {
1277
- const got = parse(headerData);
1278
- return got;
1346
+ var nullCodec = class {
1347
+ constructor() {
1348
+ this.code = 0;
1349
+ this.name = "Fireproof@unencrypted-block";
1279
1350
  }
1280
- async start() {
1281
- this.logger.Debug().Msg("starting");
1282
- const res = await this.gateway.start(this.url);
1283
- if (res.isErr()) {
1284
- return res;
1285
- }
1286
- this._onStarted.forEach((fn) => fn());
1287
- return guardVersion(this.url);
1351
+ encode(data) {
1352
+ return data;
1288
1353
  }
1289
- async load(branch) {
1290
- this.logger.Debug().Str("branch", branch || "").Msg("loading");
1291
- const url = await this.gateway.buildUrl(this.url, branch || "main");
1292
- if (url.isErr()) {
1293
- throw this.logger.Error().Result("buidUrl", url).Str("branch", branch || "").Url(this.url).Msg("got error from gateway.buildUrl").AsError();
1294
- }
1295
- const bytes = await this.gateway.get(url.Ok());
1296
- if (bytes.isErr()) {
1297
- if (isNotFoundError(bytes)) {
1298
- return void 0;
1354
+ decode(data) {
1355
+ return data;
1356
+ }
1357
+ };
1358
+ var noCrypto = class {
1359
+ constructor(url, cyrt, sthis) {
1360
+ this.ivLength = 0;
1361
+ this.code = 0;
1362
+ this.name = "Fireproof@unencrypted-block";
1363
+ this.isEncrypting = false;
1364
+ this._fingerPrint = "noCrypto:" + Math.random();
1365
+ this.logger = ensureLogger(sthis, "noCrypto");
1366
+ this.crypto = cyrt;
1367
+ this.url = url;
1368
+ }
1369
+ fingerPrint() {
1370
+ return Promise.resolve(this._fingerPrint);
1371
+ }
1372
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1373
+ codec(iv) {
1374
+ return new nullCodec();
1375
+ }
1376
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
1377
+ algo(iv) {
1378
+ return {
1379
+ name: "noCrypto",
1380
+ iv: new Uint8Array(),
1381
+ tagLength: 0
1382
+ };
1383
+ }
1384
+ _decrypt() {
1385
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1386
+ }
1387
+ _encrypt() {
1388
+ throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
1389
+ }
1390
+ };
1391
+ async function keyedCryptoFactory(url, kb, sthis) {
1392
+ const storekey = url.getParam("storekey");
1393
+ if (storekey && storekey !== "insecure") {
1394
+ let rkey = await kb.getNamedKey(storekey, true);
1395
+ if (rkey.isErr()) {
1396
+ try {
1397
+ rkey = await kb.toKeyWithFingerPrint(storekey);
1398
+ } catch (e) {
1399
+ throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
1299
1400
  }
1300
- throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
1301
- }
1302
- try {
1303
- return [this.parseHeader(textDecoder.decode(bytes.Ok()))];
1304
- } catch (e) {
1305
- throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
1306
1401
  }
1402
+ return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
1307
1403
  }
1308
- async save(meta, branch = "main") {
1309
- this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
1310
- const bytes = this.makeHeader(meta);
1311
- const url = await this.gateway.buildUrl(this.url, branch);
1312
- if (url.isErr()) {
1313
- throw this.logger.Error().Err(url.Err()).Str("branch", branch).Url(this.url).Msg("got error from gateway.buildUrl").AsError();
1314
- }
1315
- const res = await this.gateway.put(url.Ok(), textEncoder.encode(bytes));
1404
+ return new noCrypto(url, kb.rt.crypto, sthis);
1405
+ }
1406
+
1407
+ // src/blockstore/fragment-gateway.ts
1408
+ import { Result as Result3 } from "@adviser/cement";
1409
+ import { base58btc as base58btc3 } from "multiformats/bases/base58";
1410
+ import { encode as encode3, decode as decode3 } from "cborg";
1411
+ function getFragSize(url) {
1412
+ const fragSize = url.getParam("fragSize");
1413
+ let ret = 0;
1414
+ if (fragSize) {
1415
+ ret = parseInt(fragSize);
1416
+ }
1417
+ if (isNaN(ret) || ret <= 0) {
1418
+ ret = 0;
1419
+ }
1420
+ return ret;
1421
+ }
1422
+ async function getFrags(url, innerGW, headerSize, logger) {
1423
+ const fragSize = getFragSize(url);
1424
+ if (!fragSize) {
1425
+ const res = await innerGW.get(url);
1316
1426
  if (res.isErr()) {
1317
- throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
1427
+ return [res];
1428
+ }
1429
+ const data = res.unwrap();
1430
+ return [
1431
+ Result3.Ok({
1432
+ fid: new Uint8Array(0),
1433
+ ofs: 0,
1434
+ len: data.length,
1435
+ data
1436
+ })
1437
+ ];
1438
+ }
1439
+ const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
1440
+ if (firstRaw.isErr()) {
1441
+ return [firstRaw];
1442
+ }
1443
+ const firstFragment = decode3(firstRaw.unwrap());
1444
+ const blockSize = firstFragment.data.length;
1445
+ const ops = [Promise.resolve(Result3.Ok(firstFragment))];
1446
+ const fidStr = base58btc3.encode(firstFragment.fid);
1447
+ const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
1448
+ for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
1449
+ ops.push(
1450
+ (async (furl, ofs2) => {
1451
+ const raw2 = await innerGW.get(furl);
1452
+ if (raw2.isErr()) {
1453
+ return raw2;
1454
+ }
1455
+ const fragment = decode3(raw2.unwrap());
1456
+ if (base58btc3.encode(fragment.fid) !== fidStr) {
1457
+ return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
1458
+ }
1459
+ if (fragment.ofs !== ofs2) {
1460
+ return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
1461
+ }
1462
+ return Result3.Ok(fragment);
1463
+ })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
1464
+ );
1465
+ }
1466
+ return Promise.all(ops);
1467
+ }
1468
+ var FragmentGateway = class {
1469
+ constructor(sthis, innerGW) {
1470
+ this.fidLength = 4;
1471
+ this.headerSize = 32;
1472
+ this.sthis = ensureSuperLog(sthis, "FragmentGateway");
1473
+ this.logger = this.sthis.logger;
1474
+ this.innerGW = innerGW;
1475
+ }
1476
+ slicer(url, body) {
1477
+ const fragSize = getFragSize(url);
1478
+ if (!fragSize) {
1479
+ return [this.innerGW.put(url, body)];
1480
+ }
1481
+ const blocksize = fragSize - this.headerSize;
1482
+ if (blocksize <= 0) {
1483
+ throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
1484
+ }
1485
+ const ops = [];
1486
+ const fid = this.sthis.nextId(this.fidLength);
1487
+ const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
1488
+ for (let ofs = 0; ofs < body.length; ofs += blocksize) {
1489
+ const block = encode3({
1490
+ fid: fid.bin,
1491
+ ofs,
1492
+ len: body.length,
1493
+ data: body.slice(ofs, ofs + blocksize)
1494
+ });
1495
+ if (block.length > fragSize) {
1496
+ throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
1497
+ }
1498
+ ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
1499
+ }
1500
+ return ops;
1501
+ }
1502
+ buildUrl(baseUrl, key) {
1503
+ return this.innerGW.buildUrl(baseUrl, key);
1504
+ }
1505
+ async destroy(iurl) {
1506
+ return this.innerGW.destroy(iurl);
1507
+ }
1508
+ async start(url) {
1509
+ this.headerSize = encode3({
1510
+ fid: this.sthis.nextId(this.fidLength).bin,
1511
+ ofs: 1024 * 1024,
1512
+ // 32bit
1513
+ len: 16 * 1024 * 1024,
1514
+ // 32bit
1515
+ data: new Uint8Array(1024)
1516
+ }).length - 1024;
1517
+ return this.innerGW.start(url);
1518
+ }
1519
+ async close(url) {
1520
+ return this.innerGW.close(url);
1521
+ }
1522
+ async put(url, body) {
1523
+ await Promise.all(this.slicer(url, body));
1524
+ return Result3.Ok(void 0);
1525
+ }
1526
+ async get(url) {
1527
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1528
+ let buffer = void 0;
1529
+ for (const rfrag of rfrags) {
1530
+ if (rfrag.isErr()) {
1531
+ return Result3.Err(rfrag.Err());
1532
+ }
1533
+ const frag = rfrag.Ok();
1534
+ buffer = buffer || new Uint8Array(frag.len);
1535
+ buffer.set(frag.data, frag.ofs);
1318
1536
  }
1319
- return res.Ok();
1537
+ return Result3.Ok(buffer || new Uint8Array(0));
1320
1538
  }
1321
- async close() {
1322
- await this.gateway.close(this.url);
1323
- this._onClosed.forEach((fn) => fn());
1324
- return Result2.Ok(void 0);
1539
+ async subscribe(url, callback) {
1540
+ if (this.innerGW.subscribe) {
1541
+ return this.innerGW.subscribe(url, callback);
1542
+ } else {
1543
+ let lastData = void 0;
1544
+ let interval = 100;
1545
+ const fetchData = async () => {
1546
+ const result = await this.innerGW.get(url);
1547
+ if (result.isOk()) {
1548
+ const data = result.Ok();
1549
+ if (!lastData || !data.every((value, index2) => lastData && value === lastData[index2])) {
1550
+ lastData = data;
1551
+ callback(data);
1552
+ interval = 100;
1553
+ } else {
1554
+ interval *= 2;
1555
+ }
1556
+ }
1557
+ timeoutId = setTimeout(fetchData, interval);
1558
+ };
1559
+ let timeoutId = setTimeout(fetchData, interval);
1560
+ return Result3.Ok(() => {
1561
+ clearTimeout(timeoutId);
1562
+ });
1563
+ }
1325
1564
  }
1326
- async destroy() {
1327
- return this.gateway.destroy(this.url);
1565
+ async delete(url) {
1566
+ const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
1567
+ for (const rfrag of rfrags) {
1568
+ if (rfrag.isErr()) {
1569
+ return Result3.Err(rfrag.Err());
1570
+ }
1571
+ const frag = rfrag.Ok();
1572
+ const fidStr = base58btc3.encode(frag.fid);
1573
+ const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
1574
+ await this.innerGW.delete(fragUrl);
1575
+ }
1576
+ return Result3.Ok(void 0);
1328
1577
  }
1329
1578
  };
1330
- var DataStore = class extends VersionedStore {
1331
- constructor(name, url, logger, gateway) {
1332
- super(
1333
- name,
1334
- url,
1335
- ensureLogger(logger, "DataStore", {
1336
- url: () => url.toString()
1337
- })
1338
- );
1339
- this.tag = "car-base";
1340
- this.gateway = gateway;
1579
+
1580
+ // src/blockstore/store.ts
1581
+ function guardVersion(url) {
1582
+ if (!url.hasParam("version")) {
1583
+ return Result4.Err(`missing version: ${url.toString()}`);
1584
+ }
1585
+ return Result4.Ok(url);
1586
+ }
1587
+ var BaseStoreImpl = class {
1588
+ constructor(name, url, opts, sthis, logger) {
1589
+ this._onStarted = [];
1590
+ this._onClosed = [];
1591
+ this.name = name;
1592
+ this._url = url;
1593
+ this.keybag = opts.keybag;
1594
+ this.sthis = sthis;
1595
+ this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
1596
+ this.gateway = new FragmentGateway(this.sthis, opts.gateway);
1597
+ this.loader = opts.loader;
1598
+ }
1599
+ url() {
1600
+ return this._url;
1601
+ }
1602
+ onStarted(fn) {
1603
+ this._onStarted.push(fn);
1604
+ }
1605
+ onClosed(fn) {
1606
+ this._onClosed.push(fn);
1607
+ }
1608
+ async keyedCrypto() {
1609
+ return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
1341
1610
  }
1342
1611
  async start() {
1343
- this.logger.Debug().Msg("starting-gateway");
1344
- const res = await this.gateway.start(this.url);
1612
+ this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
1613
+ this._url = this._url.build().setParam("store", this.storeType).URI();
1614
+ const res = await this.gateway.start(this._url);
1345
1615
  if (res.isErr()) {
1346
1616
  this.logger.Error().Result("gw-start", res).Msg("started-gateway");
1347
1617
  return res;
1348
1618
  }
1349
- this._onStarted.forEach((fn) => fn());
1350
- const version = guardVersion(this.url);
1619
+ this._url = res.Ok();
1620
+ const kb = await this.keybag();
1621
+ const skRes = await kb.ensureKeyFromUrl(this._url, () => {
1622
+ const idx = this._url.getParam("index");
1623
+ const storeKeyName = [this.name];
1624
+ if (idx) {
1625
+ storeKeyName.push(idx);
1626
+ }
1627
+ storeKeyName.push(this.storeType);
1628
+ return storeKeyName.join(":");
1629
+ });
1630
+ if (skRes.isErr()) {
1631
+ return skRes;
1632
+ }
1633
+ this._url = skRes.Ok();
1634
+ const version = guardVersion(this._url);
1351
1635
  if (version.isErr()) {
1352
1636
  this.logger.Error().Result("version", version).Msg("guardVersion");
1353
1637
  await this.close();
1354
1638
  return version;
1355
1639
  }
1640
+ if (this.ready) {
1641
+ const fn = this.ready.bind(this);
1642
+ const ready = await exception2Result(fn);
1643
+ if (ready.isErr()) {
1644
+ await this.close();
1645
+ return ready;
1646
+ }
1647
+ }
1648
+ this._onStarted.forEach((fn) => fn());
1356
1649
  this.logger.Debug().Msg("started");
1357
1650
  return version;
1358
1651
  }
1652
+ };
1653
+ var DataStoreImpl = class extends BaseStoreImpl {
1654
+ // readonly tag: string = "car-base";
1655
+ constructor(sthis, name, url, opts) {
1656
+ super(
1657
+ name,
1658
+ url,
1659
+ {
1660
+ ...opts
1661
+ },
1662
+ sthis,
1663
+ ensureLogger(sthis, "DataStoreImpl")
1664
+ );
1665
+ this.storeType = "data";
1666
+ }
1359
1667
  async load(cid) {
1360
1668
  this.logger.Debug().Any("cid", cid).Msg("loading");
1361
- const url = await this.gateway.buildUrl(this.url, cid.toString());
1669
+ const url = await this.gateway.buildUrl(this.url(), cid.toString());
1362
1670
  if (url.isErr()) {
1363
1671
  throw this.logger.Error().Err(url.Err()).Str("cid", cid.toString()).Msg("got error from gateway.buildUrl").AsError();
1364
1672
  }
@@ -1371,7 +1679,7 @@ var DataStore = class extends VersionedStore {
1371
1679
  // eslint-disable-next-line @typescript-eslint/no-unused-vars
1372
1680
  async save(car, opts) {
1373
1681
  this.logger.Debug().Any("cid", car.cid.toString()).Msg("saving");
1374
- const url = await this.gateway.buildUrl(this.url, car.cid.toString());
1682
+ const url = await this.gateway.buildUrl(this.url(), car.cid.toString());
1375
1683
  if (url.isErr()) {
1376
1684
  throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError();
1377
1685
  }
@@ -1382,46 +1690,53 @@ var DataStore = class extends VersionedStore {
1382
1690
  return res.Ok();
1383
1691
  }
1384
1692
  async remove(cid) {
1385
- const url = await this.gateway.buildUrl(this.url, cid.toString());
1693
+ const url = await this.gateway.buildUrl(this.url(), cid.toString());
1386
1694
  if (url.isErr()) {
1387
1695
  return url;
1388
1696
  }
1389
1697
  return this.gateway.delete(url.Ok());
1390
1698
  }
1391
1699
  async close() {
1392
- await this.gateway.close(this.url);
1700
+ await this.gateway.close(this.url());
1393
1701
  this._onClosed.forEach((fn) => fn());
1394
- return Result2.Ok(void 0);
1702
+ return Result4.Ok(void 0);
1395
1703
  }
1396
1704
  destroy() {
1397
- return this.gateway.destroy(this.url);
1705
+ return this.gateway.destroy(this.url());
1398
1706
  }
1399
1707
  };
1400
- var RemoteWAL = class extends VersionedStore {
1401
- constructor(loader, url, logger, gateway) {
1402
- super(loader.name, url, ensureLogger(logger, "RemoteWAL"));
1403
- this.tag = "rwal-base";
1404
- this._ready = new ResolveOnce2();
1708
+ var WALStoreImpl = class extends BaseStoreImpl {
1709
+ constructor(loader, url, opts) {
1710
+ super(
1711
+ loader.name,
1712
+ url,
1713
+ {
1714
+ ...opts
1715
+ },
1716
+ loader.sthis,
1717
+ ensureLogger(loader.sthis, "WALStoreImpl")
1718
+ );
1719
+ this.storeType = "wal";
1720
+ this._ready = new ResolveOnce3();
1405
1721
  this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
1406
1722
  this.processing = void 0;
1407
1723
  this.processQueue = new CommitQueue();
1408
- this.loader = loader;
1409
- this.gateway = gateway;
1410
- }
1411
- async ready() {
1412
- return this._ready.once(async () => {
1413
- const walState = await this.load().catch((e) => {
1414
- this.logger.Error().Any("error", e).Msg("error loading wal");
1415
- return void 0;
1724
+ this.ready = async () => {
1725
+ return this._ready.once(async () => {
1726
+ const walState = await this.load().catch((e) => {
1727
+ this.logger.Error().Any("error", e).Msg("error loading wal");
1728
+ return void 0;
1729
+ });
1730
+ if (!walState) {
1731
+ this.walState.operations = [];
1732
+ this.walState.fileOperations = [];
1733
+ } else {
1734
+ this.walState.operations = walState.operations || [];
1735
+ this.walState.fileOperations = walState.fileOperations || [];
1736
+ }
1416
1737
  });
1417
- if (!walState) {
1418
- this.walState.operations = [];
1419
- this.walState.fileOperations = [];
1420
- } else {
1421
- this.walState.operations = walState.operations || [];
1422
- this.walState.fileOperations = walState.fileOperations || [];
1423
- }
1424
- });
1738
+ };
1739
+ this.loader = loader;
1425
1740
  }
1426
1741
  async enqueue(dbMeta, opts) {
1427
1742
  await this.ready();
@@ -1431,19 +1746,23 @@ var RemoteWAL = class extends VersionedStore {
1431
1746
  this.walState.operations.push(dbMeta);
1432
1747
  }
1433
1748
  await this.save(this.walState);
1434
- void this._process();
1749
+ void this.process();
1435
1750
  }
1436
1751
  async enqueueFile(fileCid, publicFile = false) {
1437
1752
  await this.ready();
1438
1753
  this.walState.fileOperations.push({ cid: fileCid, public: publicFile });
1439
1754
  }
1440
- async _process() {
1755
+ async process() {
1441
1756
  await this.ready();
1442
1757
  if (!this.loader.remoteCarStore) return;
1443
1758
  await this.processQueue.enqueue(async () => {
1444
- await this._doProcess();
1759
+ try {
1760
+ await this._doProcess();
1761
+ } catch (e) {
1762
+ this.logger.Error().Any("error", e).Msg("error processing wal");
1763
+ }
1445
1764
  if (this.walState.operations.length || this.walState.fileOperations.length || this.walState.noLoaderOps.length) {
1446
- setTimeout(() => void this._process(), 0);
1765
+ setTimeout(() => void this.process(), 0);
1447
1766
  }
1448
1767
  });
1449
1768
  }
@@ -1461,7 +1780,7 @@ var RemoteWAL = class extends VersionedStore {
1461
1780
  for (const cid of dbMeta.cars) {
1462
1781
  const car = await (await this.loader.carStore()).load(cid);
1463
1782
  if (!car) {
1464
- if (carLogIncludesGroup2(this.loader.carLog, dbMeta.cars))
1783
+ if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
1465
1784
  throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
1466
1785
  } else {
1467
1786
  await throwFalsy(this.loader.remoteCarStore).save(car);
@@ -1476,7 +1795,7 @@ var RemoteWAL = class extends VersionedStore {
1476
1795
  for (const cid of dbMeta.cars) {
1477
1796
  const car = await (await this.loader.carStore()).load(cid).catch(() => null);
1478
1797
  if (!car) {
1479
- if (carLogIncludesGroup2(this.loader.carLog, dbMeta.cars))
1798
+ if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
1480
1799
  throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
1481
1800
  } else {
1482
1801
  await throwFalsy(this.loader.remoteCarStore).save(car);
@@ -1501,11 +1820,7 @@ var RemoteWAL = class extends VersionedStore {
1501
1820
  const res = await Promise.allSettled(uploads);
1502
1821
  const errors = res.filter((r) => r.status === "rejected");
1503
1822
  if (errors.length) {
1504
- throw this.logger.Error().Any(
1505
- "errors",
1506
- errors.map((e) => e.reason)
1507
- ).Msg("error uploading").AsError();
1508
- errors[0].reason;
1823
+ throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
1509
1824
  }
1510
1825
  if (operations.length) {
1511
1826
  const lastOp = operations[operations.length - 1];
@@ -1520,29 +1835,11 @@ var RemoteWAL = class extends VersionedStore {
1520
1835
  })();
1521
1836
  await rmlp;
1522
1837
  }
1523
- async start() {
1524
- const res = await this.gateway.start(this.url);
1525
- if (res.isErr()) {
1526
- return res;
1527
- }
1528
- const ver = guardVersion(this.url);
1529
- if (ver.isErr()) {
1530
- await this.close();
1531
- return ver;
1532
- }
1533
- const ready = await exception2Result(() => this.ready());
1534
- this._onStarted.forEach((fn) => fn());
1535
- if (ready.isErr()) {
1536
- await this.close();
1537
- return ready;
1538
- }
1539
- return ready;
1540
- }
1541
1838
  async load() {
1542
1839
  this.logger.Debug().Msg("loading");
1543
- const filepath = await this.gateway.buildUrl(this.url, "main");
1840
+ const filepath = await this.gateway.buildUrl(this.url(), "main");
1544
1841
  if (filepath.isErr()) {
1545
- throw this.logger.Error().Err(filepath.Err()).Str("url", this.url.toString()).Msg("error building url").AsError();
1842
+ throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
1546
1843
  }
1547
1844
  const bytes = await this.gateway.get(filepath.Ok());
1548
1845
  if (bytes.isErr()) {
@@ -1552,15 +1849,15 @@ var RemoteWAL = class extends VersionedStore {
1552
1849
  throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
1553
1850
  }
1554
1851
  try {
1555
- return bytes && parse(textDecoder.decode(bytes.Ok()));
1852
+ return bytes && parse(this.sthis.txt.decode(bytes.Ok()));
1556
1853
  } catch (e) {
1557
1854
  throw this.logger.Error().Err(e).Msg("error parse").AsError();
1558
1855
  }
1559
1856
  }
1560
1857
  async save(state) {
1561
- const filepath = await this.gateway.buildUrl(this.url, "main");
1858
+ const filepath = await this.gateway.buildUrl(this.url(), "main");
1562
1859
  if (filepath.isErr()) {
1563
- throw this.logger.Error().Err(filepath.Err()).Str("url", this.url.toString()).Msg("error building url").AsError();
1860
+ throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
1564
1861
  }
1565
1862
  let encoded;
1566
1863
  try {
@@ -1568,55 +1865,315 @@ var RemoteWAL = class extends VersionedStore {
1568
1865
  } catch (e) {
1569
1866
  throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
1570
1867
  }
1571
- const res = await this.gateway.put(filepath.Ok(), textEncoder.encode(encoded));
1868
+ const res = await this.gateway.put(filepath.Ok(), this.sthis.txt.encode(encoded));
1572
1869
  if (res.isErr()) {
1573
1870
  throw this.logger.Error().Err(res.Err()).Str("filePath", filepath.Ok().toString()).Msg("error saving").AsError();
1574
1871
  }
1575
1872
  }
1576
1873
  async close() {
1577
- await this.gateway.close(this.url);
1874
+ await this.gateway.close(this.url());
1578
1875
  this._onClosed.forEach((fn) => fn());
1579
- return Result2.Ok(void 0);
1876
+ return Result4.Ok(void 0);
1580
1877
  }
1581
1878
  destroy() {
1582
- return this.gateway.destroy(this.url);
1879
+ return this.gateway.destroy(this.url());
1583
1880
  }
1584
1881
  };
1585
1882
 
1586
- // src/blockstore/store-factory.ts
1587
- function ensureIsIndex(url, isIndex) {
1588
- if (isIndex) {
1589
- url.searchParams.set("index", isIndex);
1590
- return url;
1591
- } else {
1592
- url.searchParams.delete("index");
1593
- return url;
1883
+ // src/blockstore/store-meta.ts
1884
+ import { format as format2, parse as parse2 } from "@ipld/dag-json";
1885
+ import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
1886
+ import { CID as CID2 } from "multiformats";
1887
+ import { Result as Result5 } from "@adviser/cement";
1888
+
1889
+ // src/runtime/index.ts
1890
+ var runtime_exports = {};
1891
+ __export(runtime_exports, {
1892
+ FILESTORE_VERSION: () => FILESTORE_VERSION,
1893
+ INDEXDB_VERSION: () => INDEXDB_VERSION,
1894
+ files: () => files_exports,
1895
+ getFileName: () => getFileName,
1896
+ getFileSystem: () => getFileSystem,
1897
+ getPath: () => getPath,
1898
+ kb: () => key_bag_exports,
1899
+ kc: () => keyed_crypto_exports,
1900
+ mf: () => wait_pr_multiformats_exports,
1901
+ runtimeFn: () => runtimeFn2,
1902
+ toArrayBuffer: () => toArrayBuffer
1903
+ });
1904
+
1905
+ // src/runtime/wait-pr-multiformats/index.ts
1906
+ var wait_pr_multiformats_exports = {};
1907
+ __export(wait_pr_multiformats_exports, {
1908
+ block: () => block_exports,
1909
+ codec: () => codec_interface_exports
1910
+ });
1911
+
1912
+ // src/runtime/wait-pr-multiformats/codec-interface.ts
1913
+ var codec_interface_exports = {};
1914
+
1915
+ // src/runtime/index.ts
1916
+ import { runtimeFn as runtimeFn2 } from "@adviser/cement";
1917
+
1918
+ // src/blockstore/store-meta.ts
1919
+ async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
1920
+ const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
1921
+ return Promise.all(
1922
+ crdtEntries.map(async (crdtEntry) => {
1923
+ const eventBlock = await decodeEventBlock(decodeFromBase64(crdtEntry.data));
1924
+ const dbMeta = parse2(sthis.txt.decode(eventBlock.value.data.dbMeta));
1925
+ return {
1926
+ eventCid: eventBlock.cid,
1927
+ parents: crdtEntry.parents,
1928
+ dbMeta
1929
+ };
1930
+ })
1931
+ );
1932
+ }
1933
+ async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
1934
+ try {
1935
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
1936
+ const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
1937
+ if (keyInfo.length) {
1938
+ const dbMeta = keyInfo[0].dbMeta;
1939
+ if (dbMeta.key) {
1940
+ const kb = await key_bag_exports.getKeyBag(sthis);
1941
+ const keyName = getStoreKeyName(uri);
1942
+ const res = await kb.setNamedKey(keyName, dbMeta.key);
1943
+ if (res.isErr()) {
1944
+ sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
1945
+ throw res.Err();
1946
+ }
1947
+ }
1948
+ sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
1949
+ return Result5.Ok(dbMeta);
1950
+ }
1951
+ sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
1952
+ return Result5.Ok(void 0);
1953
+ } catch (error) {
1954
+ sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
1955
+ return Result5.Err(error);
1594
1956
  }
1595
1957
  }
1596
- function toURL(pathOrUrl, isIndex) {
1597
- if (pathOrUrl instanceof URL) return ensureIsIndex(pathOrUrl, isIndex);
1958
+ async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
1598
1959
  try {
1599
- const url = new URL(pathOrUrl);
1600
- return ensureIsIndex(url, isIndex);
1601
- } catch (e) {
1602
- const url = new URL(`file://${pathOrUrl}`);
1603
- return ensureIsIndex(url, isIndex);
1960
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
1961
+ const keyName = getStoreKeyName(uri);
1962
+ const kb = await key_bag_exports.getKeyBag(sthis);
1963
+ const res = await kb.getNamedExtractableKey(keyName, true);
1964
+ if (res.isErr()) {
1965
+ sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
1966
+ throw res.Err();
1967
+ }
1968
+ const keyData = await res.Ok().extract();
1969
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
1970
+ const { dbMeta, parents } = dbMetas[0];
1971
+ const parentLinks = parents.map((p) => CID2.parse(p));
1972
+ dbMeta.key = keyData.keyStr;
1973
+ const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
1974
+ const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
1975
+ sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
1976
+ return Result5.Ok(encoded);
1977
+ } catch (error) {
1978
+ sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
1979
+ return Result5.Err(error);
1980
+ }
1981
+ }
1982
+ function getStoreKeyName(url) {
1983
+ const storeKeyName = [url.getParam("localName") || url.getParam("name")];
1984
+ const idx = url.getParam("index");
1985
+ if (idx) {
1986
+ storeKeyName.push(idx);
1987
+ }
1988
+ storeKeyName.push("data");
1989
+ return `@${storeKeyName.join(":")}@`;
1990
+ }
1991
+ async function createDbMetaEventBlock(sthis, dbMeta, parents) {
1992
+ const event = await EventBlock.create(
1993
+ {
1994
+ dbMeta: sthis.txt.encode(format2(dbMeta))
1995
+ },
1996
+ parents
1997
+ );
1998
+ return event;
1999
+ }
2000
+ async function encodeEventsWithParents(sthis, events, parents) {
2001
+ const crdtEntries = events.map((event) => {
2002
+ const base64String = encodeToBase64(event.bytes);
2003
+ return {
2004
+ cid: event.cid.toString(),
2005
+ data: base64String,
2006
+ parents: parents.map((p) => p.toString())
2007
+ };
2008
+ });
2009
+ return sthis.txt.encode(JSON.stringify(crdtEntries));
2010
+ }
2011
+ var MetaStoreImpl = class extends BaseStoreImpl {
2012
+ constructor(sthis, name, url, opts, remote) {
2013
+ super(
2014
+ name,
2015
+ url,
2016
+ {
2017
+ ...opts
2018
+ },
2019
+ sthis,
2020
+ ensureLogger(sthis, "MetaStoreImpl")
2021
+ );
2022
+ this.storeType = "meta";
2023
+ this.subscribers = /* @__PURE__ */ new Map();
2024
+ this.parents = [];
2025
+ if (remote && opts.gateway.subscribe) {
2026
+ this.onStarted(async () => {
2027
+ this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
2028
+ opts.gateway.subscribe?.(this.url(), async (message) => {
2029
+ this.logger.Debug().Msg("Received message from gateway");
2030
+ const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
2031
+ await Promise.all(
2032
+ dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
2033
+ );
2034
+ });
2035
+ });
2036
+ }
2037
+ }
2038
+ async handleByteHeads(byteHeads) {
2039
+ return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
2040
+ }
2041
+ async load() {
2042
+ const branch = "main";
2043
+ const url = await this.gateway.buildUrl(this.url(), branch);
2044
+ if (url.isErr()) {
2045
+ throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
2046
+ }
2047
+ const bytes = await this.gateway.get(url.Ok());
2048
+ if (bytes.isErr()) {
2049
+ if (isNotFoundError(bytes)) {
2050
+ return void 0;
2051
+ }
2052
+ throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
2053
+ }
2054
+ const dbMetas = await this.handleByteHeads(bytes.Ok());
2055
+ await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
2056
+ const cids = dbMetas.map((m) => m.eventCid);
2057
+ const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
2058
+ this.parents = Array.from(uniqueParentsMap.values());
2059
+ return dbMetas.map((m) => m.dbMeta);
2060
+ }
2061
+ async save(meta, branch) {
2062
+ branch = branch || "main";
2063
+ this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
2064
+ const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
2065
+ const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
2066
+ const url = await this.gateway.buildUrl(this.url(), branch);
2067
+ if (url.isErr()) {
2068
+ throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
2069
+ }
2070
+ const res = await this.gateway.put(url.Ok(), bytes);
2071
+ if (res.isErr()) {
2072
+ throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
2073
+ }
2074
+ await this.loader?.handleDbMetasFromStore([meta]);
2075
+ this.parents = [event.cid];
2076
+ return res;
2077
+ }
2078
+ async close() {
2079
+ await this.gateway.close(this.url());
2080
+ this._onClosed.forEach((fn) => fn());
2081
+ return Result5.Ok(void 0);
2082
+ }
2083
+ async destroy() {
2084
+ return this.gateway.destroy(this.url());
2085
+ }
2086
+ };
2087
+ function encodeToBase64(bytes) {
2088
+ const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
2089
+ let base64 = "";
2090
+ let i;
2091
+ for (i = 0; i < bytes.length - 2; i += 3) {
2092
+ base64 += chars[bytes[i] >> 2];
2093
+ base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
2094
+ base64 += chars[(bytes[i + 1] & 15) << 2 | bytes[i + 2] >> 6];
2095
+ base64 += chars[bytes[i + 2] & 63];
2096
+ }
2097
+ if (i < bytes.length) {
2098
+ base64 += chars[bytes[i] >> 2];
2099
+ if (i === bytes.length - 1) {
2100
+ base64 += chars[(bytes[i] & 3) << 4];
2101
+ base64 += "==";
2102
+ } else {
2103
+ base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
2104
+ base64 += chars[(bytes[i + 1] & 15) << 2];
2105
+ base64 += "=";
2106
+ }
1604
2107
  }
2108
+ return base64;
2109
+ }
2110
+ function decodeFromBase64(base64) {
2111
+ const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
2112
+ const bytes = new Uint8Array(base64.length * 3 / 4);
2113
+ let i;
2114
+ let j = 0;
2115
+ for (i = 0; i < base64.length; i += 4) {
2116
+ const a = chars.indexOf(base64[i]);
2117
+ const b = chars.indexOf(base64[i + 1]);
2118
+ const c = chars.indexOf(base64[i + 2]);
2119
+ const d = chars.indexOf(base64[i + 3]);
2120
+ bytes[j++] = a << 2 | b >> 4;
2121
+ if (base64[i + 2] !== "=") {
2122
+ bytes[j++] = (b & 15) << 4 | c >> 2;
2123
+ }
2124
+ if (base64[i + 3] !== "=") {
2125
+ bytes[j++] = (c & 3) << 6 | d;
2126
+ }
2127
+ }
2128
+ return bytes.slice(0, j);
2129
+ }
2130
+
2131
+ // src/blockstore/store-factory.ts
2132
+ function ensureIsIndex(url, isIndex) {
2133
+ if (isIndex) {
2134
+ return url.build().setParam("index", isIndex).URI();
2135
+ }
2136
+ return url.build().delParam("index").URI();
1605
2137
  }
1606
- var storeFactory = /* @__PURE__ */ new Map();
1607
2138
  function ensureName(name, url) {
1608
- if (!url.searchParams.has("name")) {
1609
- url.searchParams.set("name", name);
2139
+ if (!url.hasParam("name")) {
2140
+ return url.build().setParam("name", name).URI();
1610
2141
  }
2142
+ return url;
1611
2143
  }
2144
+ var storeFactory = /* @__PURE__ */ new Map();
1612
2145
  function buildURL(optURL, loader) {
1613
2146
  const storeOpts = loader.ebOpts.store;
1614
2147
  const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
1615
2148
  let obuUrl;
1616
2149
  if (obuItem && obuItem.overrideBaseURL) {
1617
- obuUrl = new URL(obuItem.overrideBaseURL);
2150
+ obuUrl = URI5.from(obuItem.overrideBaseURL);
1618
2151
  }
1619
- return toURL(optURL || obuUrl || dataDir(loader.name, storeOpts.stores?.base), storeOpts.isIndex);
2152
+ const ret = ensureIsIndex(
2153
+ URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
2154
+ storeOpts.isIndex
2155
+ );
2156
+ return ret;
2157
+ }
2158
+ var onceGateway = new KeyedResolvOnce2();
2159
+ async function getGatewayFromURL(url, sthis) {
2160
+ return onceGateway.get(url.toString()).once(async () => {
2161
+ const item = storeFactory.get(url.protocol);
2162
+ if (item) {
2163
+ const ret = {
2164
+ gateway: await item.gateway(sthis),
2165
+ test: await item.test(sthis)
2166
+ };
2167
+ const res = await ret.gateway.start(url);
2168
+ if (res.isErr()) {
2169
+ sthis.logger.Error().Result("start", res).Msg("start failed");
2170
+ return void 0;
2171
+ }
2172
+ return ret;
2173
+ }
2174
+ sthis.logger.Warn().Url(url).Msg("unsupported protocol");
2175
+ return void 0;
2176
+ });
1620
2177
  }
1621
2178
  function registerStoreProtocol(item) {
1622
2179
  let protocol = item.protocol;
@@ -1625,8 +2182,7 @@ function registerStoreProtocol(item) {
1625
2182
  }
1626
2183
  if (storeFactory.has(protocol)) {
1627
2184
  if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
1628
- const logger = ensureLogger({}, "registerStoreProtocol", { protocol });
1629
- logger.Warn().Msg(`protocol ${protocol} already registered`);
2185
+ throw new Error(`we need a logger here`);
1630
2186
  return () => {
1631
2187
  };
1632
2188
  }
@@ -1641,106 +2197,92 @@ function registerStoreProtocol(item) {
1641
2197
  storeFactory.delete(protocol);
1642
2198
  };
1643
2199
  }
1644
- function runStoreFactory(url, logger, run) {
1645
- const item = storeFactory.get(url.protocol);
1646
- if (!item) {
1647
- throw logger.Error().Url(url).Str("protocol", url.protocol).Any("keys", Array(storeFactory.keys())).Msg(`unsupported protocol`).AsError();
1648
- }
1649
- logger.Debug().Str("protocol", url.protocol).Msg("run");
1650
- return run(item);
1651
- }
1652
- var onceLoadDataGateway = new KeyedResolvOnce();
1653
- function loadDataGateway(url, logger) {
1654
- return onceLoadDataGateway.get(url.protocol).once(async () => {
1655
- return runStoreFactory(url, logger, async (item) => item.data(logger));
1656
- });
1657
- }
1658
- var onceDataStoreFactory = new KeyedResolvOnce();
2200
+ var onceDataStoreFactory = new KeyedResolvOnce2();
1659
2201
  async function dataStoreFactory(loader) {
1660
- const url = buildURL(loader.ebOpts.store.stores?.data, loader);
1661
- ensureName(loader.name, url);
1662
- const logger = ensureLogger(loader.logger, "dataStoreFactory", { url: url.toString() });
1663
- url.searchParams.set("store", "data");
2202
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
2203
+ const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
1664
2204
  return onceDataStoreFactory.get(url.toString()).once(async () => {
1665
- const gateway = await loadDataGateway(url, logger);
1666
- const store = new DataStore(loader.name, url, loader.logger, gateway);
1667
- const ret = await store.start();
1668
- if (ret.isErr()) {
1669
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
1670
- }
1671
- logger.Debug().Str("prepared", store.url.toString()).Msg("produced");
2205
+ const gateway = await getGatewayFromURL(url, sthis);
2206
+ if (!gateway) {
2207
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2208
+ }
2209
+ const store = new DataStoreImpl(sthis, loader.name, url, {
2210
+ gateway: gateway.gateway,
2211
+ keybag: () => getKeyBag(loader.sthis, {
2212
+ ...loader.ebOpts.keyBag
2213
+ })
2214
+ });
1672
2215
  return store;
1673
2216
  });
1674
2217
  }
1675
- var onceLoadMetaGateway = new KeyedResolvOnce();
1676
- function loadMetaGateway(url, logger) {
1677
- return onceLoadMetaGateway.get(url.protocol).once(async () => {
1678
- return runStoreFactory(url, logger, async (item) => item.meta(logger));
1679
- });
1680
- }
1681
- var onceMetaStoreFactory = new KeyedResolvOnce();
2218
+ var onceMetaStoreFactory = new KeyedResolvOnce2();
1682
2219
  async function metaStoreFactory(loader) {
1683
- const url = buildURL(loader.ebOpts.store.stores?.meta, loader);
1684
- ensureName(loader.name, url);
1685
- const logger = ensureLogger(loader.logger, "metaStoreFactory", { url: () => url.toString() });
1686
- url.searchParams.set("store", "meta");
2220
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
2221
+ const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
1687
2222
  return onceMetaStoreFactory.get(url.toString()).once(async () => {
1688
- logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
1689
- const gateway = await loadMetaGateway(url, logger);
1690
- const store = new MetaStore(loader.name, url, loader.logger, gateway);
1691
- const ret = await store.start();
1692
- if (ret.isErr()) {
1693
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
1694
- }
2223
+ sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
2224
+ const gateway = await getGatewayFromURL(url, sthis);
2225
+ if (!gateway) {
2226
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2227
+ }
2228
+ const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
2229
+ gateway: gateway.gateway,
2230
+ keybag: () => getKeyBag(loader.sthis, {
2231
+ ...loader.ebOpts.keyBag
2232
+ })
2233
+ });
1695
2234
  return store;
1696
2235
  });
1697
2236
  }
1698
- var onceWalGateway = new KeyedResolvOnce();
1699
- function loadWalGateway(url, logger) {
1700
- return onceWalGateway.get(url.protocol).once(async () => {
1701
- return runStoreFactory(url, logger, async (item) => item.wal(logger));
1702
- });
1703
- }
1704
- var onceRemoteWalFactory = new KeyedResolvOnce();
2237
+ var onceRemoteWalFactory = new KeyedResolvOnce2();
1705
2238
  async function remoteWalFactory(loader) {
1706
- const url = buildURL(loader.ebOpts.store.stores?.meta, loader);
1707
- ensureName(loader.name, url);
1708
- const logger = ensureLogger(loader.logger, "remoteWalFactory", { url: url.toString() });
1709
- url.searchParams.set("store", "wal");
2239
+ const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
2240
+ const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
1710
2241
  return onceRemoteWalFactory.get(url.toString()).once(async () => {
1711
- const gateway = await loadWalGateway(url, logger);
1712
- logger.Debug().Str("prepared", url.toString()).Msg("produced");
1713
- const store = new RemoteWAL(loader, url, loader.logger, gateway);
1714
- const ret = await store.start();
1715
- if (ret.isErr()) {
1716
- throw logger.Error().Result("start", ret).Msg("start failed").AsError();
1717
- }
2242
+ const gateway = await getGatewayFromURL(url, sthis);
2243
+ if (!gateway) {
2244
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2245
+ }
2246
+ sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
2247
+ const store = new WALStoreImpl(loader, url, {
2248
+ gateway: gateway.gateway,
2249
+ keybag: () => getKeyBag(loader.sthis, {
2250
+ ...loader.ebOpts.keyBag
2251
+ })
2252
+ });
1718
2253
  return store;
1719
2254
  });
1720
2255
  }
1721
- async function testStoreFactory(url, ilogger) {
1722
- const logger = ensureLogger(
1723
- {
1724
- logger: ilogger
1725
- },
1726
- "testStoreFactory"
1727
- );
1728
- return runStoreFactory(url, logger, async (item) => item.test(logger));
2256
+ async function testStoreFactory(url, sthis) {
2257
+ sthis = ensureSuperLog(sthis, "testStoreFactory");
2258
+ const gateway = await getGatewayFromURL(url, sthis);
2259
+ if (!gateway) {
2260
+ throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
2261
+ }
2262
+ return gateway.test;
2263
+ }
2264
+ async function ensureStart(store, logger) {
2265
+ const ret = await store.start();
2266
+ if (ret.isErr()) {
2267
+ throw logger.Error().Result("start", ret).Msg("start failed").AsError();
2268
+ }
2269
+ logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
2270
+ return store;
1729
2271
  }
1730
- function toStoreRuntime(opts, ilogger) {
1731
- const logger = ensureLogger(ilogger, "toStoreRuntime", {});
2272
+ function toStoreRuntime(opts, sthis) {
2273
+ const logger = ensureLogger(sthis, "toStoreRuntime", {});
1732
2274
  return {
1733
- makeMetaStore: (loader) => {
2275
+ makeMetaStore: async (loader) => {
1734
2276
  logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
1735
- return (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader);
2277
+ return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
1736
2278
  },
1737
- makeDataStore: (loader) => {
2279
+ makeDataStore: async (loader) => {
1738
2280
  logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
1739
- return (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader);
2281
+ return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
1740
2282
  },
1741
- makeRemoteWAL: (loader) => {
1742
- logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeRemoteWAL).Msg("makeRemoteWAL");
1743
- return (loader.ebOpts.store.makeRemoteWAL || remoteWalFactory)(loader);
2283
+ makeWALStore: async (loader) => {
2284
+ logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
2285
+ return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
1744
2286
  },
1745
2287
  encodeFile: opts.encodeFile || encodeFile,
1746
2288
  decodeFile: opts.decodeFile || decodeFile
@@ -1748,43 +2290,125 @@ function toStoreRuntime(opts, ilogger) {
1748
2290
  }
1749
2291
  registerStoreProtocol({
1750
2292
  protocol: "file:",
1751
- data: async (logger) => {
1752
- const { FileDataGateway } = await import("./store-file-VJ6BI4II.js");
1753
- return new FileDataGateway(logger);
2293
+ gateway: async (sthis) => {
2294
+ const { FileGateway } = await import("./gateway-VWWKLHUI.js");
2295
+ return new FileGateway(sthis);
1754
2296
  },
1755
- meta: async (logger) => {
1756
- const { FileMetaGateway } = await import("./store-file-VJ6BI4II.js");
1757
- return new FileMetaGateway(logger);
1758
- },
1759
- wal: async (logger) => {
1760
- const { FileWALGateway } = await import("./store-file-VJ6BI4II.js");
1761
- return new FileWALGateway(logger);
1762
- },
1763
- test: async (logger) => {
1764
- const { FileTestStore } = await import("./store-file-VJ6BI4II.js");
1765
- return new FileTestStore(logger);
2297
+ test: async (sthis) => {
2298
+ const { FileTestStore } = await import("./gateway-VWWKLHUI.js");
2299
+ return new FileTestStore(sthis);
1766
2300
  }
1767
2301
  });
1768
2302
  registerStoreProtocol({
1769
2303
  protocol: "indexdb:",
1770
- data: async (logger) => {
1771
- const { IndexDBDataGateway } = await import("./store-indexdb-WLRSICCB.js");
1772
- return new IndexDBDataGateway(logger);
1773
- },
1774
- meta: async (logger) => {
1775
- const { IndexDBMetaGateway } = await import("./store-indexdb-WLRSICCB.js");
1776
- return new IndexDBMetaGateway(logger);
1777
- },
1778
- wal: async (logger) => {
1779
- const { IndexDBMetaGateway } = await import("./store-indexdb-WLRSICCB.js");
1780
- return new IndexDBMetaGateway(logger);
2304
+ gateway: async (sthis) => {
2305
+ const { IndexDBGateway } = await import("./gateway-7OM6OSYK.js");
2306
+ return new IndexDBGateway(sthis);
1781
2307
  },
1782
- test: async (logger) => {
1783
- const { IndexDBTestStore } = await import("./store-indexdb-WLRSICCB.js");
1784
- return new IndexDBTestStore(logger);
2308
+ test: async (sthis) => {
2309
+ const { IndexDBTestStore } = await import("./gateway-7OM6OSYK.js");
2310
+ return new IndexDBTestStore(sthis);
1785
2311
  }
1786
2312
  });
1787
2313
 
2314
+ // src/blockstore/store-remote.ts
2315
+ async function RemoteDataStore(sthis, name, url, opts) {
2316
+ const ds = new DataStoreImpl(sthis, name, url, opts);
2317
+ await ds.start();
2318
+ return ds;
2319
+ }
2320
+ async function RemoteMetaStore(sthis, name, url, opts) {
2321
+ const ms = new MetaStoreImpl(sthis, name, url, opts, true);
2322
+ await ms.start();
2323
+ return ms;
2324
+ }
2325
+
2326
+ // src/blockstore/connection-base.ts
2327
+ var ConnectionBase = class {
2328
+ constructor(url, logger) {
2329
+ this.loaded = Promise.resolve();
2330
+ this.logger = logger;
2331
+ this.url = url;
2332
+ }
2333
+ async refresh() {
2334
+ await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
2335
+ await (await throwFalsy(this.loader).WALStore()).process();
2336
+ }
2337
+ async connect_X({ loader }) {
2338
+ if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
2339
+ await this.connectMeta_X({ loader });
2340
+ await this.connectStorage_X({ loader });
2341
+ }
2342
+ async connectMeta_X({ loader }) {
2343
+ if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
2344
+ this.loader = loader;
2345
+ await this.onConnect();
2346
+ const metaUrl = this.url.build().defParam("store", "meta").URI();
2347
+ const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
2348
+ if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
2349
+ const dbName = metaUrl.getParam("name");
2350
+ if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
2351
+ const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
2352
+ gateway: gateway.gateway,
2353
+ keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
2354
+ loader
2355
+ });
2356
+ this.loader.remoteMetaStore = remote;
2357
+ this.loaded = this.loader.ready().then(async () => {
2358
+ remote.load().then(async () => {
2359
+ (await throwFalsy(this.loader).WALStore()).process();
2360
+ });
2361
+ });
2362
+ }
2363
+ async connectStorage_X({ loader }) {
2364
+ if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
2365
+ this.loader = loader;
2366
+ const dataUrl = this.url.build().defParam("store", "data").URI();
2367
+ const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
2368
+ if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
2369
+ const name = dataUrl.getParam("name");
2370
+ if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
2371
+ loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
2372
+ gateway: gateway.gateway,
2373
+ keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
2374
+ });
2375
+ loader.remoteFileStore = loader.remoteCarStore;
2376
+ }
2377
+ // move this stuff to connect
2378
+ // async getDashboardURL(compact = true) {
2379
+ // const baseUrl = 'https://dashboard.fireproof.storage/'
2380
+ // if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
2381
+ // // if (compact) {
2382
+ // // await this.compact()
2383
+ // // }
2384
+ // const currents = await this.loader?.metaStore?.load()
2385
+ // if (!currents) throw new Error("Can't sync empty database: save data first")
2386
+ // if (currents.length > 1)
2387
+ // throw new Error("Can't sync database with split heads: make an update first")
2388
+ // const current = currents[0]
2389
+ // const params = {
2390
+ // car: current.car.toString()
2391
+ // }
2392
+ // if (current.key) {
2393
+ // // @ts-ignore
2394
+ // params.key = current.key.toString()
2395
+ // }
2396
+ // // @ts-ignore
2397
+ // if (this.name) {
2398
+ // // @ts-ignore
2399
+ // params.name = this.name
2400
+ // }
2401
+ // const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
2402
+ // console.log('Import to dashboard: ' + url.toString())
2403
+ // return url
2404
+ // }
2405
+ // openDashboard() {
2406
+ // void this.getDashboardURL().then(url => {
2407
+ // if (url) window.open(url.toString(), '_blank')
2408
+ // })
2409
+ // }
2410
+ };
2411
+
1788
2412
  // src/crdt-helpers.ts
1789
2413
  function time(tag) {
1790
2414
  }
@@ -1833,7 +2457,7 @@ async function writeDocContent(store, blocks, update, logger) {
1833
2457
  await processFiles(store, blocks, update.value, logger);
1834
2458
  value = { doc: update.value };
1835
2459
  }
1836
- const block = await encode3({ value, hasher: hasher2, codec: codec2 });
2460
+ const block = await encode({ value, hasher: hasher5, codec });
1837
2461
  blocks.putSync(block.cid, block.bytes);
1838
2462
  return block.cid;
1839
2463
  }
@@ -1842,10 +2466,16 @@ async function processFiles(store, blocks, doc, logger) {
1842
2466
  await processFileset(logger, store, blocks, doc._files);
1843
2467
  }
1844
2468
  if (doc._publicFiles) {
1845
- await processFileset(logger, store, blocks, doc._publicFiles, true);
2469
+ await processFileset(
2470
+ logger,
2471
+ store,
2472
+ blocks,
2473
+ doc._publicFiles
2474
+ /*, true*/
2475
+ );
1846
2476
  }
1847
2477
  }
1848
- async function processFileset(logger, store, blocks, files, publicFiles = false) {
2478
+ async function processFileset(logger, store, blocks, files) {
1849
2479
  const dbBlockstore = blocks.parent;
1850
2480
  if (!dbBlockstore.loader) throw logger.Error().Msg("Missing loader, database name is required").AsError();
1851
2481
  const t = new CarTransaction(dbBlockstore);
@@ -1867,9 +2497,10 @@ async function processFileset(logger, store, blocks, files, publicFiles = false)
1867
2497
  }
1868
2498
  }
1869
2499
  if (didPut.length) {
1870
- const car = await dbBlockstore.loader.commitFiles(t, { files }, {
1871
- public: publicFiles
1872
- });
2500
+ const car = await dbBlockstore.loader.commitFiles(
2501
+ t,
2502
+ { files }
2503
+ );
1873
2504
  if (car) {
1874
2505
  for (const name of didPut) {
1875
2506
  files[name] = { car, ...files[name] };
@@ -1903,7 +2534,7 @@ function readFileset(blocks, files, isPublic = false) {
1903
2534
  fileMeta.file = async () => await blocks.ebOpts.storeRuntime.decodeFile(
1904
2535
  {
1905
2536
  get: async (cid) => {
1906
- return await blocks.getFile(throwFalsy(fileMeta.car), cid, isPublic);
2537
+ return await blocks.getFile(throwFalsy(fileMeta.car), cid);
1907
2538
  }
1908
2539
  },
1909
2540
  fileMeta.cid,
@@ -1917,7 +2548,7 @@ function readFileset(blocks, files, isPublic = false) {
1917
2548
  async function getValueFromLink(blocks, link, logger) {
1918
2549
  const block = await blocks.get(link);
1919
2550
  if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
1920
- const { value } = await decode3({ bytes: block.bytes, hasher: hasher2, codec: codec2 });
2551
+ const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
1921
2552
  const cvalue = {
1922
2553
  ...value,
1923
2554
  cid: link
@@ -1926,17 +2557,21 @@ async function getValueFromLink(blocks, link, logger) {
1926
2557
  return cvalue;
1927
2558
  }
1928
2559
  var DirtyEventFetcher = class extends EventFetcher {
2560
+ constructor(logger, blocks) {
2561
+ super(blocks);
2562
+ this.logger = logger;
2563
+ }
1929
2564
  async get(link) {
1930
2565
  try {
1931
2566
  return super.get(link);
1932
2567
  } catch (e) {
1933
- console.error("missing event", link.toString(), e);
2568
+ this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
1934
2569
  return { value: void 0 };
1935
2570
  }
1936
2571
  }
1937
2572
  };
1938
2573
  async function clockChangesSince(blocks, head, since, opts, logger) {
1939
- const eventsFetcher = opts.dirty ? new DirtyEventFetcher(blocks) : new EventFetcher(blocks);
2574
+ const eventsFetcher = opts.dirty ? new DirtyEventFetcher(logger, blocks) : new EventFetcher(blocks);
1940
2575
  const keys = /* @__PURE__ */ new Set();
1941
2576
  const updates = await gatherUpdates(
1942
2577
  blocks,
@@ -2032,20 +2667,19 @@ async function doCompact(blockLog, head, logger) {
2032
2667
  isCompacting = false;
2033
2668
  }
2034
2669
  async function getBlock(blocks, cidString) {
2035
- const block = await blocks.get(parse2(cidString));
2670
+ const block = await blocks.get(parse3(cidString));
2036
2671
  if (!block) throw new Error(`Missing block ${cidString}`);
2037
- const { cid, value } = await decode3({ bytes: block.bytes, codec: codec2, hasher: hasher2 });
2038
- return new Block2({ cid, value, bytes: block.bytes });
2672
+ const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
2673
+ return new Block({ cid, value, bytes: block.bytes });
2039
2674
  }
2040
2675
 
2041
2676
  // src/indexer-helpers.ts
2042
- import { create as create3 } from "multiformats/block";
2043
- import { sha256 as hasher3 } from "multiformats/hashes/sha2";
2044
- import * as codec3 from "@ipld/dag-cbor";
2677
+ import { sha256 as hasher6 } from "multiformats/hashes/sha2";
2678
+ import * as codec2 from "@ipld/dag-cbor";
2045
2679
  import charwise from "charwise";
2046
2680
  import * as DbIndex from "prolly-trees/db-index";
2047
- import { bf as bf2, simpleCompare } from "prolly-trees/utils";
2048
- import { nocache as cache2 } from "prolly-trees/cache";
2681
+ import { bf, simpleCompare } from "prolly-trees/utils";
2682
+ import { nocache as cache } from "prolly-trees/cache";
2049
2683
  var IndexTree = class {
2050
2684
  };
2051
2685
  function refCompare(aRef, bRef) {
@@ -2061,8 +2695,8 @@ function compare(a, b) {
2061
2695
  if (comp !== 0) return comp;
2062
2696
  return refCompare(aRef, bRef);
2063
2697
  }
2064
- var byKeyOpts = { cache: cache2, chunker: bf2(30), codec: codec3, hasher: hasher3, compare };
2065
- var byIdOpts = { cache: cache2, chunker: bf2(30), codec: codec3, hasher: hasher3, compare: simpleCompare };
2698
+ var byKeyOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher6, compare };
2699
+ var byIdOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher6, compare: simpleCompare };
2066
2700
  function indexEntriesForChanges(changes, mapFn) {
2067
2701
  const indexEntries = [];
2068
2702
  changes.forEach(({ id: key, value, del }) => {
@@ -2090,7 +2724,7 @@ function makeProllyGetBlock(blocks) {
2090
2724
  const block = await blocks.get(address);
2091
2725
  if (!block) throw new Error(`Missing block ${address.toString()}`);
2092
2726
  const { cid, bytes } = block;
2093
- return create3({ cid, bytes, hasher: hasher3, codec: codec3 });
2727
+ return create({ cid, bytes, hasher: hasher6, codec: codec2 });
2094
2728
  };
2095
2729
  }
2096
2730
  async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
@@ -2161,25 +2795,25 @@ function encodeKey(key) {
2161
2795
  }
2162
2796
 
2163
2797
  // src/indexer.ts
2164
- function index({ _crdt }, name, mapFn, meta) {
2798
+ function index(sthis, { _crdt }, name, mapFn, meta) {
2165
2799
  if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
2166
2800
  if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
2167
2801
  if (_crdt.indexers.has(name)) {
2168
2802
  const idx = _crdt.indexers.get(name);
2169
2803
  idx.applyMapFn(name, mapFn, meta);
2170
2804
  } else {
2171
- const idx = new Index(_crdt, name, mapFn, meta);
2805
+ const idx = new Index(sthis, _crdt, name, mapFn, meta);
2172
2806
  _crdt.indexers.set(name, idx);
2173
2807
  }
2174
2808
  return _crdt.indexers.get(name);
2175
2809
  }
2176
2810
  var Index = class {
2177
- constructor(crdt, name, mapFn, meta) {
2811
+ constructor(sthis, crdt, name, mapFn, meta) {
2178
2812
  this.mapFnString = "";
2179
2813
  this.byKey = new IndexTree();
2180
2814
  this.byId = new IndexTree();
2181
2815
  this.includeDocsDefault = false;
2182
- this.logger = ensureLogger(crdt.logger, "Index");
2816
+ this.logger = ensureLogger(sthis, "Index");
2183
2817
  this.blockstore = crdt.indexBlockstore;
2184
2818
  this.crdt = crdt;
2185
2819
  this.applyMapFn(name, mapFn, meta);
@@ -2234,7 +2868,7 @@ var Index = class {
2234
2868
  }
2235
2869
  if (this.mapFnString) {
2236
2870
  if (this.mapFnString !== mapFn.toString()) {
2237
- throw this.logger.Error().Msg("cannot apply different mapFn app").AsError();
2871
+ throw this.logger.Error().Str("mapFnString", this.mapFnString).Str("mapFn", mapFn.toString()).Msg("cannot apply different mapFn app").AsError();
2238
2872
  }
2239
2873
  } else {
2240
2874
  this.mapFnString = mapFn.toString();
@@ -2369,7 +3003,7 @@ var Index = class {
2369
3003
  // src/crdt-clock.ts
2370
3004
  import { advance } from "@web3-storage/pail/clock";
2371
3005
  import { root as root2 } from "@web3-storage/pail/crdt";
2372
- import { ResolveOnce as ResolveOnce3 } from "@adviser/cement";
3006
+ import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
2373
3007
 
2374
3008
  // src/apply-head-queue.ts
2375
3009
  function applyHeadQueue(worker, logger) {
@@ -2426,9 +3060,9 @@ var CRDTClock = class {
2426
3060
  this.zoomers = /* @__PURE__ */ new Set();
2427
3061
  this.watchers = /* @__PURE__ */ new Set();
2428
3062
  this.emptyWatchers = /* @__PURE__ */ new Set();
2429
- this._ready = new ResolveOnce3();
3063
+ this._ready = new ResolveOnce4();
2430
3064
  this.blockstore = blockstore;
2431
- this.logger = ensureLogger(blockstore.logger, "CRDTClock");
3065
+ this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
2432
3066
  this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
2433
3067
  }
2434
3068
  async ready() {
@@ -2501,7 +3135,7 @@ var CRDTClock = class {
2501
3135
  }
2502
3136
  return { head: advancedHead };
2503
3137
  },
2504
- { noLoader }
3138
+ { noLoader, add: false }
2505
3139
  );
2506
3140
  this.setHead(meta.head);
2507
3141
  }
@@ -2535,13 +3169,14 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
2535
3169
 
2536
3170
  // src/crdt.ts
2537
3171
  var CRDT = class {
2538
- constructor(name, opts = {}) {
2539
- this.onceReady = new ResolveOnce4();
3172
+ constructor(sthis, name, opts = {}) {
2540
3173
  this.indexers = /* @__PURE__ */ new Map();
3174
+ this.onceReady = new ResolveOnce5();
3175
+ this.sthis = sthis;
2541
3176
  this.name = name;
2542
- this.logger = ensureLogger(opts, "CRDT");
3177
+ this.logger = ensureLogger(sthis, "CRDT");
2543
3178
  this.opts = opts;
2544
- this.blockstore = blockstoreFactory({
3179
+ this.blockstore = blockstoreFactory(sthis, {
2545
3180
  name,
2546
3181
  applyMeta: async (meta) => {
2547
3182
  const crdtMeta = meta;
@@ -2553,22 +3188,20 @@ var CRDT = class {
2553
3188
  return { head: this.clock.head };
2554
3189
  },
2555
3190
  autoCompact: this.opts.autoCompact || 100,
2556
- crypto: this.opts.crypto,
2557
3191
  store: { ...this.opts.store, isIndex: void 0 },
2558
3192
  public: this.opts.public,
2559
3193
  meta: this.opts.meta,
2560
3194
  threshold: this.opts.threshold
2561
3195
  });
2562
- this.indexBlockstore = blockstoreFactory({
3196
+ this.indexBlockstore = blockstoreFactory(sthis, {
2563
3197
  name,
2564
3198
  applyMeta: async (meta) => {
2565
3199
  const idxCarMeta = meta;
2566
3200
  if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
2567
3201
  for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
2568
- index({ _crdt: this }, name2, void 0, idx);
3202
+ index(this.sthis, { _crdt: this }, name2, void 0, idx);
2569
3203
  }
2570
3204
  },
2571
- crypto: this.opts.crypto,
2572
3205
  store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
2573
3206
  public: this.opts.public
2574
3207
  });
@@ -2579,17 +3212,6 @@ var CRDT = class {
2579
3212
  }
2580
3213
  });
2581
3214
  }
2582
- async ready() {
2583
- return this.onceReady.once(async () => {
2584
- await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
2585
- });
2586
- }
2587
- async close() {
2588
- await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
2589
- }
2590
- async destroy() {
2591
- await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
2592
- }
2593
3215
  async bulk(updates) {
2594
3216
  await this.ready();
2595
3217
  const prevHead = [...this.clock.head];
@@ -2610,6 +3232,22 @@ var CRDT = class {
2610
3232
  await this.clock.applyHead(done.meta.head, prevHead, updates);
2611
3233
  return done.meta;
2612
3234
  }
3235
+ async ready() {
3236
+ return this.onceReady.once(async () => {
3237
+ try {
3238
+ await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
3239
+ } catch (e) {
3240
+ const ee = e;
3241
+ throw this.logger.Error().Err(e).Msg(`CRDT is not ready: ${ee.stack}`).AsError();
3242
+ }
3243
+ });
3244
+ }
3245
+ async close() {
3246
+ await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
3247
+ }
3248
+ async destroy() {
3249
+ await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
3250
+ }
2613
3251
  // if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
2614
3252
  async allDocs() {
2615
3253
  await this.ready();
@@ -2654,11 +3292,12 @@ var Database = class {
2654
3292
  this._listening = false;
2655
3293
  this._listeners = /* @__PURE__ */ new Set();
2656
3294
  this._noupdate_listeners = /* @__PURE__ */ new Set();
2657
- this._ready = new ResolveOnce5();
3295
+ this._ready = new ResolveOnce6();
2658
3296
  this.name = name;
2659
3297
  this.opts = opts || this.opts;
2660
- this.logger = ensureLogger(this.opts, "Database");
2661
- this._crdt = new CRDT(name, this.opts);
3298
+ this.sthis = ensureSuperThis(this.opts);
3299
+ this.logger = ensureLogger(this.sthis, "Database");
3300
+ this._crdt = new CRDT(this.sthis, name, this.opts);
2662
3301
  this.blockstore = this._crdt.blockstore;
2663
3302
  this._writeQueue = writeQueue(async (updates) => {
2664
3303
  return await this._crdt.bulk(updates);
@@ -2682,7 +3321,7 @@ var Database = class {
2682
3321
  }
2683
3322
  async ready() {
2684
3323
  return this._ready.once(async () => {
2685
- await SysContainer.start();
3324
+ await this.sthis.start();
2686
3325
  await this._crdt.ready();
2687
3326
  await this.blockstore.ready();
2688
3327
  });
@@ -2702,7 +3341,7 @@ var Database = class {
2702
3341
  await this.ready();
2703
3342
  this.logger.Debug().Str("id", doc._id).Msg("put");
2704
3343
  const { _id, ...value } = doc;
2705
- const docId = _id || uuidv7();
3344
+ const docId = _id || this.sthis.nextId().str;
2706
3345
  const result = await this._writeQueue.push({
2707
3346
  id: docId,
2708
3347
  value: {
@@ -2767,7 +3406,7 @@ var Database = class {
2767
3406
  await this.ready();
2768
3407
  this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
2769
3408
  const _crdt = this._crdt;
2770
- const idx = typeof field === "string" ? index({ _crdt }, field) : index({ _crdt }, makeName(field.toString()), field);
3409
+ const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
2771
3410
  return await idx.query(opts);
2772
3411
  }
2773
3412
  async compact() {
@@ -2804,12 +3443,7 @@ function fireproof(name, opts) {
2804
3443
  const key = JSON.stringify(
2805
3444
  toSortedArray({
2806
3445
  name,
2807
- stores: toSortedArray(opts?.store?.stores),
2808
- makeMetaStore: !!opts?.store?.makeMetaStore,
2809
- makeDataStore: !!opts?.store?.makeDataStore,
2810
- makeRemoteWAL: !!opts?.store?.makeRemoteWAL,
2811
- encodeFile: !!opts?.store?.encodeFile,
2812
- decodeFile: !!opts?.store?.decodeFile
3446
+ stores: toSortedArray(opts?.store?.stores)
2813
3447
  })
2814
3448
  );
2815
3449
  let db = Database.databases.get(key);
@@ -2824,7 +3458,10 @@ function makeName(fnString) {
2824
3458
  let found = null;
2825
3459
  const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
2826
3460
  if (matches.length === 0) {
2827
- found = /=>\s*(.*)/.exec(fnString);
3461
+ found = /=>\s*{?\s*([^{}]+)\s*}?/.exec(fnString);
3462
+ if (found && found[1].includes("return")) {
3463
+ found = null;
3464
+ }
2828
3465
  }
2829
3466
  if (!found) {
2830
3467
  return fnString;
@@ -2835,18 +3472,22 @@ function makeName(fnString) {
2835
3472
 
2836
3473
  // src/version.ts
2837
3474
  var PACKAGE_VERSION = Object.keys({
2838
- "0.19.8-dev-global": "xxxx"
3475
+ "0.19.11-dev-dryrun": "xxxx"
2839
3476
  })[0];
2840
3477
  export {
2841
3478
  CRDT,
2842
3479
  Database,
2843
3480
  Index,
3481
+ NotFoundError,
2844
3482
  PACKAGE_VERSION,
2845
3483
  Result,
3484
+ UInt8ArrayEqual,
2846
3485
  blockstore_exports as blockstore,
2847
3486
  blockstore_exports as bs,
3487
+ dataDir,
2848
3488
  ensureLogger,
2849
- exception2Result,
3489
+ ensureSuperLog,
3490
+ ensureSuperThis,
2850
3491
  exceptionWrapper,
2851
3492
  falsyToUndef,
2852
3493
  fireproof,
@@ -2855,6 +3496,7 @@ export {
2855
3496
  getStore,
2856
3497
  index,
2857
3498
  isFalsy,
3499
+ isNotFoundError,
2858
3500
  runtime_exports as rt,
2859
3501
  runtime_exports as runtime,
2860
3502
  throwFalsy