@fireproof/core 0.19.8-dev-global → 0.19.8-dev-series-2

package/index.js

@@ -1,30 +1,33 @@
+import {
+  FILESTORE_VERSION
+} from "./chunk-7EWIAXTM.js";
+import {
+  getFileName,
+  getFileSystem,
+  getPath,
+  toArrayBuffer
+} from "./chunk-WS3YRPIA.js";
+import {
+  INDEXDB_VERSION
+} from "./chunk-PB4BKL4O.js";
 import {
   NotFoundError,
   Result,
+  UInt8ArrayEqual,
+  __export,
   dataDir,
-  decodeFile,
-  encodeFile,
   ensureLogger,
-  exception2Result,
+  ensureSuperLog,
+  ensureSuperThis,
   exceptionWrapper,
   getKey,
   getName,
   getStore,
-  isNotFoundError,
-  runtime_exports,
-  toCryptoOpts
-} from "./chunk-BNL4PVBF.js";
-import {
-  SysContainer,
-  __export,
-  falsyToUndef,
-  isFalsy,
-  throwFalsy
-} from "./chunk-JW2QT6BF.js";
+  isNotFoundError
+} from "./chunk-OFGPKRCM.js";
 
 // src/database.ts
-import { uuidv7 } from "uuidv7";
-import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
+import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
 
 // src/write-queue.ts
 function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -67,13 +70,83 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
 }
 
 // src/crdt.ts
-import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
+import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
+
+// src/runtime/wait-pr-multiformats/block.ts
+var block_exports = {};
+__export(block_exports, {
+  Block: () => Block,
+  create: () => create,
+  createUnsafe: () => createUnsafe,
+  decode: () => decode,
+  encode: () => encode
+});
+import { bytes as binary, CID } from "multiformats";
+import { Block as mfBlock } from "multiformats/block";
+var Block = mfBlock;
+async function decode({
+  bytes,
+  codec: codec3,
+  hasher: hasher7
+}) {
+  if (bytes == null) throw new Error('Missing required argument "bytes"');
+  if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
+  const value = await Promise.resolve(codec3.decode(bytes));
+  const hash = await hasher7.digest(bytes);
+  const cid = CID.create(1, codec3.code, hash);
+  return new mfBlock({ value, bytes, cid });
+}
+async function encode({
+  value,
+  codec: codec3,
+  hasher: hasher7
+}) {
+  if (typeof value === "undefined") throw new Error('Missing required argument "value"');
+  if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
+  const bytes = await Promise.resolve(codec3.encode(value));
+  const hash = await hasher7.digest(bytes);
+  const cid = CID.create(1, codec3.code, hash);
+  return new mfBlock({ value, bytes, cid });
+}
+async function create({
+  bytes,
+  cid,
+  hasher: hasher7,
+  codec: codec3
+}) {
+  if (bytes == null) throw new Error('Missing required argument "bytes"');
+  if (hasher7 == null) throw new Error('Missing required argument "hasher"');
+  const value = await Promise.resolve(codec3.decode(bytes));
+  const hash = await hasher7.digest(bytes);
+  if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
+    throw new Error("CID hash does not match bytes");
+  }
+  return createUnsafe({
+    bytes,
+    cid,
+    value,
+    codec: codec3
+  });
+}
+async function createUnsafe({
+  bytes,
+  cid,
+  value: maybeValue,
+  codec: codec3
+}) {
+  const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
+  if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
+  return new Block({
+    cid,
+    bytes,
+    value
+  });
+}
 
 // src/crdt-helpers.ts
-import { encode as encode3, decode as decode3, Block as Block2 } from "multiformats/block";
-import { parse as parse2 } from "multiformats/link";
-import { sha256 as hasher2 } from "multiformats/hashes/sha2";
-import * as codec2 from "@ipld/dag-cbor";
+import { parse as parse3 } from "multiformats/link";
+import { sha256 as hasher5 } from "multiformats/hashes/sha2";
+import * as codec from "@ipld/dag-cbor";
 import { put, get, entries, root } from "@web3-storage/pail/crdt";
 import { EventFetcher, vis } from "@web3-storage/pail/clock";
 import * as Batch from "@web3-storage/pail/crdt/batch";
@@ -84,512 +157,146 @@ __export(blockstore_exports, {
   BaseBlockstore: () => BaseBlockstore,
   CarTransaction: () => CarTransaction,
   CompactionFetcher: () => CompactionFetcher,
-  ConnectREST: () => ConnectREST,
   ConnectionBase: () => ConnectionBase,
-  DataStore: () => DataStore,
   EncryptedBlockstore: () => EncryptedBlockstore,
-  Loadable: () => Loadable,
+  FragmentGateway: () => FragmentGateway,
   Loader: () => Loader,
-  MetaStore: () => MetaStore,
-  NotFoundError: () => NotFoundError,
-  RemoteWAL: () => RemoteWAL,
-  isNotFoundError: () => isNotFoundError,
+  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
+  ensureStart: () => ensureStart,
+  getGatewayFromURL: () => getGatewayFromURL,
   parseCarFile: () => parseCarFile,
   registerStoreProtocol: () => registerStoreProtocol,
+  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
   testStoreFactory: () => testStoreFactory,
-  toStoreRuntime: () => toStoreRuntime,
-  toURL: () => toURL
+  toCIDBlock: () => toCIDBlock,
+  toStoreRuntime: () => toStoreRuntime
 });
 
-// src/blockstore/connection-base.ts
-import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
-import { MemoryBlockstore } from "@web3-storage/pail/block";
+// src/blockstore/types.ts
+function toCIDBlock(block) {
+  return block;
+}
 
-// src/blockstore/task-manager.ts
-var TaskManager = class {
-  constructor(loader) {
-    this.eventsWeHandled = /* @__PURE__ */ new Set();
-    this.queue = [];
-    this.isProcessing = false;
-    this.loader = loader;
-    this.logger = ensureLogger(loader.logger, "TaskManager");
-  }
-  async handleEvent(eventBlock) {
-    const cid = eventBlock.cid.toString();
-    const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
-    for (const parent of parents) {
-      this.eventsWeHandled.add(parent);
-    }
-    this.queue.push({ cid, eventBlock, retries: 0 });
-    this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
-    void this.processQueue();
-  }
-  async processQueue() {
-    if (this.isProcessing) return;
-    this.isProcessing = true;
-    const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
-    const first = filteredQueue[0];
-    if (!first) {
-      return;
-    }
-    try {
-      this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
-      this.eventsWeHandled.add(first.cid);
-      this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
-    } catch (err) {
-      if (first.retries++ > 3) {
-        this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
-        this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
-      }
-      await new Promise((resolve) => setTimeout(resolve, 50));
-      throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
-    } finally {
-      this.isProcessing = false;
-      if (this.queue.length > 0) {
-        void this.processQueue();
-      }
-    }
-  }
-};
+// src/blockstore/store-factory.ts
+import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
 
-// src/blockstore/connection-base.ts
-var ConnectionBase = class {
-  constructor(logger) {
-    // readonly ready: Promise<unknown>;
-    // todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
-    this.eventBlocks = new MemoryBlockstore();
-    this.parents = [];
-    this.loaded = Promise.resolve();
-    this.logger = ensureLogger(logger, "ConnectionBase");
-  }
-  async refresh() {
-    await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load("main");
-    await (await throwFalsy(this.loader).remoteWAL())._process();
-  }
-  connect({ loader }) {
-    if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
-    this.connectMeta({ loader });
-    this.connectStorage({ loader });
-  }
-  connectMeta({ loader }) {
-    if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
-    this.loader = loader;
-    this.taskManager = new TaskManager(loader);
-    this.onConnect();
-    this.logger.Warn().Msg("connectMeta: connecting to remote meta store is disabled");
-  }
-  async onConnect() {
-    return;
-  }
-  connectStorage({ loader }) {
-    if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
-    this.loader = loader;
-    this.logger.Warn().Msg("connectStorage: connecting to remote meta store is disabled");
-  }
-  async createEventBlock(bytes) {
-    const data = {
-      dbMeta: bytes
-    };
-    const event = await EventBlock.create(
-      data,
-      this.parents
+// src/runtime/files.ts
+var files_exports = {};
+__export(files_exports, {
+  decodeFile: () => decodeFile,
+  encodeFile: () => encodeFile
+});
+import * as UnixFS from "@ipld/unixfs";
+import * as raw from "multiformats/codecs/raw";
+import { withMaxChunkSize } from "@ipld/unixfs/file/chunker/fixed";
+import { withWidth } from "@ipld/unixfs/file/layout/balanced";
+import { exporter } from "ipfs-unixfs-exporter";
+var queuingStrategy = UnixFS.withCapacity();
+var settings = UnixFS.configure({
+  fileChunkEncoder: raw,
+  smallFileEncoder: raw,
+  chunker: withMaxChunkSize(1024 * 1024),
+  fileLayout: withWidth(1024)
+});
+async function collect(collectable) {
+  const chunks = [];
+  await collectable.pipeTo(
+    new WritableStream({
+      write(chunk) {
+        chunks.push(chunk);
+      }
+    })
+  );
+  return chunks;
+}
+async function encodeFile(blob) {
+  const readable = createFileEncoderStream(blob);
+  const blocks = await collect(readable);
+  return { cid: blocks.at(-1).cid, blocks };
+}
+async function decodeFile(blocks, cid, meta) {
+  const entry = await exporter(cid.toString(), blocks, { length: meta.size });
+  const chunks = [];
+  for await (const chunk of entry.content()) {
+    chunks.push(chunk);
+  }
+  return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
+}
+function createFileEncoderStream(blob) {
+  const { readable, writable } = new TransformStream({}, queuingStrategy);
+  const unixfsWriter = UnixFS.createWriter({ writable, settings });
+  const fileBuilder = new UnixFSFileBuilder("", blob);
+  void (async () => {
+    await fileBuilder.finalize(unixfsWriter);
+    await unixfsWriter.close();
+  })();
+  return readable;
+}
+var UnixFSFileBuilder = class {
+  #file;
+  constructor(name, file) {
+    this.name = name;
+    this.#file = file;
+  }
+  async finalize(writer) {
+    const unixfsFileWriter = UnixFS.createFileWriter(writer);
+    await this.#file.stream().pipeTo(
+      new WritableStream({
+        async write(chunk) {
+          await unixfsFileWriter.write(chunk);
+        }
+      })
     );
-    await this.eventBlocks.put(event.cid, event.bytes);
-    return event;
-  }
-  async decodeEventBlock(bytes) {
-    const event = await decodeEventBlock(bytes);
-    return event;
+    return await unixfsFileWriter.close();
   }
-  // move this stuff to connect
-  // async getDashboardURL(compact = true) {
-  //   const baseUrl = 'https://dashboard.fireproof.storage/'
-  //   if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
-  //   // if (compact) {
-  //   //   await this.compact()
-  //   // }
-  //   const currents = await this.loader?.metaStore?.load()
-  //   if (!currents) throw new Error("Can't sync empty database: save data first")
-  //   if (currents.length > 1)
-  //     throw new Error("Can't sync database with split heads: make an update first")
-  //   const current = currents[0]
-  //   const params = {
-  //     car: current.car.toString()
-  //   }
-  //   if (current.key) {
-  //     // @ts-ignore
-  //     params.key = current.key.toString()
-  //   }
-  //   // @ts-ignore
-  //   if (this.name) {
-  //     // @ts-ignore
-  //     params.name = this.name
-  //   }
-  //   const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
-  //   console.log('Import to dashboard: ' + url.toString())
-  //   return url
-  // }
-  // openDashboard() {
-  //   void this.getDashboardURL().then(url => {
-  //     if (url) window.open(url.toString(), '_blank')
-  //   })
-  // }
 };
 
-// src/blockstore/connect-rest.ts
-var ConnectREST = class extends ConnectionBase {
-  constructor(base, logger) {
-    super(ensureLogger(logger, "ConnectREST"));
-    this.baseUrl = new URL(base);
-  }
-  async dataUpload(bytes, params) {
-    const carCid = params.car.toString();
-    const uploadURL = new URL(`/cars/${carCid}.car`, this.baseUrl);
-    const done = await fetch(uploadURL, { method: "PUT", body: bytes });
-    if (!done.ok) {
-      throw this.logger.Error().Msg("failed to upload data " + done.statusText);
-    }
-  }
-  async dataDownload(params) {
-    const { car } = params;
-    const fetchFromUrl = new URL(`/cars/${car.toString()}.car`, this.baseUrl);
-    const response = await fetch(fetchFromUrl);
-    if (!response.ok) {
-      return void 0;
-    }
-    const bytes = new Uint8Array(await response.arrayBuffer());
-    return bytes;
+// src/blockstore/store.ts
+import { format as format2, parse as parse2 } from "@ipld/dag-json";
+import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result5 } from "@adviser/cement";
+
+// src/types.ts
+function isFalsy(value) {
+  return value === false && value === null && value === void 0;
+}
+function throwFalsy(value) {
+  if (isFalsy(value)) {
+    throw new Error("value is Falsy");
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async metaUpload(bytes, params) {
+  return value;
+}
+function falsyToUndef(value) {
+  if (isFalsy(value)) {
     return void 0;
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async metaDownload(params) {
-    return [];
-  }
-};
-
-// src/blockstore/store-factory.ts
-import { KeyedResolvOnce } from "@adviser/cement";
-
-// src/blockstore/store.ts
-import pLimit2 from "p-limit";
-import { format, parse } from "@ipld/dag-json";
-import { ResolveOnce as ResolveOnce2, Result as Result2 } from "@adviser/cement";
+  return value;
+}
 
 // src/blockstore/loader.ts
 import pLimit from "p-limit";
 import { CarReader } from "@ipld/car";
-import { ResolveOnce } from "@adviser/cement";
-
-// src/blockstore/types.ts
-function toCIDBlock(block) {
-  return block;
-}
+import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
 
 // src/blockstore/loader-helpers.ts
-import { encode, decode } from "multiformats/block";
 import { sha256 as hasher } from "multiformats/hashes/sha2";
-import * as raw from "multiformats/codecs/raw";
-import * as CBW from "@ipld/car/buffer-writer";
-import * as codec from "@ipld/dag-cbor";
-async function encodeCarFile(roots, t) {
-  let size = 0;
-  const headerSize = CBW.headerLength({ roots });
-  size += headerSize;
-  for (const { cid, bytes } of t.entries()) {
-    size += CBW.blockLength({ cid, bytes });
-  }
-  const buffer = new Uint8Array(size);
-  const writer = CBW.createWriter(buffer, { headerSize });
-  for (const r of roots) {
-    writer.addRoot(r);
-  }
-  for (const { cid, bytes } of t.entries()) {
-    writer.write({ cid, bytes });
-  }
-  writer.close();
-  return await encode({ value: writer.bytes, hasher, codec: raw });
-}
-async function encodeCarHeader(fp) {
-  return await encode({
-    value: { fp },
-    hasher,
-    codec
-  });
-}
+import * as dagCodec from "@ipld/dag-cbor";
 async function parseCarFile(reader, logger) {
   const roots = await reader.getRoots();
   const header = await reader.get(roots[0]);
   if (!header) throw logger.Error().Msg("missing header block").AsError();
-  const { value } = await decode({ bytes: header.bytes, hasher, codec });
-  const fpvalue = value;
+  const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
+  const fpvalue = dec.value;
   if (fpvalue && !fpvalue.fp) {
     throw logger.Error().Msg("missing fp").AsError();
   }
   return fpvalue.fp;
 }
 
-// src/blockstore/encrypt-helpers.ts
-import { sha256 } from "multiformats/hashes/sha2";
-import { CID as CID2 } from "multiformats";
-import { encode as encode2, decode as decode2, create as mfCreate } from "multiformats/block";
-import * as dagcbor from "@ipld/dag-cbor";
-import { MemoryBlockstore as MemoryBlockstore2 } from "@web3-storage/pail/block";
-import { bf } from "prolly-trees/utils";
-import { nocache as cache } from "prolly-trees/cache";
-import { create, load } from "prolly-trees/cid-set";
-
-// src/blockstore/encrypt-codec.ts
-import { CID } from "multiformats";
-function makeCodec(ilogger, crypto, randomBytes) {
-  const logger = ensureLogger(ilogger, "makeCodec");
-  const enc32 = (value) => {
-    value = +value;
-    const buff = new Uint8Array(4);
-    buff[3] = value >>> 24;
-    buff[2] = value >>> 16;
-    buff[1] = value >>> 8;
-    buff[0] = value & 255;
-    return buff;
-  };
-  const readUInt32LE = (buffer) => {
-    const offset = buffer.byteLength - 4;
-    return (buffer[offset] | buffer[offset + 1] << 8 | buffer[offset + 2] << 16) + buffer[offset + 3] * 16777216;
-  };
-  const concat = (buffers) => {
-    const uint8Arrays = buffers.map((b) => b instanceof ArrayBuffer ? new Uint8Array(b) : b);
-    const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0);
-    const result = new Uint8Array(totalLength);
-    let offset = 0;
-    for (const arr of uint8Arrays) {
-      result.set(arr, offset);
-      offset += arr.length;
-    }
-    return result;
-  };
-  const encode4 = ({ iv, bytes }) => concat([iv, bytes]);
-  const decode4 = (bytes) => {
-    const iv = bytes.subarray(0, 12);
-    bytes = bytes.slice(12);
-    return { iv, bytes };
-  };
-  const code = 3145728 + 1337;
-  async function subtleKey(key) {
-    return await crypto.importKey(
-      "raw",
-      // raw or jwk
-      key,
-      // raw data
-      "AES-GCM",
-      false,
-      // extractable
-      ["encrypt", "decrypt"]
-    );
-  }
-  const decrypt = async ({ key, value }) => {
-    const { bytes: inBytes, iv } = value;
-    const cryKey = await subtleKey(key);
-    const deBytes = await crypto.decrypt(
-      {
-        name: "AES-GCM",
-        iv,
-        tagLength: 128
-      },
-      cryKey,
-      inBytes
-    );
-    const bytes = new Uint8Array(deBytes);
-    const len = readUInt32LE(bytes.subarray(0, 4));
-    const cid = CID.decode(bytes.subarray(4, 4 + len));
-    return { cid, bytes: bytes.subarray(4 + len) };
-  };
-  const encrypt = async ({ key, cid, bytes }) => {
-    const len = enc32(cid.bytes.byteLength);
-    const iv = randomBytes(12);
-    const msg = concat([len, cid.bytes, bytes]);
-    try {
-      const cryKey = await subtleKey(key);
-      const deBytes = await crypto.encrypt(
-        {
-          name: "AES-GCM",
-          iv,
-          tagLength: 128
-        },
-        cryKey,
-        msg
-      );
-      bytes = new Uint8Array(deBytes);
-    } catch (e) {
-      throw logger.Error().Err(e).Msg("encrypt failed").AsError();
-    }
-    return { value: { bytes, iv } };
-  };
-  const cryptoFn = (key) => {
-    return { encrypt: (opts) => encrypt({ ...opts, key }), decrypt: (opts) => decrypt({ ...opts, key }) };
-  };
-  const name = "jchris@encrypted-block:aes-gcm";
-  return { encode: encode4, decode: decode4, code, name, encrypt, decrypt, crypto: cryptoFn };
-}
-
-// src/blockstore/encrypt-helpers.ts
-function carLogIncludesGroup(list, cidMatch) {
-  return list.some((cid) => {
-    return cid.toString() === cidMatch.toString();
-  });
-}
-function makeEncDec(logger, crypto, randomBytes) {
-  const codec4 = makeCodec(logger, crypto, randomBytes);
-  const encrypt = async function* ({
-    get: get2,
-    cids,
-    hasher: hasher4,
-    key,
-    cache: cache3,
-    chunker: chunker2,
-    root: root3
-  }) {
-    const set = /* @__PURE__ */ new Set();
-    let eroot;
-    if (!carLogIncludesGroup(cids, root3)) cids.push(root3);
-    for (const cid of cids) {
-      const unencrypted = await get2(cid);
-      if (!unencrypted) throw logger.Error().Ref("cid", cid).Msg("missing cid block").AsError();
-      const encrypted = await codec4.encrypt({ ...unencrypted, key });
-      const block2 = await encode2({ ...encrypted, codec: codec4, hasher: hasher4 });
-      yield block2;
-      set.add(block2.cid.toString());
-      if (unencrypted.cid.equals(root3)) eroot = block2.cid;
-    }
-    if (!eroot) throw logger.Error().Msg("cids does not include root").AsError();
-    const list = [...set].map((s) => CID2.parse(s));
-    let last;
-    for await (const node of create({ list, get: get2, cache: cache3, chunker: chunker2, hasher: hasher4, codec: dagcbor })) {
-      const block2 = await node.block;
-      yield block2;
-      last = block2;
-    }
-    if (!last) throw logger.Error().Msg("missing last block").AsError();
-    const head = [eroot, last.cid];
-    const block = await encode2({ value: head, codec: dagcbor, hasher: hasher4 });
-    yield block;
-  };
-  const decrypt = async function* ({
-    root: root3,
-    get: get2,
-    key,
-    cache: cache3,
-    chunker: chunker2,
-    hasher: hasher4
-  }) {
-    const getWithDecode = async (cid) => get2(cid).then(async (block) => {
-      if (!block) return;
-      const decoded = await decode2({ ...block, codec: dagcbor, hasher: hasher4 });
-      return decoded;
-    });
-    const getWithDecrypt = async (cid) => get2(cid).then(async (block) => {
-      if (!block) return;
-      const decoded = await decode2({ ...block, codec: codec4, hasher: hasher4 });
-      return decoded;
-    });
-    const decodedRoot = await getWithDecode(root3);
-    if (!decodedRoot) throw logger.Error().Msg("missing root").AsError();
-    if (!decodedRoot.bytes) throw logger.Error().Msg("missing bytes").AsError();
-    const {
-      value: [eroot, tree]
-    } = decodedRoot;
-    const rootBlock = await get2(eroot);
-    if (!rootBlock) throw logger.Error().Msg("missing root block").AsError();
-    const cidset = await load({ cid: tree, get: getWithDecode, cache: cache3, chunker: chunker2, codec: codec4, hasher: hasher4 });
-    const { result: nodes } = await cidset.getAllEntries();
-    const unwrap = async (eblock) => {
-      if (!eblock) throw logger.Error().Msg("missing block").AsError();
-      if (!eblock.value) {
-        eblock = await decode2({ ...eblock, codec: codec4, hasher: hasher4 });
-        if (!eblock.value) throw logger.Error().Msg("missing value").AsError();
-      }
-      const { bytes, cid } = await codec4.decrypt({ ...eblock, key }).catch((e) => {
-        throw e;
-      });
-      const block = await mfCreate({ cid, bytes, hasher: hasher4, codec: codec4 });
-      return block;
-    };
-    const promises = [];
-    for (const { cid } of nodes) {
-      if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap));
-    }
-    yield* promises;
-    yield unwrap(rootBlock);
-  };
-  return { encrypt, decrypt };
-}
-var chunker = bf(30);
-function hexStringToUint8Array(hexString) {
-  const length = hexString.length;
-  const uint8Array = new Uint8Array(length / 2);
-  for (let i = 0; i < length; i += 2) {
-    uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16);
-  }
-  return uint8Array;
-}
-async function encryptedEncodeCarFile(logger, crypto, key, rootCid, t) {
-  const encryptionKey = hexStringToUint8Array(key);
-  const encryptedBlocks = new MemoryBlockstore2();
-  const cidsToEncrypt = [];
-  for (const { cid, bytes } of t.entries()) {
-    cidsToEncrypt.push(cid);
-    const g = await t.get(cid);
-    if (!g) throw logger.Error().Ref("cid", cid).Int("bytes", bytes.length).Msg("missing cid block").AsError();
-  }
-  let last = null;
-  const { encrypt } = makeEncDec(logger, crypto, crypto.randomBytes);
-  for await (const block of encrypt({
-    cids: cidsToEncrypt,
-    get: t.get.bind(t),
-    key: encryptionKey,
-    hasher: sha256,
-    chunker,
-    cache,
-    root: rootCid
-  })) {
-    await encryptedBlocks.put(block.cid, block.bytes);
-    last = block;
-  }
-  if (!last) throw logger.Error().Msg("no blocks encrypted").AsError();
-  const encryptedCar = await encodeCarFile([last.cid], encryptedBlocks);
-  return encryptedCar;
-}
-async function decodeEncryptedCar(logger, crypto, key, reader) {
-  const roots = await reader.getRoots();
-  const root3 = roots[0];
-  return await decodeCarBlocks(logger, crypto, root3, reader.get.bind(reader), key);
-}
-async function decodeCarBlocks(logger, crypto, root3, get2, keyMaterial) {
-  const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial);
-  const decryptionKey = decryptionKeyUint8.buffer.slice(0, decryptionKeyUint8.byteLength);
-  const decryptedBlocks = new MemoryBlockstore2();
-  let last = null;
-  const { decrypt } = makeEncDec(logger, crypto, crypto.randomBytes);
-  for await (const block of decrypt({
-    root: root3,
-    get: get2,
-    key: decryptionKey,
-    hasher: sha256,
-    chunker,
-    cache
-  })) {
-    await decryptedBlocks.put(block.cid, block.bytes);
-    last = block;
-  }
-  if (!last) throw logger.Error().Msg("no blocks decrypted").AsError();
-  return { blocks: decryptedBlocks, root: last.cid };
-}
-
 // src/blockstore/transaction.ts
-import { MemoryBlockstore as MemoryBlockstore3 } from "@web3-storage/pail/block";
-var CarTransaction = class extends MemoryBlockstore3 {
-  constructor(parent, opts = { add: true }) {
+import { MemoryBlockstore } from "@web3-storage/pail/block";
+import { toCryptoRuntime } from "@adviser/cement";
+var CarTransaction = class extends MemoryBlockstore {
+  constructor(parent, opts = { add: true, noLoader: false }) {
     super();
     if (opts.add) {
       parent.transactions.add(this);
@@ -603,8 +310,8 @@ var CarTransaction = class extends MemoryBlockstore3 {
     return super.get(cid);
   }
 };
-function defaultedBlockstoreRuntime(opts, component, ctx) {
-  const logger = ensureLogger(opts, component, ctx);
+function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
+  const logger = ensureLogger(sthis, component, ctx);
   const store = opts.store || {};
   return {
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -621,22 +328,24 @@ function defaultedBlockstoreRuntime(opts, component, ctx) {
     threshold: 1e3 * 1e3,
     ...opts,
     logger,
-    crypto: toCryptoOpts(opts.crypto),
+    keyBag: opts.keyBag || {},
+    crypto: toCryptoRuntime(opts.crypto),
     store,
-    storeRuntime: toStoreRuntime(store, logger)
+    storeRuntime: toStoreRuntime(store, sthis)
   };
 }
-var blockstoreFactory = function(opts) {
+function blockstoreFactory(sthis, opts) {
   if (opts.name) {
-    return new EncryptedBlockstore(opts);
+    return new EncryptedBlockstore(sthis, opts);
   } else {
     return new BaseBlockstore(opts);
   }
-};
+}
 var BaseBlockstore = class {
   constructor(ebOpts = {}) {
     this.transactions = /* @__PURE__ */ new Set();
-    this.ebOpts = defaultedBlockstoreRuntime(ebOpts, "BaseBlockstore");
+    this.sthis = ensureSuperThis(ebOpts);
+    this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
     this.logger = this.ebOpts.logger;
   }
   // ready: Promise<void>;
@@ -659,8 +368,8 @@ var BaseBlockstore = class {
     throw this.logger.Error().Msg("use a transaction to put").AsError();
   }
   // TransactionMeta
-  async transaction(fn, _opts = {}) {
-    const t = new CarTransaction(this);
+  async transaction(fn, _opts) {
+    const t = new CarTransaction(this, _opts);
     const done = await fn(t);
     this.lastTxMeta = done;
     return { t, meta: done };
@@ -677,16 +386,16 @@ var BaseBlockstore = class {
   }
 };
 var EncryptedBlockstore = class extends BaseBlockstore {
-  constructor(ebOpts) {
+  constructor(sthis, ebOpts) {
     super(ebOpts);
     this.compacting = false;
-    this.logger = ensureLogger(ebOpts, "EncryptedBlockstore");
+    this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
     const { name } = ebOpts;
     if (!name) {
       throw this.logger.Error().Msg("name required").AsError();
     }
     this.name = name;
-    this.loader = new Loader(this.name, ebOpts);
+    this.loader = new Loader(this.name, ebOpts, sthis);
   }
   ready() {
     return this.loader.ready();
@@ -717,10 +426,13 @@ var EncryptedBlockstore = class extends BaseBlockstore {
     }
     throw this.logger.Error().Msg("failed to commit car files").AsError();
   }
-  async getFile(car, cid, isPublic = false) {
+  async getFile(car, cid) {
     await this.ready();
     if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
-    const reader = await this.loader.loadFileCar(car, isPublic);
+    const reader = await this.loader.loadFileCar(
+      car
+      /*, isPublic */
+    );
     const block = await reader.get(cid);
     if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
     return block.bytes;
@@ -776,10 +488,20 @@ var CompactionFetcher = class {
 };
 
 // src/blockstore/commit-queue.ts
+import { Future } from "@adviser/cement";
 var CommitQueue = class {
   constructor() {
     this.queue = [];
     this.processing = false;
+    this._waitIdleItems = /* @__PURE__ */ new Set();
+  }
+  waitIdle() {
+    if (this.queue.length === 0 && !this.processing) {
+      return Promise.resolve();
+    }
+    const fn = new Future();
+    this._waitIdleItems.add(fn);
+    return fn.asPromise();
   }
   async enqueue(fn) {
     return new Promise((resolve, reject) => {
@@ -804,38 +526,384 @@ var CommitQueue = class {
       this.processing = true;
       const queueFn = this.queue.shift();
       if (queueFn) {
-        queueFn();
+        queueFn().finally(() => {
+        });
       }
     }
+    if (this.queue.length === 0 && !this.processing) {
+      const toResolve = Array.from(this._waitIdleItems);
+      this._waitIdleItems.clear();
+      toResolve.map((fn) => fn.resolve());
+    }
   }
 };
 
-// src/blockstore/loader.ts
-import * as CBW2 from "@ipld/car/buffer-writer";
-function carLogIncludesGroup2(list, cids) {
-  return list.some((arr) => {
-    return arr.toString() === cids.toString();
-  });
-}
-function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
-  const byString = /* @__PURE__ */ new Map();
-  for (const cid of list) {
-    if (remove.has(cid.toString())) continue;
-    byString.set(cid.toString(), cid);
+// src/runtime/key-bag.ts
+var key_bag_exports = {};
+__export(key_bag_exports, {
+  KeyBag: () => KeyBag,
+  getKeyBag: () => getKeyBag,
+  registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
+});
+import {
+  KeyedResolvOnce,
+  ResolveOnce,
+  ResolveSeq,
+  Result as Result2,
+  runtimeFn,
+  toCryptoRuntime as toCryptoRuntime2,
+  URI
+} from "@adviser/cement";
+import { base58btc } from "multiformats/bases/base58";
+var KeyBag = class {
+  constructor(rt) {
+    this.rt = rt;
+    this._warnOnce = new ResolveOnce();
+    this._seq = new ResolveSeq();
+    this.logger = ensureLogger(rt.sthis, "KeyBag");
+    this.logger.Debug().Msg("KeyBag created");
+  }
+  async subtleKey(key) {
+    const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
+    if (extractable) {
+      this._warnOnce.once(
+        () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
+      );
+    }
+    return await this.rt.crypto.importKey(
+      "raw",
+      // raw or jwk
+      base58btc.decode(key),
+      // hexStringToUint8Array(key), // raw data
+      "AES-GCM",
+      extractable,
+      ["encrypt", "decrypt"]
+    );
   }
-  return [...byString.values()];
-}
-function toHexString(byteArray) {
-  return Array.from(byteArray).map((byte) => byte.toString(16).padStart(2, "0")).join("");
-}
-var Loadable = class {
-  constructor() {
-    this.name = "";
-    this.carLog = new Array();
+  async ensureKeyFromUrl(url, keyFactory) {
+    const storeKey = url.getParam("storekey");
+    if (storeKey === "insecure") {
+      return Result2.Ok(url);
+    }
+    if (!storeKey) {
+      const keyName = `@${keyFactory()}@`;
+      const ret = await this.getNamedKey(keyName);
+      if (ret.isErr()) {
+        return ret;
+      }
+      const urb = url.build().setParam("storekey", keyName);
+      return Result2.Ok(urb.URI());
+    }
+    if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
+      const ret = await this.getNamedKey(storeKey);
+      if (ret.isErr()) {
+        return ret;
+      }
+    }
+    return Result2.Ok(url);
+  }
+  async toKeyWithFingerPrint(keyStr) {
+    const material = base58btc.decode(keyStr);
+    const key = await this.subtleKey(keyStr);
+    const fpr = await this.rt.crypto.digestSHA256(material);
+    return Result2.Ok({
+      key,
+      fingerPrint: base58btc.encode(new Uint8Array(fpr))
+    });
+  }
+  async setNamedKey(name, key) {
+    return this._seq.add(() => this._setNamedKey(name, key));
+  }
+  // avoid deadlock
+  async _setNamedKey(name, key) {
+    const item = {
+      name,
+      key
+    };
+    const bag = await this.rt.getBag();
+    this.logger.Debug().Str("name", name).Msg("setNamedKey");
+    await bag.set(name, item);
+    return await this.toKeyWithFingerPrint(item.key);
+  }
+  async getNamedExtractableKey(name, failIfNotFound = false) {
+    const ret = await this.getNamedKey(name, failIfNotFound);
+    if (ret.isErr()) {
+      return ret;
+    }
+    const named = ret.Ok();
+    return Result2.Ok({
+      ...named,
+      extract: async () => {
+        const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
+        return {
+          key: ext,
+          keyStr: base58btc.encode(ext)
+        };
+      }
+    });
+  }
+  async getNamedKey(name, failIfNotFound = false) {
+    const id = this.rt.sthis.nextId(4).str;
+    return this._seq.add(async () => {
+      const bag = await this.rt.getBag();
+      const named = await bag.get(name);
+      if (named) {
+        const fpr = await this.toKeyWithFingerPrint(named.key);
+        this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
+        return fpr;
+      }
+      if (failIfNotFound) {
+        this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
+        return Result2.Err(new Error(`Key not found: ${name}`));
+      }
+      const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
+      this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
+      return ret;
+    });
   }
 };
+var keyBagProviderFactories = new Map(
+  [
+    {
+      protocol: "file:",
+      factory: async (url, sthis) => {
+        const { KeyBagProviderFile } = await import("./key-bag-file-WADZBHYG.js");
+        return new KeyBagProviderFile(url, sthis);
+      }
+    },
+    {
+      protocol: "indexdb:",
+      factory: async (url, sthis) => {
+        const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-PGVAI3FJ.js");
+        return new KeyBagProviderIndexDB(url, sthis);
+      }
+    }
+  ].map((i) => [i.protocol, i])
+);
+function registerKeyBagProviderFactory(item) {
+  const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
+  keyBagProviderFactories.set(protocol, {
+    ...item,
+    protocol
+  });
+}
+function defaultKeyBagOpts(sthis, kbo) {
+  if (kbo.keyRuntime) {
+    return kbo.keyRuntime;
+  }
+  const logger = ensureLogger(sthis, "KeyBag");
+  let url;
+  if (kbo.url) {
+    url = URI.from(kbo.url);
+    logger.Debug().Url(url).Msg("from opts");
+  } else {
+    let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
+    if (runtimeFn().isBrowser) {
+      url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
+    } else {
+      if (!bagFnameOrUrl) {
+        const home = sthis.env.get("HOME");
+        bagFnameOrUrl = `${home}/.fireproof/keybag`;
+        url = URI.from(`file://${bagFnameOrUrl}`);
+      } else {
+        url = URI.from(bagFnameOrUrl);
+      }
+    }
+    logger.Debug().Url(url).Msg("from env");
+  }
+  const kitem = keyBagProviderFactories.get(url.protocol);
+  if (!kitem) {
+    throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
+  }
+  const getBag = async () => kitem.factory(url, sthis);
+  if (url.hasParam("masterkey")) {
+    throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
+  }
+  return {
+    url,
+    crypto: kbo.crypto || toCryptoRuntime2({}),
+    sthis,
+    logger,
+    keyLength: kbo.keyLength || 16,
+    getBag,
+    id: () => {
+      return url.toString();
+    }
+  };
+}
+var _keyBags = new KeyedResolvOnce();
+async function getKeyBag(sthis, kbo = {}) {
+  await sthis.start();
+  const rt = defaultKeyBagOpts(sthis, kbo);
+  return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
+}
+
+// src/blockstore/commitor.ts
+import * as CBW from "@ipld/car/buffer-writer";
+import { sha256 as hasher2 } from "multiformats/hashes/sha2";
+import * as dagCodec2 from "@ipld/dag-cbor";
+async function encodeCarFile(roots, t, codec3) {
+  let size = 0;
+  const headerSize = CBW.headerLength({ roots });
+  size += headerSize;
+  for (const { cid, bytes } of t.entries()) {
+    size += CBW.blockLength({ cid, bytes });
+  }
+  const buffer = new Uint8Array(size);
+  const writer = CBW.createWriter(buffer, { headerSize });
+  for (const r of roots) {
+    writer.addRoot(r);
+  }
+  for (const { cid, bytes } of t.entries()) {
+    writer.write({ cid, bytes });
+  }
+  writer.close();
+  return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
+}
+async function createCarFile(encoder, cid, t) {
+  return encodeCarFile([cid], t, encoder);
+}
+async function commitFiles(fileStore, walStore, t, done) {
+  const { files: roots } = makeFileCarHeader(done);
+  const cids = [];
+  const codec3 = (await fileStore.keyedCrypto()).codec();
+  const cars = await prepareCarFilesFiles(codec3, roots, t);
+  for (const car of cars) {
+    const { cid, bytes } = car;
+    await fileStore.save({ cid, bytes });
+    await walStore.enqueueFile(
+      cid
+      /*, !!opts.public*/
+    );
+    cids.push(cid);
+  }
+  return cids;
+}
+function makeFileCarHeader(result) {
+  const files = [];
+  for (const [, meta] of Object.entries(result.files || {})) {
+    if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
+      files.push(meta.cid);
+    }
+  }
+  return { ...result, files };
+}
+async function prepareCarFilesFiles(encoder, roots, t) {
+  return [await encodeCarFile(roots, t, encoder)];
+}
+function makeCarHeader(meta, cars, compact = false) {
+  const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
+  return { ...coreHeader, meta };
+}
+async function encodeCarHeader(fp) {
+  return await encode({
+    value: { fp },
+    hasher: hasher2,
+    codec: dagCodec2
+  });
+}
+async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
+  const fp = makeCarHeader(done, params.carLog, !!opts.compact);
+  const rootBlock = await encodeCarHeader(fp);
+  const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
+  const cids = [];
+  for (const car of cars) {
+    const { cid, bytes } = car;
+    await params.carStore.save({ cid, bytes });
+    cids.push(cid);
+  }
+  const newDbMeta = { cars: cids };
+  await params.WALStore.enqueue(newDbMeta, opts);
+  await params.metaStore.save(newDbMeta);
+  return { cgrp: cids, header: fp };
+}
+async function prepareCarFiles(encoder, threshold, rootBlock, t) {
+  const carFiles = [];
+  threshold = threshold || 128e3 * 8;
+  let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
+  clonedt.putSync(rootBlock.cid, rootBlock.bytes);
+  let newsize = CBW.blockLength(toCIDBlock(rootBlock));
+  let cidRootBlock = rootBlock;
+  for (const { cid, bytes } of t.entries()) {
+    newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
+    if (newsize >= threshold) {
+      carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
+      clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
+      clonedt.putSync(cid, bytes);
+      cidRootBlock = { cid, bytes };
+      newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
+    } else {
+      clonedt.putSync(cid, bytes);
+    }
+  }
+  carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
+  return carFiles;
+}
+
+// src/blockstore/loader.ts
+import { sha256 as hasher3 } from "multiformats/hashes/sha2";
+
+// src/blockstore/task-manager.ts
+var TaskManager = class {
+  constructor(sthis, callback) {
+    this.eventsWeHandled = /* @__PURE__ */ new Set();
+    this.queue = [];
+    this.isProcessing = false;
+    this.logger = ensureLogger(sthis, "TaskManager");
+    this.callback = callback;
+  }
+  async handleEvent(cid, parents, dbMeta) {
+    for (const parent of parents) {
+      this.eventsWeHandled.add(parent.toString());
+    }
+    this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
+    this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
+    void this.processQueue();
+  }
+  async processQueue() {
+    if (this.isProcessing) return;
+    this.isProcessing = true;
+    const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
+    const first = filteredQueue[0];
+    if (!first) {
+      return;
+    }
+    try {
+      await this.callback(first.dbMeta);
+      this.eventsWeHandled.add(first.cid);
+      this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
+    } catch (err) {
+      if (first.retries++ > 3) {
+        this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
+        this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
+    } finally {
+      this.isProcessing = false;
+      if (this.queue.length > 0) {
+        void this.processQueue();
+      }
+    }
+  }
+};
+
+// src/blockstore/loader.ts
+function carLogIncludesGroup(list, cids) {
+  return list.some((arr) => {
+    return arr.toString() === cids.toString();
+  });
+}
+function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
+  const byString = /* @__PURE__ */ new Map();
+  for (const cid of list) {
+    if (remove.has(cid.toString())) continue;
+    byString.set(cid.toString(), cid);
+  }
+  return [...byString.values()];
+}
 var Loader = class {
-  constructor(name, ebOpts) {
+  constructor(name, ebOpts, sthis) {
     this.commitQueue = new CommitQueue();
     this.isCompacting = false;
     this.carReaders = /* @__PURE__ */ new Map();
@@ -845,9 +913,11 @@ var Loader = class {
     this.getBlockCache = /* @__PURE__ */ new Map();
     this.seenMeta = /* @__PURE__ */ new Set();
     this.writeLimit = pLimit(1);
-    this.onceReady = new ResolveOnce();
+    this.onceReady = new ResolveOnce2();
     this.name = name;
+    this.sthis = sthis;
     this.ebOpts = defaultedBlockstoreRuntime(
+      sthis,
       {
         ...ebOpts,
         name
@@ -855,34 +925,42 @@ var Loader = class {
       "Loader"
     );
     this.logger = this.ebOpts.logger;
+    this.taskManager = new TaskManager(sthis, async (dbMeta) => {
+      await this.handleDbMetasFromStore([dbMeta]);
+    });
   }
   // readonly id = uuidv4();
+  async keyBag() {
+    return getKeyBag(this.sthis, this.ebOpts.keyBag);
+  }
   async carStore() {
     return this.ebOpts.storeRuntime.makeDataStore(this);
   }
   async fileStore() {
     return this.ebOpts.storeRuntime.makeDataStore(this);
   }
-  async remoteWAL() {
-    return this.ebOpts.storeRuntime.makeRemoteWAL(this);
+  async WALStore() {
+    return this.ebOpts.storeRuntime.makeWALStore(this);
   }
   async metaStore() {
     return this.ebOpts.storeRuntime.makeMetaStore(this);
   }
   async ready() {
     return this.onceReady.once(async () => {
-      const metas = this.ebOpts.meta ? [this.ebOpts.meta] : await (await this.metaStore()).load("main");
-      if (metas) {
+      const metas = await (await this.metaStore()).load();
+      if (this.ebOpts.meta) {
+        await this.handleDbMetasFromStore([this.ebOpts.meta]);
+      } else if (metas) {
         await this.handleDbMetasFromStore(metas);
       }
     });
   }
   async close() {
-    const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.remoteWAL()]);
+    const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
     await Promise.all(toClose.map((store) => store.close()));
   }
   async destroy() {
-    const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.remoteWAL()]);
+    const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
     await Promise.all(toDestroy.map((store) => store.destroy()));
   }
   // async snapToCar(carCid: AnyLink | string) {
@@ -896,6 +974,7 @@ var Loader = class {
   //   await this._applyCarHeader(carHeader, true)
   // }
   async handleDbMetasFromStore(metas) {
+    this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
     for (const meta of metas) {
       await this.writeLimit(async () => {
         await this.mergeDbMetaIntoClock(meta);
@@ -908,10 +987,7 @@ var Loader = class {
     }
     if (this.seenMeta.has(meta.cars.toString())) return;
     this.seenMeta.add(meta.cars.toString());
-    if (meta.key) {
-      await this.setKey(meta.key);
-    }
-    if (carLogIncludesGroup2(this.carLog, meta.cars)) {
+    if (carLogIncludesGroup(this.carLog, meta.cars)) {
       return;
     }
     const carHeader = await this.loadCarHeaderFromMeta(meta);
@@ -920,45 +996,60 @@ var Loader = class {
     this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
     await this.ebOpts.applyMeta?.(carHeader.meta);
   }
-  async ingestKeyFromMeta(meta) {
-    const { key } = meta;
-    if (key) {
-      await this.setKey(key);
-    }
-  }
+  // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
+  //   const { key } = meta;
+  //   if (key) {
+  //     await this.setKey(key);
+  //   }
+  // }
   async loadCarHeaderFromMeta({ cars: cids }) {
     const reader = await this.loadCar(cids[0]);
     return await parseCarFile(reader, this.logger);
   }
-  async _getKey() {
-    if (this.key) return this.key;
-    if (!this.ebOpts.public) {
-      await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
-    }
-    return this.key || void 0;
-  }
-  async commitFiles(t, done, opts = { noLoader: false, compact: false }) {
-    return this.commitQueue.enqueue(() => this._commitInternalFiles(t, done, opts));
-  }
-  // can these skip the queue? or have a file queue?
-  async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
+  // async _getKey(): Promise<string | undefined> {
+  //   if (this.key) return this.key;
+  //   // generate a random key
+  //   if (!this.ebOpts.public) {
+  //     await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
+  //   }
+  //   return this.key || undefined;
+  // }
+  async commitFiles(t, done) {
     await this.ready();
-    const { files: roots } = this.makeFileCarHeader(done);
-    const cids = [];
-    const cars = await this.prepareCarFilesFiles(roots, t, !!opts.public);
-    for (const car of cars) {
-      const { cid, bytes } = car;
-      await (await this.fileStore()).save({ cid, bytes });
-      await (await this.remoteWAL()).enqueueFile(cid, !!opts.public);
-      cids.push(cid);
-    }
-    return cids;
+    const fstore = await this.fileStore();
+    const wstore = await this.WALStore();
+    return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
   }
-  async loadFileCar(cid, isPublic = false) {
-    return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore, isPublic);
+  async loadFileCar(cid) {
+    return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
   }
   async commit(t, done, opts = { noLoader: false, compact: false }) {
-    return this.commitQueue.enqueue(() => this._commitInternal(t, done, opts));
+    await this.ready();
+    const fstore = await this.fileStore();
+    const params = {
+      encoder: (await fstore.keyedCrypto()).codec(),
+      carLog: this.carLog,
+      carStore: fstore,
+      WALStore: await this.WALStore(),
+      metaStore: await this.metaStore(),
+      threshold: this.ebOpts.threshold
+    };
+    return this.commitQueue.enqueue(async () => {
+      await this.cacheTransaction(t);
+      const ret = await commit(params, t, done, opts);
+      await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
+      return ret.cgrp;
+    });
+  }
+  async updateCarLog(cids, fp, compact) {
+    if (compact) {
+      const previousCompactCid = fp.compact[fp.compact.length - 1];
+      fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
+      this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
+      await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
+    } else {
+      this.carLog.unshift(cids);
+    }
   }
   async cacheTransaction(t) {
     for await (const block of t.entries()) {
@@ -978,79 +1069,6 @@ var Loader = class {
       }
     }
   }
-  async _commitInternal(t, done, opts = { noLoader: false, compact: false }) {
-    await this.ready();
-    const fp = this.makeCarHeader(done, this.carLog, !!opts.compact);
-    const rootBlock = await encodeCarHeader(fp);
-    const cars = await this.prepareCarFiles(rootBlock, t, !!opts.public);
-    const cids = [];
-    for (const car of cars) {
-      const { cid, bytes } = car;
-      await (await this.carStore()).save({ cid, bytes });
-      cids.push(cid);
-    }
-    await this.cacheTransaction(t);
-    const newDbMeta = { cars: cids, key: this.key || null };
-    await (await this.remoteWAL()).enqueue(newDbMeta, opts);
-    await (await this.metaStore()).save(newDbMeta);
-    await this.updateCarLog(cids, fp, !!opts.compact);
-    return cids;
-  }
-  async prepareCarFilesFiles(roots, t, isPublic) {
-    const theKey = isPublic ? null : await this._getKey();
-    const car = theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, roots[0], t) : await encodeCarFile(roots, t);
-    return [car];
-  }
-  async prepareCarFiles(rootBlock, t, isPublic) {
-    const theKey = isPublic ? void 0 : await this._getKey();
-    const carFiles = [];
-    const threshold = this.ebOpts.threshold || 1e3 * 1e3;
-    let clonedt = new CarTransaction(t.parent, { add: false });
-    clonedt.putSync(rootBlock.cid, rootBlock.bytes);
-    let newsize = CBW2.blockLength(toCIDBlock(rootBlock));
-    let cidRootBlock = rootBlock;
-    for (const { cid, bytes } of t.entries()) {
-      newsize += CBW2.blockLength(toCIDBlock({ cid, bytes }));
-      if (newsize >= threshold) {
-        carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
-        clonedt = new CarTransaction(t.parent, { add: false });
-        clonedt.putSync(cid, bytes);
-        cidRootBlock = { cid, bytes };
-        newsize = CBW2.blockLength(toCIDBlock({ cid, bytes }));
-      } else {
-        clonedt.putSync(cid, bytes);
-      }
-    }
-    carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
-    return carFiles;
-  }
-  async createCarFile(theKey, cid, t) {
-    try {
-      return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, cid, t) : await encodeCarFile([cid], t);
-    } catch (e) {
-      console.error("error creating car file", e);
-      throw e;
-    }
-  }
-  makeFileCarHeader(result) {
-    const files = [];
-    for (const [, meta] of Object.entries(result.files || {})) {
-      if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
-        files.push(meta.cid);
-      }
-    }
-    return { ...result, files };
-  }
-  async updateCarLog(cids, fp, compact) {
-    if (compact) {
-      const previousCompactCid = fp.compact[fp.compact.length - 1];
-      fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
-      this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
-      await this.removeCidsForCompact(previousCompactCid[0]);
-    } else {
-      this.carLog.unshift(cids);
-    }
-  }
   async removeCidsForCompact(cid) {
     const carHeader = await this.loadCarHeaderFromMeta({
       cars: [cid]
@@ -1069,9 +1087,9 @@ var Loader = class {
   //     await this.remoteWAL!.enqueue(dbMeta, { public: false })
   //   }
   // }
-  async *entries(cache3 = true) {
+  async *entries(cache2 = true) {
     await this.ready();
-    if (cache3) {
+    if (cache2) {
       for (const [, block] of this.getBlockCache) {
         yield block;
       }
@@ -1153,10 +1171,6 @@ var Loader = class {
     }
     return got;
   }
-  makeCarHeader(meta, cars, compact = false) {
-    const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
-    return { ...coreHeader, meta };
-  }
   async loadCar(cid) {
     if (!this.carStore) {
       throw this.logger.Error().Msg("car store not initialized").AsError();
@@ -1164,94 +1178,516 @@ var Loader = class {
     const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
     return loaded;
   }
+  async makeDecoderAndCarReader(cid, local, remote) {
+    const cidsString = cid.toString();
+    let loadedCar = void 0;
+    let activeStore = local;
+    try {
+      this.logger.Debug().Str("cid", cidsString).Msg("loading car");
+      loadedCar = await local.load(cid);
+      this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
+    } catch (e) {
+      if (remote) {
+        const remoteCar = await remote.load(cid);
+        if (remoteCar) {
+          this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
+          await local.save(remoteCar);
+          loadedCar = remoteCar;
+          activeStore = remote;
+        }
+      } else {
+        this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
+      }
+    }
+    if (!loadedCar) {
+      throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
+    }
+    const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
+    const rawReader = await CarReader.fromBytes(bytes.value);
+    const readerP = Promise.resolve(rawReader);
+    const cachedReaderP = readerP.then(async (reader) => {
+      await this.cacheCarReader(cidsString, reader).catch((e) => {
+        this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
+        return;
+      });
+      return reader;
+    });
+    this.carReaders.set(cidsString, cachedReaderP);
+    return readerP;
+  }
   //What if instead it returns an Array of CarHeader
-  async storesLoadCar(cid, local, remote, publicFiles) {
+  async storesLoadCar(cid, local, remote) {
     const cidsString = cid.toString();
-    if (!this.carReaders.has(cidsString)) {
-      this.carReaders.set(
-        cidsString,
-        (async () => {
-          let loadedCar = void 0;
-          try {
-            this.logger.Debug().Str("cid", cidsString).Msg("loading car");
-            loadedCar = await local.load(cid);
-            this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
-          } catch (e) {
-            if (remote) {
-              const remoteCar = await remote.load(cid);
-              if (remoteCar) {
-                this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
-                await local.save(remoteCar);
-                loadedCar = remoteCar;
-              }
-            } else {
-              this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
-            }
-          }
-          if (!loadedCar) {
-            throw this.logger.Error().Url(local.url).Str("cid", cidsString).Msg("missing car files").AsError();
-          }
-          const rawReader = await CarReader.fromBytes(loadedCar.bytes);
-          const readerP = publicFiles ? Promise.resolve(rawReader) : this.ensureDecryptedReader(rawReader);
-          const cachedReaderP = readerP.then(async (reader) => {
-            await this.cacheCarReader(cidsString, reader).catch(() => {
-              return;
-            });
-            return reader;
-          });
-          this.carReaders.set(cidsString, cachedReaderP);
-          return readerP;
-        })().catch((e) => {
-          this.carReaders.delete(cidsString);
-          throw e;
-        })
-      );
+    let dacr = this.carReaders.get(cidsString);
+    if (!dacr) {
+      dacr = this.makeDecoderAndCarReader(cid, local, remote);
+      this.carReaders.set(cidsString, dacr);
+    }
+    return dacr;
+  }
+  async getMoreReaders(cids) {
+    const limit = pLimit(5);
+    const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
+    await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
+  }
+};
+
+// src/runtime/keyed-crypto.ts
+var keyed_crypto_exports = {};
+__export(keyed_crypto_exports, {
+  BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
+  keyedCryptoFactory: () => keyedCryptoFactory
+});
+import { base58btc as base58btc2 } from "multiformats/bases/base58";
+import { sha256 as hasher4 } from "multiformats/hashes/sha2";
+import * as CBOR from "cborg";
+var generateIV = {
+  random: {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    calc: async (ko, crypto, data) => {
+      return crypto.randomBytes(ko.ivLength);
+    },
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    verify: async (ko, crypto, iv, data) => {
+      return true;
+    }
+  },
+  hash: {
+    calc: async (ko, crypto, data) => {
+      const hash = await hasher4.digest(data);
+      const hashBytes = new Uint8Array(hash.bytes);
+      const hashArray = new Uint8Array(ko.ivLength);
+      for (let i = 0; i < hashBytes.length; i++) {
+        hashArray[i % ko.ivLength] ^= hashBytes[i];
+      }
+      return hashArray;
+    },
+    verify: async function(ko, crypto, iv, data) {
+      return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
+    }
+  }
+};
+function getGenerateIVFn(url, opts) {
+  const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
+  return generateIV[ivhash] || generateIV["hash"];
+}
+var BlockIvKeyIdCodec = class {
+  constructor(ko, iv, opts) {
+    this.code = 3147065;
+    this.name = "Fireproof@encrypted-block:aes-gcm";
+    this.ko = ko;
+    this.iv = iv;
+    this.opts = opts || {};
+  }
+  async encode(data) {
+    const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
+    const { iv } = this.ko.algo(calcIv);
+    const fprt = await this.ko.fingerPrint();
+    const keyId = base58btc2.decode(fprt);
+    this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
+    return CBOR.encode({
+      iv,
+      keyId,
+      data: await this.ko._encrypt({ iv, bytes: data })
+    });
+  }
+  async decode(abytes) {
+    let bytes;
+    if (abytes instanceof Uint8Array) {
+      bytes = abytes;
+    } else {
+      bytes = new Uint8Array(abytes);
+    }
+    const { iv, keyId, data } = CBOR.decode(bytes);
+    const fprt = await this.ko.fingerPrint();
+    this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
+    if (base58btc2.encode(keyId) !== fprt) {
+      throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
+    }
+    const result = await this.ko._decrypt({ iv, bytes: data });
+    if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
+      throw this.ko.logger.Error().Msg("iv missmatch").AsError();
+    }
+    return result;
+  }
+};
+var keyedCrypto = class {
+  constructor(url, key, cyopt, sthis) {
+    this.ivLength = 12;
+    this.isEncrypting = true;
+    this.logger = ensureLogger(sthis, "keyedCrypto");
+    this.crypto = cyopt;
+    this.key = key;
+    this.url = url;
+  }
+  fingerPrint() {
+    return Promise.resolve(this.key.fingerPrint);
+  }
+  codec(iv, opts) {
+    return new BlockIvKeyIdCodec(this, iv, opts);
+  }
+  algo(iv) {
+    return {
+      name: "AES-GCM",
+      iv: iv || this.crypto.randomBytes(this.ivLength),
+      tagLength: 128
+    };
+  }
+  async _decrypt(data) {
+    this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
+    return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
+  }
+  async _encrypt(data) {
+    this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
+    const a = this.algo(data.iv);
+    return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
+  }
+};
+var nullCodec = class {
+  constructor() {
+    this.code = 0;
+    this.name = "Fireproof@unencrypted-block";
+  }
+  encode(data) {
+    return data;
+  }
+  decode(data) {
+    return data;
+  }
+};
+var noCrypto = class {
+  constructor(url, cyrt, sthis) {
+    this.ivLength = 0;
+    this.code = 0;
+    this.name = "Fireproof@unencrypted-block";
+    this.isEncrypting = false;
+    this._fingerPrint = "noCrypto:" + Math.random();
+    this.logger = ensureLogger(sthis, "noCrypto");
+    this.crypto = cyrt;
+    this.url = url;
+  }
+  fingerPrint() {
+    return Promise.resolve(this._fingerPrint);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  codec(iv) {
+    return new nullCodec();
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  algo(iv) {
+    return {
+      name: "noCrypto",
+      iv: new Uint8Array(),
+      tagLength: 0
+    };
+  }
+  _decrypt() {
+    throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
+  }
+  _encrypt() {
+    throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
+  }
+};
+async function keyedCryptoFactory(url, kb, sthis) {
+  const storekey = url.getParam("storekey");
+  if (storekey && storekey !== "insecure") {
+    let rkey = await kb.getNamedKey(storekey, true);
+    if (rkey.isErr()) {
+      try {
+        rkey = await kb.toKeyWithFingerPrint(storekey);
+      } catch (e) {
+        throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
+      }
+    }
+    return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
+  }
+  return new noCrypto(url, kb.rt.crypto, sthis);
+}
+
+// src/blockstore/fragment-gateway.ts
+import { Result as Result3 } from "@adviser/cement";
+import { base58btc as base58btc3 } from "multiformats/bases/base58";
+import { encode as encode3, decode as decode3 } from "cborg";
+function getFragSize(url) {
+  const fragSize = url.getParam("fragSize");
+  let ret = 0;
+  if (fragSize) {
+    ret = parseInt(fragSize);
+  }
+  if (isNaN(ret) || ret <= 0) {
+    ret = 0;
+  }
+  return ret;
+}
+async function getFrags(url, innerGW, headerSize, logger) {
+  const fragSize = getFragSize(url);
+  if (!fragSize) {
+    const res = await innerGW.get(url);
+    if (res.isErr()) {
+      return [res];
+    }
+    const data = res.unwrap();
+    return [
+      Result3.Ok({
+        fid: new Uint8Array(0),
+        ofs: 0,
+        len: data.length,
+        data
+      })
+    ];
+  }
+  const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
+  if (firstRaw.isErr()) {
+    return [firstRaw];
+  }
+  const firstFragment = decode3(firstRaw.unwrap());
+  const blockSize = firstFragment.data.length;
+  const ops = [Promise.resolve(Result3.Ok(firstFragment))];
+  const fidStr = base58btc3.encode(firstFragment.fid);
+  const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
+  for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
+    ops.push(
+      (async (furl, ofs2) => {
+        const raw2 = await innerGW.get(furl);
+        if (raw2.isErr()) {
+          return raw2;
+        }
+        const fragment = decode3(raw2.unwrap());
+        if (base58btc3.encode(fragment.fid) !== fidStr) {
+          return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
+        }
+        if (fragment.ofs !== ofs2) {
+          return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
+        }
+        return Result3.Ok(fragment);
+      })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
+    );
+  }
+  return Promise.all(ops);
+}
+var FragmentGateway = class {
+  constructor(sthis, innerGW) {
+    this.fidLength = 4;
+    this.headerSize = 32;
+    this.sthis = ensureSuperLog(sthis, "FragmentGateway");
+    this.logger = this.sthis.logger;
+    this.innerGW = innerGW;
+  }
+  slicer(url, body) {
+    const fragSize = getFragSize(url);
+    if (!fragSize) {
+      return [this.innerGW.put(url, body)];
+    }
+    const blocksize = fragSize - this.headerSize;
+    if (blocksize <= 0) {
+      throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
+    }
+    const ops = [];
+    const fid = this.sthis.nextId(this.fidLength);
+    const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
+    for (let ofs = 0; ofs < body.length; ofs += blocksize) {
+      const block = encode3({
+        fid: fid.bin,
+        ofs,
+        len: body.length,
+        data: body.slice(ofs, ofs + blocksize)
+      });
+      if (block.length > fragSize) {
+        throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
+      }
+      ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
+    }
+    return ops;
+  }
+  buildUrl(baseUrl, key) {
+    return this.innerGW.buildUrl(baseUrl, key);
+  }
+  async destroy(iurl) {
+    return this.innerGW.destroy(iurl);
+  }
+  async start(url) {
+    this.headerSize = encode3({
+      fid: this.sthis.nextId(this.fidLength).bin,
+      ofs: 1024 * 1024,
+      // 32bit
+      len: 16 * 1024 * 1024,
+      // 32bit
+      data: new Uint8Array(1024)
+    }).length - 1024;
+    return this.innerGW.start(url);
+  }
+  async close(url) {
+    return this.innerGW.close(url);
+  }
+  async put(url, body) {
+    await Promise.all(this.slicer(url, body));
+    return Result3.Ok(void 0);
+  }
+  async get(url) {
+    const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
+    let buffer = void 0;
+    for (const rfrag of rfrags) {
+      if (rfrag.isErr()) {
+        return Result3.Err(rfrag.Err());
+      }
+      const frag = rfrag.Ok();
+      buffer = buffer || new Uint8Array(frag.len);
+      buffer.set(frag.data, frag.ofs);
+    }
+    return Result3.Ok(buffer || new Uint8Array(0));
+  }
+  async subscribe(url, callback) {
+    if (this.innerGW.subscribe) {
+      return this.innerGW.subscribe(url, callback);
+    } else {
+      return Result3.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
+    }
+  }
+  async delete(url) {
+    const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
+    for (const rfrag of rfrags) {
+      if (rfrag.isErr()) {
+        return Result3.Err(rfrag.Err());
+      }
+      const frag = rfrag.Ok();
+      const fidStr = base58btc3.encode(frag.fid);
+      const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
+      await this.innerGW.delete(fragUrl);
+    }
+    return Result3.Ok(void 0);
+  }
+};
+
+// src/blockstore/meta-key-helper.ts
+import { format, parse } from "@ipld/dag-json";
+import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
+import { CID as CID2 } from "multiformats";
+import { base64pad } from "multiformats/bases/base64";
+import { Result as Result4 } from "@adviser/cement";
+async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
+  const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
+  if (!crdtEntries.length) {
+    sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
+    return [];
+  }
+  if (!crdtEntries.map) {
+    sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
+    return [];
+  }
+  return Promise.all(
+    crdtEntries.map(async (crdtEntry) => {
+      const eventBlock = await decodeEventBlock(base64pad.decode(crdtEntry.data));
+      const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta));
+      return {
+        eventCid: eventBlock.cid,
+        parents: crdtEntry.parents,
+        dbMeta
+      };
+    })
+  );
+}
+async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
+  try {
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
+    const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
+    if (keyInfo.length) {
+      const dbMeta = keyInfo[0].dbMeta;
+      if (dbMeta.key) {
+        const kb = await getKeyBag(sthis);
+        const keyName = getStoreKeyName(uri);
+        const res = await kb.setNamedKey(keyName, dbMeta.key);
+        if (res.isErr()) {
+          sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
+          throw res.Err();
+        }
+      }
+      sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
+      return Result4.Ok(dbMeta);
     }
-    return this.carReaders.get(cidsString);
+    sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
+    return Result4.Ok(void 0);
+  } catch (error) {
+    sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
+    return Result4.Err(error);
   }
-  async ensureDecryptedReader(reader) {
-    const theKey = await this._getKey();
-    if (this.ebOpts.public || !(theKey && this.ebOpts.crypto)) {
-      return reader;
+}
+async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
+  try {
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
+    const keyName = getStoreKeyName(uri);
+    const kb = await getKeyBag(sthis);
+    const res = await kb.getNamedExtractableKey(keyName, true);
+    if (res.isErr()) {
+      sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
+      throw res.Err();
     }
-    const { blocks, root: root3 } = await decodeEncryptedCar(this.logger, this.ebOpts.crypto, theKey, reader);
+    const keyData = await res.Ok().extract();
+    const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
+    const { dbMeta, parents } = dbMetas[0];
+    const parentLinks = parents.map((p) => CID2.parse(p));
+    dbMeta.key = keyData.keyStr;
+    const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
+    const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
+    return Result4.Ok(encoded);
+  } catch (error) {
+    sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
+    return Result4.Err(error);
+  }
+}
+function getStoreKeyName(url) {
+  const storeKeyName = [url.getParam("localName") || url.getParam("name")];
+  const idx = url.getParam("index");
+  if (idx) {
+    storeKeyName.push(idx);
+  }
+  storeKeyName.push("data");
+  return `@${storeKeyName.join(":")}@`;
+}
+async function createDbMetaEventBlock(sthis, dbMeta, parents) {
+  const event = await EventBlock.create(
+    {
+      dbMeta: sthis.txt.encode(format(dbMeta))
+    },
+    parents
+  );
+  return event;
+}
+async function encodeEventsWithParents(sthis, events, parents) {
+  const crdtEntries = events.map((event) => {
+    const base64String = base64pad.encode(event.bytes);
     return {
-      getRoots: () => [root3],
-      get: blocks.get.bind(blocks),
-      blocks: blocks.entries.bind(blocks)
+      cid: event.cid.toString(),
+      data: base64String,
+      parents: parents.map((p) => p.toString())
     };
-  }
-  async setKey(key) {
-    if (this.key && this.key !== key)
-      throw this.logger.Error().Str("this.key", this.key).Str("key", key).Msg("setting key").AsError();
-    this.key = key;
-    const encoder = new TextEncoder();
-    const data = encoder.encode(key);
-    const hashBuffer = await this.ebOpts.crypto.digestSHA256(data);
-    const hashArray = Array.from(new Uint8Array(hashBuffer));
-    this.keyId = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
-  }
-  async getMoreReaders(cids) {
-    const limit = pLimit(5);
-    const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
-    await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
-  }
-};
+  });
+  return sthis.txt.encode(JSON.stringify(crdtEntries));
+}
 
 // src/blockstore/store.ts
+import pRetry from "p-retry";
+import pMap from "p-map";
 function guardVersion(url) {
-  if (!url.searchParams.has("version")) {
-    return Result2.Err(`missing version: ${url.toString()}`);
+  if (!url.hasParam("version")) {
+    return Result5.Err(`missing version: ${url.toString()}`);
   }
-  return Result2.Ok(url);
+  return Result5.Ok(url);
 }
-var VersionedStore = class {
-  constructor(name, url, logger) {
+var BaseStoreImpl = class {
+  constructor(name, url, opts, sthis, logger) {
     this._onStarted = [];
     this._onClosed = [];
     this.name = name;
-    this.url = url;
-    this.logger = logger;
+    this._url = url;
+    this.keybag = opts.keybag;
+    this.sthis = sthis;
+    this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
+    this.gateway = new FragmentGateway(this.sthis, opts.gateway);
+    this.loader = opts.loader;
+  }
+  url() {
+    return this._url;
   }
   onStarted(fn) {
     this._onStarted.push(fn);
@@ -1259,38 +1695,93 @@ var VersionedStore = class {
   onClosed(fn) {
     this._onClosed.push(fn);
   }
-};
-var textEncoder = new TextEncoder();
-var textDecoder = new TextDecoder();
-var MetaStore = class extends VersionedStore {
-  constructor(name, url, logger, gateway) {
-    super(name, url, ensureLogger(logger, "MetaStore", {}));
-    this.tag = "header-base";
-    this.gateway = gateway;
-  }
-  makeHeader({ cars, key }) {
-    const toEncode = { cars };
-    if (key) toEncode.key = key;
-    return format(toEncode);
-  }
-  parseHeader(headerData) {
-    const got = parse(headerData);
-    return got;
+  async ready() {
+    return;
+  }
+  async keyedCrypto() {
+    return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
   }
   async start() {
-    this.logger.Debug().Msg("starting");
-    const res = await this.gateway.start(this.url);
+    this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
+    this._url = this._url.build().setParam("store", this.storeType).URI();
+    const res = await this.gateway.start(this._url);
     if (res.isErr()) {
+      this.logger.Error().Result("gw-start", res).Msg("started-gateway");
       return res;
     }
+    this._url = res.Ok();
+    const kb = await this.keybag();
+    const skRes = await kb.ensureKeyFromUrl(this._url, () => {
+      const idx = this._url.getParam("index");
+      const storeKeyName = [this.name];
+      if (idx) {
+        storeKeyName.push(idx);
+      }
+      storeKeyName.push(this.storeType);
+      return storeKeyName.join(":");
+    });
+    if (skRes.isErr()) {
+      return skRes;
+    }
+    this._url = skRes.Ok();
+    const version = guardVersion(this._url);
+    if (version.isErr()) {
+      this.logger.Error().Result("version", version).Msg("guardVersion");
+      await this.close();
+      return version;
+    }
+    if (this.ready) {
+      const fn = this.ready.bind(this);
+      const ready = await exception2Result(fn);
+      if (ready.isErr()) {
+        await this.close();
+        return ready;
+      }
+    }
     this._onStarted.forEach((fn) => fn());
-    return guardVersion(this.url);
+    this.logger.Debug().Msg("started");
+    return version;
+  }
+};
+var MetaStoreImpl = class extends BaseStoreImpl {
+  // remote: boolean;
+  constructor(sthis, name, url, opts) {
+    super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
+    this.storeType = "meta";
+    this.subscribers = /* @__PURE__ */ new Map();
+    this.parents = [];
+    if (
+      /*this.remote && */
+      opts.gateway.subscribe
+    ) {
+      this.onStarted(async () => {
+        this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
+        opts.gateway.subscribe?.(this.url(), async (message) => {
+          this.logger.Debug().Msg("Received message from gateway");
+          const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
+          await Promise.all(
+            dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
+          );
+          this.updateParentsFromDbMetas(dbMetas);
+        });
+      });
+    }
+  }
+  updateParentsFromDbMetas(dbMetas) {
+    const cids = dbMetas.map((m) => m.eventCid);
+    const dbMetaParents = dbMetas.flatMap((m) => m.parents);
+    const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
+    const dbMetaParentsSet = new Set(dbMetaParents.map((p) => p.toString()));
+    this.parents = Array.from(uniqueParentsMap.values()).filter((p) => !dbMetaParentsSet.has(p.toString()));
   }
-  async load(branch) {
-    this.logger.Debug().Str("branch", branch || "").Msg("loading");
-    const url = await this.gateway.buildUrl(this.url, branch || "main");
+  async handleByteHeads(byteHeads) {
+    return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
+  }
+  async load() {
+    const branch = "main";
+    const url = await this.gateway.buildUrl(this.url(), branch);
     if (url.isErr()) {
-      throw this.logger.Error().Result("buidUrl", url).Str("branch", branch || "").Url(this.url).Msg("got error from gateway.buildUrl").AsError();
+      throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
     }
     const bytes = await this.gateway.get(url.Ok());
     if (bytes.isErr()) {
@@ -1299,66 +1790,45 @@ var MetaStore = class extends VersionedStore {
       }
       throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
     }
-    try {
-      return [this.parseHeader(textDecoder.decode(bytes.Ok()))];
-    } catch (e) {
-      throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
-    }
+    const dbMetas = await this.handleByteHeads(bytes.Ok());
+    await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
+    this.updateParentsFromDbMetas(dbMetas);
+    return dbMetas.map((m) => m.dbMeta);
   }
-  async save(meta, branch = "main") {
+  async save(meta, branch) {
+    branch = branch || "main";
     this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
-    const bytes = this.makeHeader(meta);
-    const url = await this.gateway.buildUrl(this.url, branch);
+    const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
+    const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
+    const url = await this.gateway.buildUrl(this.url(), branch);
     if (url.isErr()) {
-      throw this.logger.Error().Err(url.Err()).Str("branch", branch).Url(this.url).Msg("got error from gateway.buildUrl").AsError();
+      throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
     }
-    const res = await this.gateway.put(url.Ok(), textEncoder.encode(bytes));
+    this.parents = [event.cid];
+    const res = await this.gateway.put(url.Ok(), bytes);
     if (res.isErr()) {
       throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
     }
-    return res.Ok();
+    return res;
   }
   async close() {
-    await this.gateway.close(this.url);
+    await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return Result2.Ok(void 0);
+    return Result5.Ok(void 0);
   }
   async destroy() {
-    return this.gateway.destroy(this.url);
+    return this.gateway.destroy(this.url());
   }
 };
-var DataStore = class extends VersionedStore {
-  constructor(name, url, logger, gateway) {
-    super(
-      name,
-      url,
-      ensureLogger(logger, "DataStore", {
-        url: () => url.toString()
-      })
-    );
-    this.tag = "car-base";
-    this.gateway = gateway;
-  }
-  async start() {
-    this.logger.Debug().Msg("starting-gateway");
-    const res = await this.gateway.start(this.url);
-    if (res.isErr()) {
-      this.logger.Error().Result("gw-start", res).Msg("started-gateway");
-      return res;
-    }
-    this._onStarted.forEach((fn) => fn());
-    const version = guardVersion(this.url);
-    if (version.isErr()) {
-      this.logger.Error().Result("version", version).Msg("guardVersion");
-      await this.close();
-      return version;
-    }
-    this.logger.Debug().Msg("started");
-    return version;
+var DataStoreImpl = class extends BaseStoreImpl {
+  // readonly tag: string = "car-base";
+  constructor(sthis, name, url, opts) {
+    super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
+    this.storeType = "data";
   }
   async load(cid) {
     this.logger.Debug().Any("cid", cid).Msg("loading");
-    const url = await this.gateway.buildUrl(this.url, cid.toString());
+    const url = await this.gateway.buildUrl(this.url(), cid.toString());
     if (url.isErr()) {
       throw this.logger.Error().Err(url.Err()).Str("cid", cid.toString()).Msg("got error from gateway.buildUrl").AsError();
     }
@@ -1371,7 +1841,7 @@ var DataStore = class extends VersionedStore {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   async save(car, opts) {
     this.logger.Debug().Any("cid", car.cid.toString()).Msg("saving");
-    const url = await this.gateway.buildUrl(this.url, car.cid.toString());
+    const url = await this.gateway.buildUrl(this.url(), car.cid.toString());
     if (url.isErr()) {
       throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError();
     }
@@ -1382,31 +1852,30 @@ var DataStore = class extends VersionedStore {
     return res.Ok();
   }
   async remove(cid) {
-    const url = await this.gateway.buildUrl(this.url, cid.toString());
+    const url = await this.gateway.buildUrl(this.url(), cid.toString());
     if (url.isErr()) {
       return url;
     }
     return this.gateway.delete(url.Ok());
   }
   async close() {
-    await this.gateway.close(this.url);
+    await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return Result2.Ok(void 0);
+    return Result5.Ok(void 0);
   }
   destroy() {
-    return this.gateway.destroy(this.url);
+    return this.gateway.destroy(this.url());
   }
 };
-var RemoteWAL = class extends VersionedStore {
-  constructor(loader, url, logger, gateway) {
-    super(loader.name, url, ensureLogger(logger, "RemoteWAL"));
-    this.tag = "rwal-base";
-    this._ready = new ResolveOnce2();
+var WALStoreImpl = class extends BaseStoreImpl {
+  constructor(loader, url, opts) {
+    super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
+    this.storeType = "wal";
+    this._ready = new ResolveOnce3();
     this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
     this.processing = void 0;
     this.processQueue = new CommitQueue();
     this.loader = loader;
-    this.gateway = gateway;
   }
   async ready() {
     return this._ready.once(async () => {
@@ -1425,124 +1894,121 @@ var RemoteWAL = class extends VersionedStore {
   }
   async enqueue(dbMeta, opts) {
     await this.ready();
-    if (opts.noLoader) {
+    if (opts.compact) {
+      this.walState.operations = [];
+      this.walState.noLoaderOps = [dbMeta];
+    } else if (opts.noLoader) {
       this.walState.noLoaderOps.push(dbMeta);
     } else {
       this.walState.operations.push(dbMeta);
     }
     await this.save(this.walState);
-    void this._process();
+    if (!opts.noLoader) {
+      void this.process();
+    }
   }
   async enqueueFile(fileCid, publicFile = false) {
     await this.ready();
     this.walState.fileOperations.push({ cid: fileCid, public: publicFile });
   }
-  async _process() {
+  async process() {
     await this.ready();
     if (!this.loader.remoteCarStore) return;
     await this.processQueue.enqueue(async () => {
-      await this._doProcess();
+      try {
+        await this._doProcess();
+      } catch (e) {
+        this.logger.Error().Any("error", e).Msg("error processing wal");
+      }
       if (this.walState.operations.length || this.walState.fileOperations.length || this.walState.noLoaderOps.length) {
-        setTimeout(() => void this._process(), 0);
+        setTimeout(() => void this.process(), 0);
       }
     });
   }
   async _doProcess() {
     if (!this.loader.remoteCarStore) return;
-    const rmlp = (async () => {
-      const operations = [...this.walState.operations];
-      const fileOperations = [...this.walState.fileOperations];
-      const uploads = [];
-      const noLoaderOps = [...this.walState.noLoaderOps];
-      const limit = pLimit2(5);
-      if (operations.length + fileOperations.length + noLoaderOps.length === 0) return;
-      for (const dbMeta of noLoaderOps) {
-        const uploadP = limit(async () => {
-          for (const cid of dbMeta.cars) {
-            const car = await (await this.loader.carStore()).load(cid);
-            if (!car) {
-              if (carLogIncludesGroup2(this.loader.carLog, dbMeta.cars))
-                throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
-            } else {
-              await throwFalsy(this.loader.remoteCarStore).save(car);
+    const operations = [...this.walState.operations];
+    const noLoaderOps = [...this.walState.noLoaderOps];
+    const fileOperations = [...this.walState.fileOperations];
+    if (operations.length + noLoaderOps.length + fileOperations.length === 0) return;
+    const concurrencyLimit = 3;
+    const retryableUpload = (fn, description) => pRetry(fn, {
+      retries: 5,
+      onFailedAttempt: (error) => {
+        this.logger.Warn().Msg(`Attempt ${error.attemptNumber} failed for ${description}. There are ${error.retriesLeft} retries left.`);
+      }
+    });
+    try {
+      await pMap(
+        noLoaderOps,
+        async (dbMeta) => {
+          await retryableUpload(async () => {
+            for (const cid of dbMeta.cars) {
+              const car = await (await this.loader.carStore()).load(cid);
+              if (!car) {
+                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                  throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
+                }
+              } else {
+                await throwFalsy(this.loader.remoteCarStore).save(car);
+              }
             }
             this.walState.noLoaderOps = this.walState.noLoaderOps.filter((op) => op !== dbMeta);
-          }
-        });
-        uploads.push(uploadP);
-      }
-      for (const dbMeta of operations) {
-        const uploadP = limit(async () => {
-          for (const cid of dbMeta.cars) {
-            const car = await (await this.loader.carStore()).load(cid).catch(() => null);
-            if (!car) {
-              if (carLogIncludesGroup2(this.loader.carLog, dbMeta.cars))
-                throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
-            } else {
-              await throwFalsy(this.loader.remoteCarStore).save(car);
+          }, `noLoaderOp with dbMeta.cars=${dbMeta.cars.toString()}`);
+        },
+        { concurrency: concurrencyLimit }
+      );
+      await pMap(
+        operations,
+        async (dbMeta) => {
+          await retryableUpload(async () => {
+            for (const cid of dbMeta.cars) {
+              const car = await (await this.loader.carStore()).load(cid);
+              if (!car) {
+                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                  throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
+                }
+              } else {
+                await throwFalsy(this.loader.remoteCarStore).save(car);
+              }
             }
-          }
-          this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
-        });
-        uploads.push(uploadP);
-      }
-      if (fileOperations.length) {
-        const dbLoader = this.loader;
-        for (const { cid: fileCid, public: publicFile } of fileOperations) {
-          const uploadP = limit(async () => {
-            const fileBlock = await (await dbLoader.fileStore()).load(fileCid);
-            await dbLoader.remoteFileStore?.save(fileBlock, { public: publicFile });
+            this.walState.operations = this.walState.operations.filter((op) => op !== dbMeta);
+          }, `operation with dbMeta.cars=${dbMeta.cars.toString()}`);
+        },
+        { concurrency: concurrencyLimit }
+      );
+      await pMap(
+        fileOperations,
+        async ({ cid: fileCid, public: publicFile }) => {
+          await retryableUpload(async () => {
+            const fileBlock = await (await this.loader.fileStore()).load(fileCid);
+            if (!fileBlock) {
+              throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
+            }
+            await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
             this.walState.fileOperations = this.walState.fileOperations.filter((op) => op.cid !== fileCid);
-          });
-          uploads.push(uploadP);
-        }
-      }
-      try {
-        const res = await Promise.allSettled(uploads);
-        const errors = res.filter((r) => r.status === "rejected");
-        if (errors.length) {
-          throw this.logger.Error().Any(
-            "errors",
-            errors.map((e) => e.reason)
-          ).Msg("error uploading").AsError();
-          errors[0].reason;
-        }
-        if (operations.length) {
-          const lastOp = operations[operations.length - 1];
-          await this.loader.remoteMetaStore?.save(lastOp).catch((e) => {
-            this.walState.operations.push(lastOp);
-            throw this.logger.Error().Any("error", e).Msg("error saving remote meta").AsError();
-          });
-        }
-      } finally {
-        await this.save(this.walState);
+          }, `fileOperation with cid=${fileCid.toString()}`);
+        },
+        { concurrency: concurrencyLimit }
+      );
+      if (operations.length) {
+        const lastOp = operations[operations.length - 1];
+        await retryableUpload(async () => {
+          await this.loader.remoteMetaStore?.save(lastOp);
+        }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
       }
-    })();
-    await rmlp;
-  }
-  async start() {
-    const res = await this.gateway.start(this.url);
-    if (res.isErr()) {
-      return res;
-    }
-    const ver = guardVersion(this.url);
-    if (ver.isErr()) {
-      await this.close();
-      return ver;
-    }
-    const ready = await exception2Result(() => this.ready());
-    this._onStarted.forEach((fn) => fn());
-    if (ready.isErr()) {
-      await this.close();
-      return ready;
+    } catch (error) {
+      this.logger.Error().Any("error", error).Msg("Processing failed");
+      return;
+    } finally {
+      await this.save(this.walState);
     }
-    return ready;
   }
   async load() {
     this.logger.Debug().Msg("loading");
-    const filepath = await this.gateway.buildUrl(this.url, "main");
+    const filepath = await this.gateway.buildUrl(this.url(), "main");
     if (filepath.isErr()) {
-      throw this.logger.Error().Err(filepath.Err()).Str("url", this.url.toString()).Msg("error building url").AsError();
+      throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
     }
     const bytes = await this.gateway.get(filepath.Ok());
     if (bytes.isErr()) {
@@ -1552,71 +2018,83 @@ var RemoteWAL = class extends VersionedStore {
       throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
     }
     try {
-      return bytes && parse(textDecoder.decode(bytes.Ok()));
+      return bytes && parse2(this.sthis.txt.decode(bytes.Ok()));
     } catch (e) {
       throw this.logger.Error().Err(e).Msg("error parse").AsError();
     }
   }
   async save(state) {
-    const filepath = await this.gateway.buildUrl(this.url, "main");
+    const filepath = await this.gateway.buildUrl(this.url(), "main");
     if (filepath.isErr()) {
-      throw this.logger.Error().Err(filepath.Err()).Str("url", this.url.toString()).Msg("error building url").AsError();
+      throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
     }
     let encoded;
     try {
-      encoded = format(state);
+      encoded = format2(state);
     } catch (e) {
       throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
     }
-    const res = await this.gateway.put(filepath.Ok(), textEncoder.encode(encoded));
+    const res = await this.gateway.put(filepath.Ok(), this.sthis.txt.encode(encoded));
     if (res.isErr()) {
       throw this.logger.Error().Err(res.Err()).Str("filePath", filepath.Ok().toString()).Msg("error saving").AsError();
     }
   }
   async close() {
-    await this.gateway.close(this.url);
+    await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return Result2.Ok(void 0);
+    return Result5.Ok(void 0);
   }
   destroy() {
-    return this.gateway.destroy(this.url);
+    return this.gateway.destroy(this.url());
   }
 };
 
 // src/blockstore/store-factory.ts
 function ensureIsIndex(url, isIndex) {
   if (isIndex) {
-    url.searchParams.set("index", isIndex);
-    return url;
-  } else {
-    url.searchParams.delete("index");
-    return url;
-  }
-}
-function toURL(pathOrUrl, isIndex) {
-  if (pathOrUrl instanceof URL) return ensureIsIndex(pathOrUrl, isIndex);
-  try {
-    const url = new URL(pathOrUrl);
-    return ensureIsIndex(url, isIndex);
-  } catch (e) {
-    const url = new URL(`file://${pathOrUrl}`);
-    return ensureIsIndex(url, isIndex);
+    return url.build().setParam("index", isIndex).URI();
   }
+  return url.build().delParam("index").URI();
 }
-var storeFactory = /* @__PURE__ */ new Map();
 function ensureName(name, url) {
-  if (!url.searchParams.has("name")) {
-    url.searchParams.set("name", name);
+  if (!url.hasParam("name")) {
+    return url.build().setParam("name", name).URI();
   }
+  return url;
 }
+var storeFactory = /* @__PURE__ */ new Map();
 function buildURL(optURL, loader) {
   const storeOpts = loader.ebOpts.store;
   const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
   let obuUrl;
   if (obuItem && obuItem.overrideBaseURL) {
-    obuUrl = new URL(obuItem.overrideBaseURL);
+    obuUrl = URI5.from(obuItem.overrideBaseURL);
   }
-  return toURL(optURL || obuUrl || dataDir(loader.name, storeOpts.stores?.base), storeOpts.isIndex);
+  const ret = ensureIsIndex(
+    URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
+    storeOpts.isIndex
+  );
+  return ret;
+}
+var onceGateway = new KeyedResolvOnce2();
+async function getGatewayFromURL(url, sthis) {
+  return onceGateway.get(url.toString()).once(async () => {
+    const item = storeFactory.get(url.protocol);
+    if (item) {
+      const ret = {
+        gateway: await item.gateway(sthis),
+        test: await item.test(sthis)
+      };
+      const res = await ret.gateway.start(url);
+      if (res.isErr()) {
+        sthis.logger.Error().Result("start", res).Msg("start failed");
+        return void 0;
+      }
+      return ret;
+    }
+    sthis.logger.Warn().Url(url).Msg("unsupported protocol");
+    return void 0;
+  });
 }
 function registerStoreProtocol(item) {
   let protocol = item.protocol;
@@ -1625,8 +2103,7 @@ function registerStoreProtocol(item) {
   }
   if (storeFactory.has(protocol)) {
     if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
-      const logger = ensureLogger({}, "registerStoreProtocol", { protocol });
-      logger.Warn().Msg(`protocol ${protocol} already registered`);
+      throw new Error(`we need a logger here`);
       return () => {
       };
     }
@@ -1641,106 +2118,92 @@ function registerStoreProtocol(item) {
     storeFactory.delete(protocol);
   };
 }
-function runStoreFactory(url, logger, run) {
-  const item = storeFactory.get(url.protocol);
-  if (!item) {
-    throw logger.Error().Url(url).Str("protocol", url.protocol).Any("keys", Array(storeFactory.keys())).Msg(`unsupported protocol`).AsError();
-  }
-  logger.Debug().Str("protocol", url.protocol).Msg("run");
-  return run(item);
-}
-var onceLoadDataGateway = new KeyedResolvOnce();
-function loadDataGateway(url, logger) {
-  return onceLoadDataGateway.get(url.protocol).once(async () => {
-    return runStoreFactory(url, logger, async (item) => item.data(logger));
-  });
-}
-var onceDataStoreFactory = new KeyedResolvOnce();
+var onceDataStoreFactory = new KeyedResolvOnce2();
 async function dataStoreFactory(loader) {
-  const url = buildURL(loader.ebOpts.store.stores?.data, loader);
-  ensureName(loader.name, url);
-  const logger = ensureLogger(loader.logger, "dataStoreFactory", { url: url.toString() });
-  url.searchParams.set("store", "data");
+  const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
+  const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
   return onceDataStoreFactory.get(url.toString()).once(async () => {
-    const gateway = await loadDataGateway(url, logger);
-    const store = new DataStore(loader.name, url, loader.logger, gateway);
-    const ret = await store.start();
-    if (ret.isErr()) {
-      throw logger.Error().Result("start", ret).Msg("start failed").AsError();
-    }
-    logger.Debug().Str("prepared", store.url.toString()).Msg("produced");
+    const gateway = await getGatewayFromURL(url, sthis);
+    if (!gateway) {
+      throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+    }
+    const store = new DataStoreImpl(sthis, loader.name, url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, {
+        ...loader.ebOpts.keyBag
+      })
+    });
     return store;
   });
 }
-var onceLoadMetaGateway = new KeyedResolvOnce();
-function loadMetaGateway(url, logger) {
-  return onceLoadMetaGateway.get(url.protocol).once(async () => {
-    return runStoreFactory(url, logger, async (item) => item.meta(logger));
-  });
-}
-var onceMetaStoreFactory = new KeyedResolvOnce();
+var onceMetaStoreFactory = new KeyedResolvOnce2();
 async function metaStoreFactory(loader) {
-  const url = buildURL(loader.ebOpts.store.stores?.meta, loader);
-  ensureName(loader.name, url);
-  const logger = ensureLogger(loader.logger, "metaStoreFactory", { url: () => url.toString() });
-  url.searchParams.set("store", "meta");
+  const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
+  const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
   return onceMetaStoreFactory.get(url.toString()).once(async () => {
-    logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
-    const gateway = await loadMetaGateway(url, logger);
-    const store = new MetaStore(loader.name, url, loader.logger, gateway);
-    const ret = await store.start();
-    if (ret.isErr()) {
-      throw logger.Error().Result("start", ret).Msg("start failed").AsError();
-    }
+    sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
+    const gateway = await getGatewayFromURL(url, sthis);
+    if (!gateway) {
+      throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+    }
+    const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, {
+        ...loader.ebOpts.keyBag
+      })
+    });
     return store;
   });
 }
-var onceWalGateway = new KeyedResolvOnce();
-function loadWalGateway(url, logger) {
-  return onceWalGateway.get(url.protocol).once(async () => {
-    return runStoreFactory(url, logger, async (item) => item.wal(logger));
-  });
-}
-var onceRemoteWalFactory = new KeyedResolvOnce();
+var onceRemoteWalFactory = new KeyedResolvOnce2();
 async function remoteWalFactory(loader) {
-  const url = buildURL(loader.ebOpts.store.stores?.meta, loader);
-  ensureName(loader.name, url);
-  const logger = ensureLogger(loader.logger, "remoteWalFactory", { url: url.toString() });
-  url.searchParams.set("store", "wal");
+  const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
+  const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
   return onceRemoteWalFactory.get(url.toString()).once(async () => {
-    const gateway = await loadWalGateway(url, logger);
-    logger.Debug().Str("prepared", url.toString()).Msg("produced");
-    const store = new RemoteWAL(loader, url, loader.logger, gateway);
-    const ret = await store.start();
-    if (ret.isErr()) {
-      throw logger.Error().Result("start", ret).Msg("start failed").AsError();
-    }
+    const gateway = await getGatewayFromURL(url, sthis);
+    if (!gateway) {
+      throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+    }
+    sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
+    const store = new WALStoreImpl(loader, url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, {
+        ...loader.ebOpts.keyBag
+      })
+    });
     return store;
   });
 }
-async function testStoreFactory(url, ilogger) {
-  const logger = ensureLogger(
-    {
-      logger: ilogger
-    },
-    "testStoreFactory"
-  );
-  return runStoreFactory(url, logger, async (item) => item.test(logger));
+async function testStoreFactory(url, sthis) {
+  sthis = ensureSuperLog(sthis, "testStoreFactory");
+  const gateway = await getGatewayFromURL(url, sthis);
+  if (!gateway) {
+    throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+  }
+  return gateway.test;
+}
+async function ensureStart(store, logger) {
+  const ret = await store.start();
+  if (ret.isErr()) {
+    throw logger.Error().Result("start", ret).Msg("start failed").AsError();
+  }
+  logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
+  return store;
 }
-function toStoreRuntime(opts, ilogger) {
-  const logger = ensureLogger(ilogger, "toStoreRuntime", {});
+function toStoreRuntime(opts, sthis) {
+  const logger = ensureLogger(sthis, "toStoreRuntime", {});
   return {
-    makeMetaStore: (loader) => {
+    makeMetaStore: async (loader) => {
       logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
-      return (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader);
+      return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
     },
-    makeDataStore: (loader) => {
+    makeDataStore: async (loader) => {
       logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
-      return (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader);
+      return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
     },
-    makeRemoteWAL: (loader) => {
-      logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeRemoteWAL).Msg("makeRemoteWAL");
-      return (loader.ebOpts.store.makeRemoteWAL || remoteWalFactory)(loader);
+    makeWALStore: async (loader) => {
+      logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
+      return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
     },
     encodeFile: opts.encodeFile || encodeFile,
     decodeFile: opts.decodeFile || decodeFile
@@ -1748,43 +2211,131 @@ function toStoreRuntime(opts, ilogger) {
 }
 registerStoreProtocol({
   protocol: "file:",
-  data: async (logger) => {
-    const { FileDataGateway } = await import("./store-file-VJ6BI4II.js");
-    return new FileDataGateway(logger);
+  gateway: async (sthis) => {
+    const { FileGateway } = await import("./gateway-5FCWPX5W.js");
+    return new FileGateway(sthis);
   },
-  meta: async (logger) => {
-    const { FileMetaGateway } = await import("./store-file-VJ6BI4II.js");
-    return new FileMetaGateway(logger);
-  },
-  wal: async (logger) => {
-    const { FileWALGateway } = await import("./store-file-VJ6BI4II.js");
-    return new FileWALGateway(logger);
-  },
-  test: async (logger) => {
-    const { FileTestStore } = await import("./store-file-VJ6BI4II.js");
-    return new FileTestStore(logger);
+  test: async (sthis) => {
+    const { FileTestStore } = await import("./gateway-5FCWPX5W.js");
+    return new FileTestStore(sthis);
   }
 });
 registerStoreProtocol({
   protocol: "indexdb:",
-  data: async (logger) => {
-    const { IndexDBDataGateway } = await import("./store-indexdb-WLRSICCB.js");
-    return new IndexDBDataGateway(logger);
-  },
-  meta: async (logger) => {
-    const { IndexDBMetaGateway } = await import("./store-indexdb-WLRSICCB.js");
-    return new IndexDBMetaGateway(logger);
-  },
-  wal: async (logger) => {
-    const { IndexDBMetaGateway } = await import("./store-indexdb-WLRSICCB.js");
-    return new IndexDBMetaGateway(logger);
+  gateway: async (sthis) => {
+    const { IndexDBGateway } = await import("./gateway-H7UD6TNB.js");
+    return new IndexDBGateway(sthis);
   },
-  test: async (logger) => {
-    const { IndexDBTestStore } = await import("./store-indexdb-WLRSICCB.js");
-    return new IndexDBTestStore(logger);
+  test: async (sthis) => {
+    const { IndexDBTestStore } = await import("./gateway-H7UD6TNB.js");
+    return new IndexDBTestStore(sthis);
   }
 });
 
+// src/blockstore/store-remote.ts
+async function RemoteDataStore(sthis, name, url, opts) {
+  const ds = new DataStoreImpl(sthis, name, url, opts);
+  await ds.start();
+  return ds;
+}
+async function RemoteMetaStore(sthis, name, url, opts) {
+  const ms = new MetaStoreImpl(
+    sthis,
+    name,
+    url,
+    opts
+    /* , true*/
+  );
+  await ms.start();
+  return ms;
+}
+
+// src/blockstore/connection-base.ts
+var ConnectionBase = class {
+  constructor(url, logger) {
+    this.loaded = Promise.resolve();
+    this.logger = logger;
+    this.url = url;
+  }
+  async refresh() {
+    await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
+    await (await throwFalsy(this.loader).WALStore()).process();
+  }
+  async connect_X({ loader }) {
+    if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
+    await this.connectMeta_X({ loader });
+    await this.connectStorage_X({ loader });
+  }
+  async connectMeta_X({ loader }) {
+    if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
+    this.loader = loader;
+    await this.onConnect();
+    const metaUrl = this.url.build().defParam("store", "meta").URI();
+    const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
+    if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
+    const dbName = metaUrl.getParam("name");
+    if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
+    const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
+      loader
+    });
+    this.loader.remoteMetaStore = remote;
+    this.loaded = this.loader.ready().then(async () => {
+      remote.load().then(async () => {
+        (await throwFalsy(this.loader).WALStore()).process();
+      });
+    });
+  }
+  async connectStorage_X({ loader }) {
+    if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
+    this.loader = loader;
+    const dataUrl = this.url.build().defParam("store", "data").URI();
+    const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
+    if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
+    const name = dataUrl.getParam("name");
+    if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
+    loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
+    });
+    loader.remoteFileStore = loader.remoteCarStore;
+  }
+  // move this stuff to connect
+  // async getDashboardURL(compact = true) {
+  //   const baseUrl = 'https://dashboard.fireproof.storage/'
+  //   if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
+  //   // if (compact) {
+  //   //   await this.compact()
+  //   // }
+  //   const currents = await this.loader?.metaStore?.load()
+  //   if (!currents) throw new Error("Can't sync empty database: save data first")
+  //   if (currents.length > 1)
+  //     throw new Error("Can't sync database with split heads: make an update first")
+  //   const current = currents[0]
+  //   const params = {
+  //     car: current.car.toString()
+  //   }
+  //   if (current.key) {
+  //     // @ts-ignore
+  //     params.key = current.key.toString()
+  //   }
+  //   // @ts-ignore
+  //   if (this.name) {
+  //     // @ts-ignore
+  //     params.name = this.name
+  //   }
+  //   const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
+  //   console.log('Import to dashboard: ' + url.toString())
+  //   return url
+  // }
+  // openDashboard() {
+  //   void this.getDashboardURL().then(url => {
+  //     if (url) window.open(url.toString(), '_blank')
+  //   })
+  // }
+};
+
 // src/crdt-helpers.ts
 function time(tag) {
 }
@@ -1833,7 +2384,7 @@ async function writeDocContent(store, blocks, update, logger) {
     await processFiles(store, blocks, update.value, logger);
     value = { doc: update.value };
   }
-  const block = await encode3({ value, hasher: hasher2, codec: codec2 });
+  const block = await encode({ value, hasher: hasher5, codec });
   blocks.putSync(block.cid, block.bytes);
   return block.cid;
 }
@@ -1842,10 +2393,16 @@ async function processFiles(store, blocks, doc, logger) {
     await processFileset(logger, store, blocks, doc._files);
   }
   if (doc._publicFiles) {
-    await processFileset(logger, store, blocks, doc._publicFiles, true);
+    await processFileset(
+      logger,
+      store,
+      blocks,
+      doc._publicFiles
+      /*, true*/
+    );
   }
 }
-async function processFileset(logger, store, blocks, files, publicFiles = false) {
+async function processFileset(logger, store, blocks, files) {
   const dbBlockstore = blocks.parent;
   if (!dbBlockstore.loader) throw logger.Error().Msg("Missing loader, database name is required").AsError();
   const t = new CarTransaction(dbBlockstore);
@@ -1867,9 +2424,10 @@ async function processFileset(logger, store, blocks, files, publicFiles = false)
     }
   }
   if (didPut.length) {
-    const car = await dbBlockstore.loader.commitFiles(t, { files }, {
-      public: publicFiles
-    });
+    const car = await dbBlockstore.loader.commitFiles(
+      t,
+      { files }
+    );
     if (car) {
       for (const name of didPut) {
         files[name] = { car, ...files[name] };
@@ -1903,7 +2461,7 @@ function readFileset(blocks, files, isPublic = false) {
         fileMeta.file = async () => await blocks.ebOpts.storeRuntime.decodeFile(
           {
             get: async (cid) => {
-              return await blocks.getFile(throwFalsy(fileMeta.car), cid, isPublic);
+              return await blocks.getFile(throwFalsy(fileMeta.car), cid);
             }
           },
           fileMeta.cid,
@@ -1917,7 +2475,7 @@ function readFileset(blocks, files, isPublic = false) {
 async function getValueFromLink(blocks, link, logger) {
   const block = await blocks.get(link);
   if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
-  const { value } = await decode3({ bytes: block.bytes, hasher: hasher2, codec: codec2 });
+  const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
   const cvalue = {
     ...value,
     cid: link
@@ -1926,17 +2484,21 @@ async function getValueFromLink(blocks, link, logger) {
   return cvalue;
 }
 var DirtyEventFetcher = class extends EventFetcher {
+  constructor(logger, blocks) {
+    super(blocks);
+    this.logger = logger;
+  }
   async get(link) {
     try {
       return super.get(link);
     } catch (e) {
-      console.error("missing event", link.toString(), e);
+      this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
       return { value: void 0 };
     }
   }
 };
 async function clockChangesSince(blocks, head, since, opts, logger) {
-  const eventsFetcher = opts.dirty ? new DirtyEventFetcher(blocks) : new EventFetcher(blocks);
+  const eventsFetcher = opts.dirty ? new DirtyEventFetcher(logger, blocks) : new EventFetcher(blocks);
   const keys = /* @__PURE__ */ new Set();
   const updates = await gatherUpdates(
     blocks,
@@ -2032,20 +2594,19 @@ async function doCompact(blockLog, head, logger) {
   isCompacting = false;
 }
 async function getBlock(blocks, cidString) {
-  const block = await blocks.get(parse2(cidString));
+  const block = await blocks.get(parse3(cidString));
   if (!block) throw new Error(`Missing block ${cidString}`);
-  const { cid, value } = await decode3({ bytes: block.bytes, codec: codec2, hasher: hasher2 });
-  return new Block2({ cid, value, bytes: block.bytes });
+  const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
+  return new Block({ cid, value, bytes: block.bytes });
 }
 
 // src/indexer-helpers.ts
-import { create as create3 } from "multiformats/block";
-import { sha256 as hasher3 } from "multiformats/hashes/sha2";
-import * as codec3 from "@ipld/dag-cbor";
+import { sha256 as hasher6 } from "multiformats/hashes/sha2";
+import * as codec2 from "@ipld/dag-cbor";
 import charwise from "charwise";
 import * as DbIndex from "prolly-trees/db-index";
-import { bf as bf2, simpleCompare } from "prolly-trees/utils";
-import { nocache as cache2 } from "prolly-trees/cache";
+import { bf, simpleCompare } from "prolly-trees/utils";
+import { nocache as cache } from "prolly-trees/cache";
 var IndexTree = class {
 };
 function refCompare(aRef, bRef) {
@@ -2061,8 +2622,8 @@ function compare(a, b) {
   if (comp !== 0) return comp;
   return refCompare(aRef, bRef);
 }
-var byKeyOpts = { cache: cache2, chunker: bf2(30), codec: codec3, hasher: hasher3, compare };
-var byIdOpts = { cache: cache2, chunker: bf2(30), codec: codec3, hasher: hasher3, compare: simpleCompare };
+var byKeyOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher6, compare };
+var byIdOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher6, compare: simpleCompare };
 function indexEntriesForChanges(changes, mapFn) {
   const indexEntries = [];
   changes.forEach(({ id: key, value, del }) => {
@@ -2090,7 +2651,7 @@ function makeProllyGetBlock(blocks) {
     const block = await blocks.get(address);
     if (!block) throw new Error(`Missing block ${address.toString()}`);
     const { cid, bytes } = block;
-    return create3({ cid, bytes, hasher: hasher3, codec: codec3 });
+    return create({ cid, bytes, hasher: hasher6, codec: codec2 });
   };
 }
 async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
@@ -2161,25 +2722,25 @@ function encodeKey(key) {
 }
 
 // src/indexer.ts
-function index({ _crdt }, name, mapFn, meta) {
+function index(sthis, { _crdt }, name, mapFn, meta) {
   if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
   if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
   if (_crdt.indexers.has(name)) {
     const idx = _crdt.indexers.get(name);
     idx.applyMapFn(name, mapFn, meta);
   } else {
-    const idx = new Index(_crdt, name, mapFn, meta);
+    const idx = new Index(sthis, _crdt, name, mapFn, meta);
     _crdt.indexers.set(name, idx);
   }
   return _crdt.indexers.get(name);
 }
 var Index = class {
-  constructor(crdt, name, mapFn, meta) {
+  constructor(sthis, crdt, name, mapFn, meta) {
     this.mapFnString = "";
     this.byKey = new IndexTree();
     this.byId = new IndexTree();
     this.includeDocsDefault = false;
-    this.logger = ensureLogger(crdt.logger, "Index");
+    this.logger = ensureLogger(sthis, "Index");
     this.blockstore = crdt.indexBlockstore;
     this.crdt = crdt;
     this.applyMapFn(name, mapFn, meta);
@@ -2234,7 +2795,7 @@ var Index = class {
           }
           if (this.mapFnString) {
             if (this.mapFnString !== mapFn.toString()) {
-              throw this.logger.Error().Msg("cannot apply different mapFn app").AsError();
+              throw this.logger.Error().Str("mapFnString", this.mapFnString).Str("mapFn", mapFn.toString()).Msg("cannot apply different mapFn app").AsError();
             }
           } else {
             this.mapFnString = mapFn.toString();
@@ -2369,7 +2930,7 @@ var Index = class {
 // src/crdt-clock.ts
 import { advance } from "@web3-storage/pail/clock";
 import { root as root2 } from "@web3-storage/pail/crdt";
-import { ResolveOnce as ResolveOnce3 } from "@adviser/cement";
+import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
 
 // src/apply-head-queue.ts
 function applyHeadQueue(worker, logger) {
@@ -2384,7 +2945,7 @@ function applyHeadQueue(worker, logger) {
         queue.sort((a, b) => b.updates ? 1 : -1);
         const task = queue.shift();
         if (!task) continue;
-        await worker(task.newHead, task.prevHead, task.updates !== null).catch((e) => {
+        await worker(task.newHead, task.prevHead, task.updates !== void 0).catch((e) => {
           throw logger.Error().Err(e).Msg("int_applyHead worker error").AsError();
         });
         if (task.updates) {
@@ -2426,9 +2987,9 @@ var CRDTClock = class {
     this.zoomers = /* @__PURE__ */ new Set();
     this.watchers = /* @__PURE__ */ new Set();
     this.emptyWatchers = /* @__PURE__ */ new Set();
-    this._ready = new ResolveOnce3();
+    this._ready = new ResolveOnce4();
     this.blockstore = blockstore;
-    this.logger = ensureLogger(blockstore.logger, "CRDTClock");
+    this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
     this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
   }
   async ready() {
@@ -2474,6 +3035,7 @@ var CRDTClock = class {
     this.zoomers.add(fn);
   }
   async int_applyHead(newHead, prevHead, localUpdates) {
+    const noLoader = !localUpdates;
     const ogHead = sortClockHead(this.head);
     newHead = sortClockHead(newHead);
     if (compareClockHeads(ogHead, newHead)) {
@@ -2484,7 +3046,6 @@ var CRDTClock = class {
       this.setHead(newHead);
       return;
     }
-    const noLoader = !localUpdates;
     if (!this.blockstore) {
       throw this.logger.Error().Msg("missing blockstore").AsError();
     }
@@ -2501,7 +3062,7 @@ var CRDTClock = class {
         }
         return { head: advancedHead };
       },
-      { noLoader }
+      { noLoader, add: false }
     );
     this.setHead(meta.head);
   }
@@ -2535,13 +3096,14 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
 
 // src/crdt.ts
 var CRDT = class {
-  constructor(name, opts = {}) {
-    this.onceReady = new ResolveOnce4();
+  constructor(sthis, name, opts = {}) {
     this.indexers = /* @__PURE__ */ new Map();
+    this.onceReady = new ResolveOnce5();
+    this.sthis = sthis;
     this.name = name;
-    this.logger = ensureLogger(opts, "CRDT");
+    this.logger = ensureLogger(sthis, "CRDT");
     this.opts = opts;
-    this.blockstore = blockstoreFactory({
+    this.blockstore = blockstoreFactory(sthis, {
       name,
       applyMeta: async (meta) => {
         const crdtMeta = meta;
@@ -2553,22 +3115,20 @@ var CRDT = class {
         return { head: this.clock.head };
       },
       autoCompact: this.opts.autoCompact || 100,
-      crypto: this.opts.crypto,
       store: { ...this.opts.store, isIndex: void 0 },
       public: this.opts.public,
       meta: this.opts.meta,
       threshold: this.opts.threshold
     });
-    this.indexBlockstore = blockstoreFactory({
+    this.indexBlockstore = blockstoreFactory(sthis, {
       name,
       applyMeta: async (meta) => {
         const idxCarMeta = meta;
         if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
         for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
-          index({ _crdt: this }, name2, void 0, idx);
+          index(this.sthis, { _crdt: this }, name2, void 0, idx);
         }
       },
-      crypto: this.opts.crypto,
       store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
       public: this.opts.public
     });
@@ -2579,17 +3139,6 @@ var CRDT = class {
       }
     });
   }
-  async ready() {
-    return this.onceReady.once(async () => {
-      await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
-    });
-  }
-  async close() {
-    await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
-  }
-  async destroy() {
-    await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
-  }
   async bulk(updates) {
     await this.ready();
     const prevHead = [...this.clock.head];
@@ -2610,6 +3159,21 @@ var CRDT = class {
     await this.clock.applyHead(done.meta.head, prevHead, updates);
     return done.meta;
   }
+  async ready() {
+    return this.onceReady.once(async () => {
+      try {
+        await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
+      } catch (e) {
+        throw this.logger.Error().Err(e).Msg(`CRDT is not ready`).AsError();
+      }
+    });
+  }
+  async close() {
+    await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
+  }
+  async destroy() {
+    await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
+  }
   // if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
   async allDocs() {
     await this.ready();
@@ -2654,11 +3218,12 @@ var Database = class {
     this._listening = false;
     this._listeners = /* @__PURE__ */ new Set();
     this._noupdate_listeners = /* @__PURE__ */ new Set();
-    this._ready = new ResolveOnce5();
+    this._ready = new ResolveOnce6();
     this.name = name;
     this.opts = opts || this.opts;
-    this.logger = ensureLogger(this.opts, "Database");
-    this._crdt = new CRDT(name, this.opts);
+    this.sthis = ensureSuperThis(this.opts);
+    this.logger = ensureLogger(this.sthis, "Database");
+    this._crdt = new CRDT(this.sthis, name, this.opts);
     this.blockstore = this._crdt.blockstore;
     this._writeQueue = writeQueue(async (updates) => {
       return await this._crdt.bulk(updates);
@@ -2682,7 +3247,7 @@ var Database = class {
   }
   async ready() {
     return this._ready.once(async () => {
-      await SysContainer.start();
+      await this.sthis.start();
       await this._crdt.ready();
       await this.blockstore.ready();
     });
@@ -2702,7 +3267,7 @@ var Database = class {
     await this.ready();
     this.logger.Debug().Str("id", doc._id).Msg("put");
     const { _id, ...value } = doc;
-    const docId = _id || uuidv7();
+    const docId = _id || this.sthis.timeOrderedNextId().str;
     const result = await this._writeQueue.push({
       id: docId,
       value: {
@@ -2767,7 +3332,7 @@ var Database = class {
     await this.ready();
     this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
     const _crdt = this._crdt;
-    const idx = typeof field === "string" ? index({ _crdt }, field) : index({ _crdt }, makeName(field.toString()), field);
+    const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
     return await idx.query(opts);
   }
   async compact() {
@@ -2804,12 +3369,7 @@ function fireproof(name, opts) {
   const key = JSON.stringify(
     toSortedArray({
       name,
-      stores: toSortedArray(opts?.store?.stores),
-      makeMetaStore: !!opts?.store?.makeMetaStore,
-      makeDataStore: !!opts?.store?.makeDataStore,
-      makeRemoteWAL: !!opts?.store?.makeRemoteWAL,
-      encodeFile: !!opts?.store?.encodeFile,
-      decodeFile: !!opts?.store?.decodeFile
+      stores: toSortedArray(opts?.store?.stores)
     })
   );
   let db = Database.databases.get(key);
@@ -2824,7 +3384,10 @@ function makeName(fnString) {
   let found = null;
   const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
   if (matches.length === 0) {
-    found = /=>\s*(.*)/.exec(fnString);
+    found = /=>\s*{?\s*([^{}]+)\s*}?/.exec(fnString);
+    if (found && found[1].includes("return")) {
+      found = null;
+    }
   }
   if (!found) {
     return fnString;
@@ -2833,20 +3396,53 @@ function makeName(fnString) {
   }
 }
 
+// src/runtime/index.ts
+var runtime_exports = {};
+__export(runtime_exports, {
+  FILESTORE_VERSION: () => FILESTORE_VERSION,
+  INDEXDB_VERSION: () => INDEXDB_VERSION,
+  files: () => files_exports,
+  getFileName: () => getFileName,
+  getFileSystem: () => getFileSystem,
+  getPath: () => getPath,
+  kb: () => key_bag_exports,
+  kc: () => keyed_crypto_exports,
+  mf: () => wait_pr_multiformats_exports,
+  runtimeFn: () => runtimeFn2,
+  toArrayBuffer: () => toArrayBuffer
+});
+
+// src/runtime/wait-pr-multiformats/index.ts
+var wait_pr_multiformats_exports = {};
+__export(wait_pr_multiformats_exports, {
+  block: () => block_exports,
+  codec: () => codec_interface_exports
+});
+
+// src/runtime/wait-pr-multiformats/codec-interface.ts
+var codec_interface_exports = {};
+
+// src/runtime/index.ts
+import { runtimeFn as runtimeFn2 } from "@adviser/cement";
+
 // src/version.ts
 var PACKAGE_VERSION = Object.keys({
-  "0.19.8-dev-global": "xxxx"
+  "0.19.8-dev-series-2": "xxxx"
 })[0];
 export {
   CRDT,
   Database,
   Index,
+  NotFoundError,
   PACKAGE_VERSION,
   Result,
+  UInt8ArrayEqual,
   blockstore_exports as blockstore,
   blockstore_exports as bs,
+  dataDir,
   ensureLogger,
-  exception2Result,
+  ensureSuperLog,
+  ensureSuperThis,
   exceptionWrapper,
   falsyToUndef,
   fireproof,
@@ -2855,6 +3451,7 @@ export {
   getStore,
   index,
   isFalsy,
+  isNotFoundError,
   runtime_exports as rt,
   runtime_exports as runtime,
   throwFalsy