@fireproof/core 0.20.0-dev-preview-40 → 0.20.0-dev-preview-50

package/index.cjs

@@ -31,6 +31,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   CRDTImpl: () => CRDTImpl,
+  DataAndMetaAndWalAndBaseStore: () => DataAndMetaAndWalAndBaseStore,
   DatabaseImpl: () => DatabaseImpl,
   Index: () => Index,
   LedgerFactory: () => LedgerFactory,
@@ -47,6 +48,7 @@ __export(index_exports, {
   ensureLogger: () => ensureLogger,
   ensureSuperLog: () => ensureSuperLog,
   ensureSuperThis: () => ensureSuperThis,
+  ensureURIDefaults: () => ensureURIDefaults,
   exceptionWrapper: () => exceptionWrapper,
   falsyToUndef: () => falsyToUndef,
   fireproof: () => fireproof,
@@ -64,6 +66,7 @@ __export(index_exports, {
   onSuperThis: () => onSuperThis,
   rt: () => runtime_exports,
   runtime: () => runtime_exports,
+  storeType2DataMetaWal: () => storeType2DataMetaWal,
   throwFalsy: () => throwFalsy,
   toSortedArray: () => toSortedArray,
   toStoreURIRuntime: () => toStoreURIRuntime
@@ -100,8 +103,13 @@ var PARAM = {
   FRAG_LEN: "len",
   FRAG_HEAD: "headerSize",
   EXTRACTKEY: "extractKey",
-  SELF_REFLECT: "selfReflect"
+  SELF_REFLECT: "selfReflect",
   // if no subscribe in Gateway see your own META updates
+  CAR_PARALLEL: "parallel",
+  CAR_CACHE_SIZE: "carCacheSize",
+  CAR_COMPACT_CACHE_SIZE: "carCompactCacheSize",
+  CAR_META_CACHE_SIZE: "carMetaCacheSize",
+  GENESIS_CID: "baembeiarootfireproofgenesisblockaaaafireproofgenesisblocka"
   // FS = "fs",
 };
 function throwFalsy(value) {
@@ -116,6 +124,18 @@ function falsyToUndef(value) {
   }
   return value;
 }
+var DataAndMetaAndWalAndBaseStore = class {
+  constructor(dam) {
+    this.wal = dam.wal;
+    this.file = dam.file;
+    this.car = dam.car;
+    this.meta = dam.meta;
+    this.baseStores = [this.file, this.car, this.meta];
+    if (this.wal) {
+      this.baseStores.push(this.wal);
+    }
+  }
+};
 
 // src/utils.ts
 var import_base58 = require("multiformats/bases/base58");
@@ -326,21 +346,25 @@ function ensureLogger(sthis, componentName, ctx) {
   return out;
 }
 function getStore(url, sthis, joiner) {
-  const store = url.getParam(PARAM.STORE);
-  switch (store) {
-    case "data":
+  const fromUrl = url.getParam(PARAM.STORE);
+  let pathPart;
+  switch (fromUrl) {
+    case "car":
+    case "file":
+      pathPart = "data";
+      break;
     case "wal":
     case "meta":
+      pathPart = fromUrl;
       break;
     default:
       throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
-      throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
   }
-  let name = store;
+  let name = pathPart;
   if (url.hasParam("index")) {
     name = joiner(url.getParam(PARAM.INDEX) || "idx", name);
   }
-  return { store, name };
+  return { pathPart, fromUrl, name };
 }
 function getKey(url, logger) {
   const result = url.getParam(PARAM.KEY);
@@ -442,6 +466,35 @@ function makeName(fnString) {
     return found[1];
   }
 }
+function storeType2DataMetaWal(store) {
+  switch (store) {
+    case "car":
+    case "file":
+      return "data";
+    case "meta":
+    case "wal":
+      return store;
+    default:
+      throw new Error(`unknown store ${store}`);
+  }
+}
+function ensureURIDefaults(sthis, name, curi, uri, store, ctx) {
+  ctx = ctx || {};
+  const ret = (curi ? import_cement.URI.from(curi) : uri).build().setParam(PARAM.STORE, store).defParam(PARAM.NAME, name);
+  if (!ret.hasParam(PARAM.NAME)) {
+    throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Ledger name is required").AsError();
+  }
+  if (ctx.idx) {
+    ret.defParam(PARAM.INDEX, "idx");
+    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${storeType2DataMetaWal(store)}-idx@`);
+  } else {
+    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${storeType2DataMetaWal(store)}@`);
+  }
+  if (store === "car") {
+    ret.defParam(PARAM.SUFFIX, ".car");
+  }
+  return ret.URI();
+}
 
 // src/write-queue.ts
 var import_cement2 = require("@adviser/cement");
@@ -1123,8 +1176,10 @@ function getFileName(url, sthis) {
   const key = url.getParam("key");
   if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
   const res = (0, import_core.getStore)(url, sthis, (...a) => a.join("-"));
-  switch (res.store) {
-    case "data":
+  switch (res.fromUrl) {
+    case "file":
+      return sthis.pathOps.join(res.name, key);
+    case "car":
       return sthis.pathOps.join(res.name, key + ".car");
     case "wal":
     case "meta":
@@ -1272,8 +1327,13 @@ var MemoryGateway = class {
   close(baseUrl) {
     return Promise.resolve(import_cement6.Result.Ok(void 0));
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   destroy(baseUrl) {
+    const keyUrl = baseUrl.toString();
+    for (const key of this.memorys.keys()) {
+      if (key.startsWith(keyUrl)) {
+        this.memorys.delete(key);
+      }
+    }
     this.memorys.clear();
     return Promise.resolve(import_cement6.Result.Ok(void 0));
   }
@@ -1285,7 +1345,7 @@ var MemoryGateway = class {
   get(url) {
     const x = this.memorys.get(url.toString());
     if (!x) {
-      return Promise.resolve(import_cement6.Result.Err(new NotFoundError("not found")));
+      return Promise.resolve(import_cement6.Result.Err(new NotFoundError(`not found: ${url.toString()}`)));
     }
     return Promise.resolve(import_cement6.Result.Ok(x));
   }
@@ -1391,9 +1451,6 @@ async function decode2DbMetaEvents(sthis, rserializedMeta) {
   if (!Array.isArray(serializedMeta)) {
     return sthis.logger.Debug().Any("metaEntries", serializedMeta).Msg("No data in MetaEntries").ResultError();
   }
-  if (!serializedMeta.length) {
-    return sthis.logger.Debug().Msg("No MetaEntries found").ResultError();
-  }
   return import_cement8.Result.Ok(
     await Promise.all(
       serializedMeta.map(async (metaEntry) => {
@@ -1458,10 +1515,9 @@ async function fpDeserialize(sthis, url, intoRaw, pdecoder) {
     ...pdecoder
   };
   switch (url.getParam(PARAM.STORE)) {
-    case "data":
-      if (url.getParam(PARAM.SUFFIX) === ".car") {
-        return makeFPEnvelope(FPEnvelopeTypes.CAR, await decoder.car(sthis, raw2));
-      }
+    case "car":
+      return makeFPEnvelope(FPEnvelopeTypes.CAR, await decoder.car(sthis, raw2));
+    case "file":
       return makeFPEnvelope(FPEnvelopeTypes.FILE, await decoder.file(sthis, raw2));
     case "wal":
       return makeFPEnvelope(FPEnvelopeTypes.WAL, await decode2WalState(sthis, await decoder.wal(sthis, raw2)));
@@ -1523,7 +1579,7 @@ var DefSerdeGateway = class {
       const urlWithoutKey = url.build().delParam(PARAM.KEY).delParam(PARAM.SELF_REFLECT).toString();
       this.subscribeFn.set(urlWithoutKey, rawCallback);
       return import_cement9.Result.Ok(() => {
-        this.subscribeFn.delete(url.toString());
+        this.subscribeFn.delete(urlWithoutKey);
       });
     }
     const unreg = await this.gw.subscribe(url, rawCallback, sthis);
@@ -1652,37 +1708,49 @@ var Block = import_block.Block;
 async function decode({
   bytes,
   codec: codec3,
-  hasher: hasher7
+  hasher: hasher6
 }) {
   if (bytes == null) throw new Error('Missing required argument "bytes"');
-  if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
+  if (codec3 == null || hasher6 == null) throw new Error("Missing required argument: codec or hasher");
   const value = await Promise.resolve(codec3.decode(bytes));
-  const hash = await hasher7.digest(bytes);
+  let toHash = bytes;
+  if (codec3.valueToHashBytes) {
+    toHash = await Promise.resolve(codec3.valueToHashBytes(value));
+  }
+  const hash = await hasher6.digest(toHash);
   const cid = import_multiformats2.CID.create(1, codec3.code, hash);
-  return new import_block.Block({ value, bytes, cid });
+  return new import_block.Block({ value, bytes: toHash, cid });
 }
 async function encode({
   value,
   codec: codec3,
-  hasher: hasher7
+  hasher: hasher6
 }) {
   if (typeof value === "undefined") throw new Error('Missing required argument "value"');
-  if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
-  const bytes = await Promise.resolve(codec3.encode(value));
-  const hash = await hasher7.digest(bytes);
+  if (codec3 == null || hasher6 == null) throw new Error("Missing required argument: codec or hasher");
+  let bytes;
+  let hash;
+  if (codec3.bytesToHash) {
+    const hashable = await Promise.resolve(codec3.bytesToHash(value));
+    hash = await hasher6.digest(hashable);
+    bytes = await Promise.resolve(codec3.encode(value));
+  } else {
+    bytes = await Promise.resolve(codec3.encode(value));
+    hash = await hasher6.digest(bytes);
+  }
   const cid = import_multiformats2.CID.create(1, codec3.code, hash);
-  return new import_block.Block({ value, bytes, cid });
+  return new Block({ value, bytes, cid });
 }
 async function create({
   bytes,
   cid,
-  hasher: hasher7,
+  hasher: hasher6,
   codec: codec3
 }) {
   if (bytes == null) throw new Error('Missing required argument "bytes"');
-  if (hasher7 == null) throw new Error('Missing required argument "hasher"');
+  if (hasher6 == null) throw new Error('Missing required argument "hasher"');
   const value = await Promise.resolve(codec3.decode(bytes));
-  const hash = await hasher7.digest(bytes);
+  const hash = await hasher6.digest(bytes);
   if (!import_multiformats2.bytes.equals(cid.multihash.bytes, hash.bytes)) {
     throw new Error("CID hash does not match bytes");
   }
@@ -2080,6 +2148,9 @@ var DatabaseImpl = class {
     this.id = ledger.id;
     this.logger = ensureLogger(this.sthis, "Database");
   }
+  attach(a) {
+    return this.ledger.attach(a);
+  }
   get name() {
     return this.ledger.name;
   }
@@ -2175,19 +2246,27 @@ var import_cement19 = require("@adviser/cement");
 // src/blockstore/index.ts
 var blockstore_exports = {};
 __export(blockstore_exports, {
+  AttachedRemotesImpl: () => AttachedRemotesImpl,
+  BaseActiveStore: () => BaseActiveStore,
   BaseBlockstoreImpl: () => BaseBlockstoreImpl,
   Car2FPMsg: () => Car2FPMsg,
+  CarActiveStore: () => CarActiveStore,
+  CarLog: () => CarLog,
   CarTransactionImpl: () => CarTransactionImpl,
   CompactionFetcher: () => CompactionFetcher,
-  ConnectionBase: () => ConnectionBase,
   DbMetaEventEqual: () => DbMetaEventEqual,
   DbMetaEventsEqual: () => DbMetaEventsEqual,
   EncryptedBlockstore: () => EncryptedBlockstore,
   FPEnvelopeTypes: () => FPEnvelopeTypes,
   File2FPMsg: () => File2FPMsg,
+  FileActiveStore: () => FileActiveStore,
   InterceptorGateway: () => InterceptorGateway,
   Loader: () => Loader,
+  MetaActiveStore: () => MetaActiveStore,
   PassThroughGateway: () => PassThroughGateway,
+  TaskManager: () => TaskManager,
+  WALActiveStore: () => WALActiveStore,
+  createAttachedStores: () => createAttachedStores,
   createDbMetaEvent: () => createDbMetaEvent,
   defaultGatewayFactoryItem: () => defaultGatewayFactoryItem,
   ensureStoreEnDeFile: () => ensureStoreEnDeFile,
@@ -2201,6 +2280,37 @@ __export(blockstore_exports, {
 });
 
 // src/blockstore/types.ts
+var CarLog = class {
+  constructor() {
+    this._logs = [];
+  }
+  get length() {
+    return this._logs.length;
+  }
+  last() {
+    const x = [...this._logs[this._logs.length - 1]];
+    Object.freeze(x);
+    return x;
+  }
+  xunshift(logs) {
+    this._logs.unshift(logs);
+  }
+  update(logs) {
+    this._logs.length = 0;
+    this._logs.push(...logs);
+  }
+  asArray() {
+    const a = [
+      ...this._logs.map((l) => {
+        const x = [...l];
+        Object.freeze(x);
+        return x;
+      })
+    ];
+    Object.freeze(a);
+    return a;
+  }
+};
 function toCIDBlock(block) {
   return block;
 }
@@ -2210,24 +2320,34 @@ function DbMetaEventEqual(a, b) {
 function DbMetaEventsEqual(a, b) {
   return a.length === b.length && a.every((e, i) => DbMetaEventEqual(e, b[i]));
 }
+var BaseActiveStore = class {
+};
+var CarActiveStore = class extends BaseActiveStore {
+};
+var FileActiveStore = class extends BaseActiveStore {
+};
+var MetaActiveStore = class extends BaseActiveStore {
+};
+var WALActiveStore = class extends BaseActiveStore {
+};
 
 // src/blockstore/store-factory.ts
-var import_cement16 = require("@adviser/cement");
+var import_cement17 = require("@adviser/cement");
 
 // src/blockstore/store.ts
-var import_cement15 = require("@adviser/cement");
+var import_cement16 = require("@adviser/cement");
 
 // src/blockstore/loader.ts
 var import_p_limit = __toESM(require("p-limit"), 1);
 var import_reader = require("@ipld/car/reader");
-var import_cement13 = require("@adviser/cement");
+var import_cement14 = require("@adviser/cement");
 
 // src/blockstore/loader-helpers.ts
 var import_sha22 = require("multiformats/hashes/sha2");
 var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
 async function parseCarFile(reader, logger) {
-  const roots = await reader.getRoots();
-  const header = await reader.get(roots[0]);
+  const roots = await reader.roots;
+  const header = reader.blocks.find((i) => i.cid.equals(roots[0]));
   if (!header) throw logger.Error().Msg("missing header block").AsError();
   const dec = await decode({ bytes: header.bytes, hasher: import_sha22.sha256, codec: dagCodec });
   const fpvalue = dec.value;
@@ -2242,6 +2362,7 @@ var import_block4 = require("@fireproof/vendor/@web3-storage/pail/block");
 var import_cement11 = require("@adviser/cement");
 var CarTransactionImpl = class {
   #memblock = new import_block4.MemoryBlockstore();
+  #hackUnshift;
   constructor(parent, opts = { add: true, noLoader: false }) {
     if (opts.add) {
       parent.transactions.add(this);
@@ -2249,7 +2370,7 @@ var CarTransactionImpl = class {
     this.parent = parent;
   }
   async get(cid) {
-    return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
+    return await this.superGet(cid) ?? falsyToUndef(await this.parent.get(cid));
   }
   async superGet(cid) {
     return this.#memblock.get(cid);
@@ -2260,7 +2381,16 @@ var CarTransactionImpl = class {
   putSync(cid, bytes) {
     this.#memblock.putSync(cid, bytes);
   }
+  unshift(cid, bytes) {
+    if (this.#hackUnshift) {
+      throw new Error("unshift already called");
+    }
+    this.#hackUnshift = { cid, bytes };
+  }
   async *entries() {
+    if (this.#hackUnshift) {
+      yield this.#hackUnshift;
+    }
     for await (const blk of this.#memblock.entries()) {
       yield blk;
     }
@@ -2286,6 +2416,11 @@ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
     keyBag: opts.keyBag || {},
     crypto: (0, import_cement11.toCryptoRuntime)(opts.crypto),
     storeUrls: opts.storeUrls,
+    taskManager: {
+      removeAfter: 3,
+      retryTimeout: 50,
+      ...opts.taskManager
+    },
     // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
     // store,
     storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
@@ -2378,10 +2513,8 @@ var EncryptedBlockstore = class extends BaseBlockstoreImpl {
   async get(cid) {
     const got = await super.get(cid);
     if (got) return got;
-    if (!this.loader) {
-      return;
-    }
-    return falsyToUndef(await this.loader.getBlock(cid));
+    const ret = falsyToUndef(await this.loader.getBlock(cid, this.loader.attachedStores.local()));
+    return ret;
   }
   async transaction(fn, opts = { noLoader: false }) {
     this.logger.Debug().Msg("enter transaction");
@@ -2401,11 +2534,8 @@ var EncryptedBlockstore = class extends BaseBlockstoreImpl {
   async getFile(car, cid) {
     await this.ready();
     if (!this.loader) throw this.logger.Error().Msg("loader required to get file, ledger must be named").AsError();
-    const reader = await this.loader.loadFileCar(
-      car
-      /*, isPublic */
-    );
-    const block = await reader.get(cid);
+    const reader = await this.loader.loadFileCar(car, this.loader.attachedStores.local());
+    const block = await reader.blocks.find((i) => i.cid.equals(cid));
     if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
     return block.bytes;
   }
@@ -2512,8 +2642,8 @@ var CommitQueue = class {
 
 // src/blockstore/commitor.ts
 var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
-var import_sha23 = require("multiformats/hashes/sha2");
 var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
+var import_sha23 = require("multiformats/hashes/sha2");
 async function encodeCarFile(roots, t, codec3) {
   let size = 0;
   const headerSize = CBW.headerLength({ roots });
@@ -2538,7 +2668,7 @@ async function createCarFile(encoder, cid, t) {
 async function commitFiles(fileStore, walStore, t, done) {
   const { files: roots } = makeFileCarHeader(done);
   const cids = [];
-  const codec3 = (await fileStore.keyedCrypto()).codec();
+  const codec3 = await fileStore.keyedCrypto().then((i) => i.codec());
   const cars = await prepareCarFilesFiles(codec3, roots, t);
   for (const car of cars) {
     const { cid, bytes } = car;
@@ -2564,7 +2694,7 @@ async function prepareCarFilesFiles(encoder, roots, t) {
   return [await encodeCarFile(roots, t, encoder)];
 }
 function makeCarHeader(meta, cars, compact = false) {
-  const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
+  const coreHeader = compact ? { cars: [], compact: cars.asArray() } : { cars: cars.asArray(), compact: [] };
   return { ...coreHeader, meta };
 }
 async function encodeCarHeader(fp) {
@@ -2591,7 +2721,7 @@ async function commit(params, t, done, opts = { noLoader: false, compact: false
 }
 async function prepareCarFiles(encoder, threshold, rootBlock, t) {
   const carFiles = [];
-  threshold = threshold || 128e3 * 8;
+  threshold = threshold || 16 * 65536;
   let clonedt = new CarTransactionImpl(t.parent, { add: false, noLoader: false });
   clonedt.putSync(rootBlock.cid, rootBlock.bytes);
   let newsize = CBW.blockLength(toCIDBlock(rootBlock));
@@ -2617,19 +2747,20 @@ var import_sha24 = require("multiformats/hashes/sha2");
 
 // src/blockstore/task-manager.ts
 var TaskManager = class {
-  constructor(sthis, callback) {
+  constructor(sthis, callback, params) {
     // we need to remove the events after some time
     this.eventsWeHandled = /* @__PURE__ */ new Set();
     this.queue = [];
     this.isProcessing = false;
     this.logger = ensureLogger(sthis, "TaskManager");
     this.callback = callback;
+    this.params = params;
   }
-  async handleEvent(cid, parents, dbMeta) {
+  async handleEvent(cid, parents, dbMeta, store) {
     for (const parent of parents) {
       this.eventsWeHandled.add(parent.toString());
     }
-    this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
+    this.queue.push({ cid: cid.toString(), dbMeta, retries: 0, store });
     this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
     void this.processQueue();
   }
@@ -2643,7 +2774,7 @@ var TaskManager = class {
       return;
     }
     try {
-      await this.callback(first.dbMeta);
+      await this.callback(first.dbMeta, first.store);
       this.eventsWeHandled.add(first.cid);
       this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
     } catch (err) {
@@ -2652,7 +2783,7 @@ var TaskManager = class {
         this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
       }
       await new Promise((resolve) => setTimeout(resolve, 50));
-      throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
+      this.logger.Warn().Err(err).Msg("retry to process event block");
     } finally {
       this.isProcessing = false;
       if (this.queue.length > 0) {
@@ -2662,13 +2793,299 @@ var TaskManager = class {
   }
 };
 
+// src/blockstore/attachable-store.ts
+var import_cement13 = require("@adviser/cement");
+var AttachedImpl = class {
+  constructor(gws, stores, unreg) {
+    this.gatewayUrls = gws;
+    this.stores = new DataAndMetaAndWalAndBaseStore(stores);
+    this.unreg = unreg;
+  }
+  async detach() {
+    const toClose = [this.stores.car.close(), this.stores.file.close(), this.stores.meta.close()];
+    if (this.stores.wal) {
+      toClose.push(this.stores.wal.close());
+    }
+    await Promise.all(toClose);
+    this.unreg();
+  }
+  status() {
+    return "attached";
+  }
+};
+var FileActiveStoreImpl = class extends FileActiveStore {
+  constructor(ref, active, attached) {
+    super();
+    this.ref = ref;
+    this.active = active;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return this.xattached.remotes();
+  }
+};
+var CarActiveStoreImpl = class extends CarActiveStore {
+  constructor(ref, active, attached) {
+    super();
+    this.ref = ref;
+    this.active = active;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return [this.active, ...this.xattached.remotes().filter((i) => i !== this.active)];
+  }
+};
+var CarAttachedStoresImpl = class {
+  constructor(attached) {
+    this.attached = attached;
+  }
+  local() {
+    return this.attached.local().active.car;
+  }
+  remotes() {
+    return this.attached.remotes().map(({ active }) => active.car);
+  }
+};
+var FileAttachedStoresImpl = class {
+  constructor(attached) {
+    this.attached = attached;
+  }
+  local() {
+    return this.attached.local().active.file;
+  }
+  remotes() {
+    return this.attached.remotes().map(({ active }) => active.file);
+  }
+};
+var MetaActiveStoreImpl = class extends MetaActiveStore {
+  constructor(ref, active, attached) {
+    super();
+    this.ref = ref;
+    this.active = active;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return [this.active, ...this.xattached.remotes().filter((i) => i !== this.active)];
+  }
+};
+var MetaAttachedStoresImpl = class {
+  constructor(attached) {
+    this.attached = attached;
+  }
+  local() {
+    return this.attached.local().active.meta;
+  }
+  remotes() {
+    return this.attached.remotes().map(({ active }) => active.meta);
+  }
+};
+var WALActiveStoreImpl = class extends WALActiveStore {
+  constructor(ref, active, attached) {
+    super();
+    this.ref = ref;
+    this.active = active;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return this.xattached.remotes();
+  }
+};
+var WALAttachedStoresImpl = class {
+  constructor(attached) {
+    this.attached = attached;
+  }
+  local() {
+    return this.attached.local().active.wal;
+  }
+  remotes() {
+    return this.attached.remotes().filter(({ active }) => active.wal).map(({ active }) => active.wal);
+  }
+};
+var ActiveStoreImpl = class {
+  constructor(active, attached) {
+    this.active = active;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return this.xattached.remotes();
+  }
+  baseStores() {
+    const bs = [this.active.car, this.active.file, this.active.meta];
+    if (this.active.wal) {
+      bs.push(this.active.wal);
+    }
+    return bs;
+  }
+  carStore() {
+    return new CarActiveStoreImpl(this, this.active.car, new CarAttachedStoresImpl(this.xattached));
+  }
+  fileStore() {
+    return new FileActiveStoreImpl(this, this.active.file, new FileAttachedStoresImpl(this.xattached));
+  }
+  metaStore() {
+    return new MetaActiveStoreImpl(this, this.active.meta, new MetaAttachedStoresImpl(this.xattached));
+  }
+  walStore() {
+    if (!this.active.wal) {
+      throw this.xattached.loadable.sthis.logger.Error().Msg("wal store not set").AsError();
+    }
+    return new WALActiveStoreImpl(this, this.active.wal, new WALAttachedStoresImpl(this.xattached));
+  }
+};
+function isLoadable(unknown) {
+  return !!unknown.sthis && !!unknown.attachedStores;
+}
+async function createAttachedStores(urlOrGup, arOrLoadable, name = "local") {
+  let ar;
+  if (!isLoadable(arOrLoadable)) {
+    ar = arOrLoadable;
+  } else {
+    ar = arOrLoadable.attachedStores;
+  }
+  let gup;
+  if (!urlOrGup) {
+    throw new Error("urlOrGup is required");
+  }
+  if ((0, import_cement13.isCoerceURI)(urlOrGup)) {
+    const url = urlOrGup;
+    gup = {
+      car: { url },
+      file: { url },
+      meta: { url },
+      wal: { url }
+    };
+  } else {
+    gup = urlOrGup;
+  }
+  return await ar.attach({
+    name,
+    prepare: async () => gup
+  });
+}
+var AttachedRemotesImpl = class {
+  constructor(loadable) {
+    this._remotes = new import_cement13.KeyedResolvOnce();
+    this.loadable = loadable;
+  }
+  forRemotes(action) {
+    return Promise.all(this.remotes().map((i) => action(i))).then(() => void 0);
+  }
+  remotes() {
+    return this._remotes.values().filter(({ value }) => value.isOk() && !value.Ok().stores.wal).map(({ value }) => value.Ok().stores).map((i) => this.activate(i));
+  }
+  local() {
+    if (!this._local) {
+      throw this.loadable.sthis.logger.Error().Msg("local store not set").AsError();
+    }
+    return new ActiveStoreImpl(this._local.stores, this);
+  }
+  activate(store) {
+    if ((0, import_cement13.isCoerceURI)(store)) {
+      const activateUrl = import_cement13.URI.from(store);
+      let maxScore = 0;
+      let maxStore;
+      for (const { value } of this._remotes.values()) {
+        if (value.isErr()) {
+          continue;
+        }
+        for (const url of value.Ok().stores.baseStores.map((i) => i.url())) {
+          const mr = url.match(activateUrl);
+          if (mr.score > maxScore) {
+            maxScore = mr.score;
+            maxStore = value.Ok().stores;
+          }
+        }
+      }
+      if (!maxStore) {
+        throw this.loadable.sthis.logger.Error().Url(activateUrl).Msg("no store found").AsError();
+      }
+      store = maxStore;
+    }
+    return new ActiveStoreImpl(store, this);
+  }
+  async detach() {
+    await Promise.all(
+      this._remotes.values().map(async ({ value: rvalue }) => {
+        if (rvalue.isOk()) {
+          await rvalue.Ok().detach();
+        }
+      })
+    );
+  }
+  async attach(attached) {
+    const gwp = await attached.prepare();
+    const gws = {
+      car: {
+        ...gwp.car,
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, gwp.car.url, import_cement13.URI.from(gwp.car.url), "car")
+      },
+      file: {
+        ...gwp.file,
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, void 0, import_cement13.URI.from(gwp.file.url), "file", { file: true })
+      },
+      meta: {
+        ...gwp.meta,
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, void 0, import_cement13.URI.from(gwp.meta.url), "meta")
+      },
+      wal: gwp.wal ? {
+        ...gwp.wal,
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, void 0, import_cement13.URI.from(gwp.wal.url), "wal")
+      } : void 0
+    };
+    const key = JSON.stringify(
+      toSortedArray({
+        carUrl: gws.car.url.toString(),
+        filesUrl: gws.file.url.toString(),
+        metaUrl: gws.meta.url.toString(),
+        walUrl: gws.wal?.url.toString()
+      })
+    );
+    return this._remotes.get(key).once(async () => {
+      const rt = toStoreRuntime(this.loadable.sthis);
+      const result = new AttachedImpl(
+        gws,
+        await rt.makeStores({
+          byStore: gws,
+          loader: this.loadable
+        }),
+        () => {
+          this._remotes.unget(key);
+        }
+      );
+      if (result.stores.wal) {
+        if (this._local) {
+          throw this.loadable.sthis.logger.Error().Msg("local store could only set once").AsError();
+        }
+        this._local = result;
+      }
+      return result;
+    });
+  }
+};
+
 // src/blockstore/loader.ts
 function carLogIncludesGroup(list, cids) {
-  return list.some((arr) => {
-    return arr.toString() === cids.toString();
-  });
+  const cidSet = cids.map((cid) => cid.toString()).sort().join(",");
+  return list.some(
+    (arr) => cidSet === arr.map((cid) => cid.toString()).sort().join(",")
+  );
 }
-function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
+function uniqueCids(list, remove = new import_cement14.LRUSet()) {
   const byString = /* @__PURE__ */ new Map();
   for (const cid of list) {
     if (remove.has(cid.toString())) continue;
@@ -2680,18 +3097,9 @@ var Loader = class {
   constructor(sthis, ebOpts) {
     this.commitQueue = new CommitQueue();
     this.isCompacting = false;
-    this.carReaders = /* @__PURE__ */ new Map();
-    this.seenCompacted = /* @__PURE__ */ new Set();
-    this.processedCars = /* @__PURE__ */ new Set();
-    this.carLog = [];
-    this.getBlockCache = /* @__PURE__ */ new Map();
-    this.seenMeta = /* @__PURE__ */ new Set();
-    this.writeLimit = (0, import_p_limit.default)(1);
-    this._carStore = new import_cement13.ResolveOnce();
-    this._fileStore = new import_cement13.ResolveOnce();
-    this._WALStore = new import_cement13.ResolveOnce();
-    this._metaStore = new import_cement13.ResolveOnce();
-    this.onceReady = new import_cement13.ResolveOnce();
+    this.maxConcurrentWrite = (0, import_p_limit.default)(1);
+    this.carLog = new CarLog();
+    this.onceReady = new import_cement14.ResolveOnce();
     this.sthis = sthis;
     this.ebOpts = defaultedBlockstoreRuntime(
       sthis,
@@ -2701,76 +3109,75 @@ var Loader = class {
       },
       "Loader"
     );
-    this.logger = this.ebOpts.logger;
-    this.taskManager = new TaskManager(sthis, async (dbMeta) => {
-      await this.handleDbMetasFromStore([dbMeta]);
+    this.logger = ensureLogger(sthis, "Loader");
+    this.cidCache = new import_cement14.KeyedResolvOnce({
+      lru: {
+        maxEntries: parseInt(this.ebOpts.storeUrls.car.getParam(PARAM.CAR_CACHE_SIZE, "1000"), 10)
+      }
     });
-  }
-  async carStore() {
-    return this._carStore.once(
-      async () => this.ebOpts.storeRuntime.makeDataStore({
-        // sthis: this.sthis,
-        gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-        url: this.ebOpts.storeUrls.data,
-        // keybag: await this.keyBag(),
-        loader: this
-      })
-    );
-  }
-  async fileStore() {
-    return this._fileStore.once(
-      async () => this.ebOpts.storeRuntime.makeDataStore({
-        // sthis: this.sthis,
-        gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-        url: this.ebOpts.storeUrls.file,
-        // keybag: await this.keyBag(),
-        loader: this
-      })
-    );
-  }
-  async WALStore() {
-    return this._WALStore.once(
-      async () => this.ebOpts.storeRuntime.makeWALStore({
-        // sthis: this.sthis,
-        gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-        url: this.ebOpts.storeUrls.wal,
-        // keybag: await this.keyBag(),
-        loader: this
-      })
+    this.seenMeta = new import_cement14.LRUSet({
+      maxEntries: parseInt(this.ebOpts.storeUrls.meta.getParam(PARAM.CAR_META_CACHE_SIZE, "1000"), 10)
+    });
+    this.seenCompacted = new import_cement14.LRUSet({
+      maxEntries: parseInt(this.ebOpts.storeUrls.car.getParam(PARAM.CAR_COMPACT_CACHE_SIZE, "1000"), 10)
+    });
+    this.maxConcurrentCarReader = (0, import_p_limit.default)(parseInt(this.ebOpts.storeUrls.car.getParam(PARAM.CAR_PARALLEL, "5"), 10));
+    this.taskManager = new TaskManager(
+      sthis,
+      async (dbMeta, activeStore) => {
+        await this.handleDbMetasFromStore([dbMeta], activeStore);
+      },
+      this.ebOpts.taskManager
     );
+    this.attachedStores = new AttachedRemotesImpl(this);
   }
-  async metaStore() {
-    return this._metaStore.once(
-      async () => this.ebOpts.storeRuntime.makeMetaStore({
-        // sthis: this.sthis,
-        gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-        url: this.ebOpts.storeUrls.meta,
-        // keybag: await this.keyBag(),
-        loader: this
-      })
-    );
+  async attach(attached) {
+    const at = await this.attachedStores.attach(attached);
+    if (!at.stores.wal) {
+      try {
+        const dbMeta = await at.stores.meta.load();
+        if (!Array.isArray(dbMeta)) {
+          throw this.logger.Error().Msg("missing dbMeta").AsError();
+        }
+        await this.handleDbMetasFromStore(dbMeta, this.attachedStores.activate(at.stores));
+      } catch (e) {
+        this.logger.Error().Err(e).Msg("error attaching store");
+        at.detach();
+      }
+    }
+    return at;
   }
   keyBag() {
     return getKeyBag(this.sthis, this.ebOpts.keyBag);
   }
   async ready() {
     return this.onceReady.once(async () => {
-      const metas = await (await this.metaStore()).load();
+      await createAttachedStores(
+        {
+          car: { url: this.ebOpts.storeUrls.car, gatewayInterceptor: this.ebOpts.gatewayInterceptor },
+          file: { url: this.ebOpts.storeUrls.file, gatewayInterceptor: this.ebOpts.gatewayInterceptor },
+          meta: { url: this.ebOpts.storeUrls.meta, gatewayInterceptor: this.ebOpts.gatewayInterceptor },
+          wal: { url: this.ebOpts.storeUrls.wal, gatewayInterceptor: this.ebOpts.gatewayInterceptor }
+        },
+        this.attachedStores
+      );
+      const local = this.attachedStores.local();
+      const metas = await local.active.meta.load();
       if (this.ebOpts.meta) {
-        await this.handleDbMetasFromStore([this.ebOpts.meta]);
+        await this.handleDbMetasFromStore([this.ebOpts.meta, ...metas || []], local);
       } else if (metas) {
-        await this.handleDbMetasFromStore(metas);
+        await this.handleDbMetasFromStore(metas, local);
       }
     });
   }
   async close() {
     await this.commitQueue.waitIdle();
-    const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
-    await Promise.all(toClose.map((store) => store.close()));
+    await this.attachedStores.detach();
   }
   async destroy() {
-    const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
-    await Promise.all(toDestroy.map((store) => store.destroy()));
+    await Promise.all(
+      this.attachedStores.local().baseStores().map((store) => store.destroy())
+    );
   }
   // async snapToCar(carCid: AnyLink | string) {
   //   await this.ready
@@ -2782,28 +3189,38 @@ var Loader = class {
   //   await this.getMoreReaders(carHeader.cars)
   //   await this._applyCarHeader(carHeader, true)
   // }
-  async handleDbMetasFromStore(metas) {
-    this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
+  async handleDbMetasFromStore(metas, activeStore) {
+    this.logger.Debug().Any("metas", metas).Url(activeStore.active.car.url()).Msg("handleDbMetasFromStore");
     for (const meta of metas) {
-      await this.writeLimit(async () => {
-        await this.mergeDbMetaIntoClock(meta);
+      await this.maxConcurrentWrite(async () => {
+        await this.mergeDbMetaIntoClock(meta, activeStore);
       });
     }
   }
-  async mergeDbMetaIntoClock(meta) {
+  async mergeDbMetaIntoClock(meta, activeStore) {
     if (this.isCompacting) {
       throw this.logger.Error().Msg("cannot merge while compacting").AsError();
     }
-    if (this.seenMeta.has(meta.cars.toString())) return;
-    this.seenMeta.add(meta.cars.toString());
-    if (carLogIncludesGroup(this.carLog, meta.cars)) {
-      return;
+    try {
+      this.isCompacting = true;
+      const metaKey = meta.cars.map((i) => i.toString()).sort().join(",");
+      if (this.seenMeta.has(metaKey)) return;
+      this.seenMeta.add(metaKey);
+      if (carLogIncludesGroup(this.carLog.asArray(), meta.cars)) {
+        return;
+      }
+      const carHeader = await this.loadCarHeaderFromMeta(meta, activeStore);
+      carHeader.compact.map((c) => c.toString()).forEach((k) => this.seenCompacted.add(k), this.seenCompacted);
+      try {
+        await this.getMoreReaders(carHeader.cars.flat(), activeStore);
+      } catch (e) {
+        this.logger.Error().Err(e).Msg("error getting more readers");
+      }
+      this.carLog.update(uniqueCids([meta.cars, ...this.carLog.asArray(), ...carHeader.cars], this.seenCompacted));
+      await this.ebOpts.applyMeta?.(carHeader.meta);
+    } finally {
+      this.isCompacting = false;
     }
-    const carHeader = await this.loadCarHeaderFromMeta(meta);
-    carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
-    await this.getMoreReaders(carHeader.cars.flat());
-    this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
-    await this.ebOpts.applyMeta?.(carHeader.meta);
   }
   // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
   //   const { key } = meta;
@@ -2811,8 +3228,8 @@ var Loader = class {
   //     await this.setKey(key);
   //   }
   // }
-  async loadCarHeaderFromMeta({ cars: cids }) {
-    const reader = await this.loadCar(cids[0]);
+  async loadCarHeaderFromMeta(dbm, astore) {
+    const reader = await this.loadCar(dbm.cars[0], astore);
     return await parseCarFile(reader, this.logger);
   }
   // async _getKey(): Promise<string | undefined> {
@@ -2825,22 +3242,22 @@ var Loader = class {
   // }
   async commitFiles(t, done) {
     await this.ready();
-    const fstore = await this.fileStore();
-    const wstore = await this.WALStore();
+    const fstore = this.attachedStores.local().active.file;
+    const wstore = this.attachedStores.local().active.wal;
     return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
   }
-  async loadFileCar(cid) {
-    return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
+  async loadFileCar(cid, store) {
+    return await this.storesLoadCar(cid, store.fileStore());
   }
   async commit(t, done, opts = { noLoader: false, compact: false }) {
     await this.ready();
-    const carStore = await this.carStore();
+    const carStore = this.attachedStores.local().active.car;
     const params = {
       encoder: (await carStore.keyedCrypto()).codec(),
       carLog: this.carLog,
       carStore,
-      WALStore: await this.WALStore(),
-      metaStore: await this.metaStore(),
+      WALStore: this.attachedStores.local().active.wal,
+      metaStore: this.attachedStores.local().active.meta,
       threshold: this.ebOpts.threshold
     };
     return this.commitQueue.enqueue(async () => {
@@ -2850,41 +3267,54 @@ var Loader = class {
       return ret.cgrp;
     });
   }
-  async updateCarLog(cids, fp, compact) {
+  async updateCarLog(cids, cHeader, compact) {
     if (compact) {
-      const previousCompactCid = fp.compact[fp.compact.length - 1];
-      fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
-      this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
-      await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
+      const previousCompactCid = cHeader.compact[cHeader.compact.length - 1];
+      cHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
+      this.carLog.update(uniqueCids([...this.carLog.asArray(), ...cHeader.cars, cids], this.seenCompacted));
+      await this.removeCidsForCompact(previousCompactCid[0], this.attachedStores.local()).catch((e) => e);
     } else {
-      this.carLog.unshift(cids);
+      this.carLog.xunshift(cids);
     }
   }
   async cacheTransaction(t) {
     for await (const block of t.entries()) {
       const sBlock = block.cid.toString();
-      if (!this.getBlockCache.has(sBlock)) {
-        this.getBlockCache.set(sBlock, block);
-      }
-    }
-  }
-  async cacheCarReader(carCidStr, reader) {
-    if (this.processedCars.has(carCidStr)) return;
-    this.processedCars.add(carCidStr);
-    for await (const block of reader.blocks()) {
-      const sBlock = block.cid.toString();
-      if (!this.getBlockCache.has(sBlock)) {
-        this.getBlockCache.set(sBlock, block);
-      }
+      this.cidCache.get(sBlock).once(
+        () => ({
+          type: "block",
+          cid: block.cid,
+          blocks: [block],
+          roots: []
+        })
+      );
     }
   }
-  async removeCidsForCompact(cid) {
-    const carHeader = await this.loadCarHeaderFromMeta({
-      cars: [cid]
-    });
+  // /**
+  //  *
+  //  * @returns the list of blocks which was read from the car file
+  //  */
+  // private async readCar(reader: CarReader): Promise<AnyBlock[]> {
+  //   const blocks: AnyBlock[] = [];
+  //   for await (const block of reader.blocks()) {
+  //     const sBlock = block.cid.toString();
+  //     this.cidCache.get(sBlock).once(() => {
+  //       blocks.push(block);
+  //       return [block];
+  //     });
+  //   }
+  //   return blocks;
+  // }
+  async removeCidsForCompact(cid, store) {
+    const carHeader = await this.loadCarHeaderFromMeta(
+      {
+        cars: [cid]
+      },
+      store
+    );
     for (const cids of carHeader.compact) {
       for (const cid2 of cids) {
-        await (await this.carStore()).remove(cid2);
+        await this.attachedStores.local().active.car.remove(cid2);
       }
     }
   }
@@ -2896,148 +3326,134 @@ var Loader = class {
   //     await this.remoteWAL!.enqueue(dbMeta, { public: false })
   //   }
   // }
-  async *entries(cache2 = true) {
+  async *entries() {
     await this.ready();
-    if (cache2) {
-      for (const [, block] of this.getBlockCache) {
-        yield block;
-      }
-    } else {
-      for (const [, block] of this.getBlockCache) {
-        yield block;
-      }
-      for (const cids of this.carLog) {
-        for (const cid of cids) {
-          const reader = await this.loadCar(cid);
-          if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
-          for await (const block of reader.blocks()) {
-            const sCid = block.cid.toString();
-            if (!this.getBlockCache.has(sCid)) {
-              yield block;
-            }
-          }
+    const seen = /* @__PURE__ */ new Set();
+    for (const carCids of this.carLog.asArray()) {
+      for (const carCid of carCids) {
+        const reader = await this.loadCar(carCid, this.attachedStores.local());
+        if (!reader || reader.type !== "car") {
+          throw this.logger.Error().Any("reader", reader.type).Str("cid", carCid.toString()).Msg("missing car reader").AsError();
+        }
+        for (const block of reader.blocks) {
+          const cidStr = block.cid.toString();
+          if (seen.has(cidStr)) continue;
+          seen.add(cidStr);
+          yield block;
         }
       }
     }
   }
-  async getBlock(cid) {
+  async getBlock(cid, store) {
     await this.ready();
-    const sCid = cid.toString();
-    if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-    const getCarCid = async (carCid) => {
-      if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-      const reader = await this.loadCar(carCid);
-      if (!reader) {
-        throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
-      }
-      await this.cacheCarReader(carCid.toString(), reader).catch(() => {
-        return;
-      });
-      if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-      throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
-    };
-    const getCompactCarCids = async (carCid) => {
-      const reader = await this.loadCar(carCid);
-      if (!reader) {
-        throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
-      }
-      const header = await parseCarFile(reader, this.logger);
-      const compacts = header.compact;
-      let got2;
-      const batchSize2 = 5;
-      for (let i = 0; i < compacts.length; i += batchSize2) {
-        const promises = [];
-        for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
-          for (const cid2 of compacts[j]) {
-            promises.push(getCarCid(cid2));
+    const cidStr = cid.toString();
+    const ci = await this.cidCache.get(cidStr).once(async () => {
+      const getCompactCarCids = async (carCid) => {
+        const sCid = carCid.toString();
+        const reader = await this.loadCar(carCid, store);
+        const header = await parseCarFile(reader, this.logger);
+        const compacts = header.compact;
+        const got2 = await Promise.allSettled(compacts.map((compact) => compact.map((cid2) => this.loadCar(cid2, store)).flat()));
+        got2.filter((result) => result.status === "rejected").forEach((result) => {
+          this.logger.Error().Err(result.reason).Str("cid", sCid).Msg("error getting compacted block");
+        });
+      };
+      let got;
+      for (const carCids of this.carLog.asArray()) {
+        for (const carCid of carCids) {
+          const ci2 = await this.loadCar(carCid, store);
+          if (!ci2) {
+            this.logger.Error().Str("cid", carCid.toString()).Msg("missing CarCID");
+            continue;
+          }
+          got = ci2.blocks.find((block) => block.cid.equals(cid));
+          if (got) {
+            break;
           }
         }
-        try {
-          got2 = await Promise.any(promises);
-        } catch {
-        }
-        if (got2) break;
-      }
-      if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-      throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
-    };
-    let got;
-    const batchSize = 5;
-    for (let i = 0; i < this.carLog.length; i += batchSize) {
-      const batch = this.carLog.slice(i, i + batchSize);
-      const promises = batch.flatMap((slice) => slice.map(getCarCid));
-      try {
-        got = await Promise.any(promises);
-      } catch {
       }
-      if (got) break;
-    }
-    if (!got) {
-      try {
-        got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
-      } catch {
+      if (!got) {
+        await getCompactCarCids(this.carLog.last()[0]);
       }
+      return {
+        type: "block",
+        cid,
+        blocks: got ? [got] : [],
+        roots: []
+      };
+    });
+    if (!(ci.type === "block" && ci.blocks.length === 1)) {
+      throw this.logger.Error().Str("cid", cidStr).Any("block", ci).Msg("missing block").AsError();
     }
-    return got;
+    return ci.blocks[0];
   }
-  async loadCar(cid) {
-    if (!this.carStore) {
-      throw this.logger.Error().Msg("car store not initialized").AsError();
-    }
-    const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
+  async loadCar(cid, store) {
+    const loaded = await this.storesLoadCar(cid, store.carStore());
     return loaded;
   }
-  async makeDecoderAndCarReader(cid, local, remote) {
-    const cidsString = cid.toString();
+  async makeDecoderAndCarReader(carCid, store) {
+    const carCidStr = carCid.toString();
     let loadedCar = void 0;
-    let activeStore = local;
+    let activeStore = store.local();
     try {
-      this.logger.Debug().Any("cid", cidsString).Msg("loading car");
-      loadedCar = await local.load(cid);
+      this.logger.Debug().Any("cid", carCidStr).Msg("loading car");
+      loadedCar = await store.local().load(carCid);
       this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
     } catch (e) {
-      if (remote) {
-        const remoteCar = await remote.load(cid);
-        if (remoteCar) {
-          this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
-          await local.save(remoteCar);
-          loadedCar = remoteCar;
-          activeStore = remote;
+      if (!isNotFoundError(e)) {
+        throw this.logger.Error().Str("cid", carCidStr).Err(e).Msg("loading car");
+      }
+      for (const remote of store.remotes()) {
+        try {
+          const remoteCar = await remote.load(carCid);
+          if (remoteCar) {
+            this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
+            await store.local().save(remoteCar);
+            loadedCar = remoteCar;
+            activeStore = remote;
+            break;
+          } else {
+            this.logger.Error().Str("cid", carCidStr).Err(e).Msg("loading car");
+          }
+        } catch (e2) {
+          this.logger.Warn().Str("cid", carCidStr).Url(remote.url()).Err(e2).Msg("loading car");
         }
-      } else {
-        this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
       }
     }
     if (!loadedCar) {
-      throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
+      throw this.logger.Error().Url(store.local().url()).Str("cid", carCidStr).Msg("missing car files").AsError();
     }
     const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha24.sha256, codec: (await activeStore.keyedCrypto()).codec() });
-    const rawReader = await import_reader.CarReader.fromBytes(bytes.value);
-    const readerP = Promise.resolve(rawReader);
-    const cachedReaderP = readerP.then(async (reader) => {
-      await this.cacheCarReader(cidsString, reader).catch((e) => {
-        this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
-        return;
-      });
-      return reader;
-    });
-    this.carReaders.set(cidsString, cachedReaderP);
-    return readerP;
+    const rawReader = await import_reader.CarReader.fromBytes(bytes.value.data);
+    const blocks = [];
+    for await (const block of rawReader.blocks()) {
+      const sBlock = block.cid.toString();
+      blocks.push(block);
+      this.cidCache.get(sBlock).once(() => ({
+        type: "block",
+        cid: block.cid,
+        blocks: [block],
+        roots: []
+      }));
+    }
+    return {
+      type: "car",
+      cid: carCid,
+      blocks,
+      roots: await rawReader.getRoots()
+    };
   }
   //What if instead it returns an Array of CarHeader
-  async storesLoadCar(cid, local, remote) {
-    const cidsString = cid.toString();
-    let dacr = this.carReaders.get(cidsString);
-    if (!dacr) {
-      dacr = this.makeDecoderAndCarReader(cid, local, remote);
-      this.carReaders.set(cidsString, dacr);
-    }
-    return dacr;
+  async storesLoadCar(carCid, store) {
+    const carCidStr = carCid.toString();
+    return this.cidCache.get(carCidStr).once(async () => {
+      return this.maxConcurrentCarReader(() => this.makeDecoderAndCarReader(carCid, store));
+    });
   }
-  async getMoreReaders(cids) {
-    const limit = (0, import_p_limit.default)(5);
-    const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
-    await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid))));
+  async getMoreReaders(cids, store) {
+    for (const cid of cids) {
+      await this.loadCar(cid, store);
+    }
   }
 };
 
@@ -3082,12 +3498,21 @@ function getGenerateIVFn(url, opts) {
 }
 var BlockIvKeyIdCodec = class {
   constructor(ko, iv, opts) {
-    this.code = 3147065;
+    this.code = 24;
     this.name = "Fireproof@encrypted-block:aes-gcm";
     this.ko = ko;
     this.iv = iv;
     this.opts = opts || {};
   }
+  // hashAsBytes(data: IvKeyIdData): AsyncHashAsBytes<Uint8Array<ArrayBufferLike>> {
+  //   return data;
+  // }
+  valueToHashBytes(value) {
+    return Promise.resolve(value.data);
+  }
+  bytesToHash(data) {
+    return Promise.resolve(data);
+  }
   async encode(data) {
     const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
     const { iv } = this.ko.algo(calcIv);
@@ -3119,11 +3544,16 @@ var BlockIvKeyIdCodec = class {
     if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
       throw this.ko.logger.Error().Msg("iv missmatch").AsError();
     }
-    return result;
+    return {
+      iv,
+      keyId,
+      data: result
+    };
   }
 };
 var cryptoAction = class {
   constructor(url, key, cyopt, sthis) {
+    this.code = 24;
     this.ivLength = 12;
     this.isEncrypting = true;
     this.logger = ensureLogger(sthis, "cryptoAction");
@@ -3157,14 +3587,19 @@ var cryptoAction = class {
 };
 var nullCodec = class {
   constructor() {
-    this.code = 0;
+    this.code = 24;
     this.name = "Fireproof@unencrypted-block";
+    this.empty = new Uint8Array();
   }
-  encode(data) {
+  async encode(data) {
     return data;
   }
-  decode(data) {
-    return data;
+  async decode(data) {
+    return {
+      iv: this.empty,
+      keyId: this.empty,
+      data
+    };
   }
 };
 var noCrypto = class {
@@ -3232,39 +3667,39 @@ var import_p_retry = __toESM(require("p-retry"), 1);
 var import_p_map = __toESM(require("p-map"), 1);
 
 // src/blockstore/interceptor-gateway.ts
-var import_cement14 = require("@adviser/cement");
+var import_cement15 = require("@adviser/cement");
 var PassThroughGateway = class {
   async buildUrl(ctx, url, key) {
     const op = { url, key };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async start(ctx, url) {
     const op = { url };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async close(ctx, url) {
     const op = { url };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async delete(ctx, url) {
     const op = { url };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async destroy(ctx, url) {
     const op = { url };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async put(ctx, url, body) {
     const op = { url, body };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async get(ctx, url) {
     const op = { url };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
   async subscribe(ctx, url, callback) {
     const op = { url, callback };
-    return import_cement14.Result.Ok({ op });
+    return import_cement15.Result.Ok({ op });
   }
 };
 var passThrougthGateway = new PassThroughGateway();
@@ -3276,7 +3711,7 @@ var InterceptorGateway = class {
   async buildUrl(ctx, baseUrl, key) {
     const rret = await this.interceptor.buildUrl(ctx, baseUrl, key);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3287,7 +3722,7 @@ var InterceptorGateway = class {
   async destroy(ctx, iurl) {
     const rret = await this.interceptor.destroy(ctx, iurl);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3298,7 +3733,7 @@ var InterceptorGateway = class {
   async start(ctx, url) {
     const rret = await this.interceptor.start(ctx, url);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3309,7 +3744,7 @@ var InterceptorGateway = class {
   async close(ctx, url) {
     const rret = await this.interceptor.close(ctx, url);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3320,7 +3755,7 @@ var InterceptorGateway = class {
   async put(ctx, url, fpEnv) {
     const rret = await this.interceptor.put(ctx, url, fpEnv);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3331,7 +3766,7 @@ var InterceptorGateway = class {
   async get(ctx, url) {
     const rret = await this.interceptor.get(ctx, url);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3341,11 +3776,11 @@ var InterceptorGateway = class {
   }
   async subscribe(ctx, url, callback) {
     if (!this.innerGW.subscribe) {
-      return import_cement14.Result.Err(ctx.loader.sthis.logger.Error().Url(url).Msg("subscribe not supported").AsError());
+      return import_cement15.Result.Err(ctx.loader.sthis.logger.Error().Url(url).Msg("subscribe not supported").AsError());
     }
     const rret = await this.interceptor.subscribe(ctx, url, callback);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3356,7 +3791,7 @@ var InterceptorGateway = class {
   async delete(ctx, url) {
     const rret = await this.interceptor.delete(ctx, url);
     if (rret.isErr()) {
-      return import_cement14.Result.Err(rret.Err());
+      return import_cement15.Result.Err(rret.Err());
     }
     const ret = rret.unwrap();
     if (ret.stop && ret.value) {
@@ -3372,9 +3807,9 @@ var InterceptorGateway = class {
 // src/blockstore/store.ts
 function guardVersion(url) {
   if (!url.hasParam("version")) {
-    return import_cement15.Result.Err(`missing version: ${url.toString()}`);
+    return import_cement16.Result.Err(`missing version: ${url.toString()}`);
   }
-  return import_cement15.Result.Ok(url);
+  return import_cement16.Result.Ok(url);
 }
 var BaseStoreImpl = class {
   // readonly loader: Loadable;
@@ -3408,7 +3843,7 @@ var BaseStoreImpl = class {
   async keyedCrypto() {
     return keyedCryptoFactory(this._url, await this.loader.keyBag(), this.sthis);
   }
-  async start() {
+  async start(dam) {
     this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
     this._url = this._url.build().setParam(PARAM.STORE, this.storeType).URI();
     const res = await this.gateway.start({ loader: this.loader }, this._url);
@@ -3419,13 +3854,8 @@ var BaseStoreImpl = class {
     this._url = res.Ok();
     const kb = await this.loader.keyBag();
     const skRes = await kb.ensureKeyFromUrl(this._url, () => {
-      const idx = this._url.getParam(PARAM.INDEX);
-      const storeKeyName = [this.url().getParam(PARAM.NAME)];
-      if (idx) {
-        storeKeyName.push(idx);
-      }
-      storeKeyName.push(this.storeType);
-      return storeKeyName.join(":");
+      const key = this._url.getParam(PARAM.KEY);
+      return key;
     });
     if (skRes.isErr()) {
       return skRes;
@@ -3439,13 +3869,13 @@ var BaseStoreImpl = class {
     }
     if (this.ready) {
       const fn = this.ready.bind(this);
-      const ready = await (0, import_cement15.exception2Result)(fn);
+      const ready = await (0, import_cement16.exception2Result)(fn);
       if (ready.isErr()) {
         await this.close();
         return ready;
       }
     }
-    this._onStarted.forEach((fn) => fn());
+    this._onStarted.forEach((fn) => fn(dam));
     this.logger.Debug().Msg("started");
     return version;
   }
@@ -3474,12 +3904,19 @@ var MetaStoreImpl = class extends BaseStoreImpl {
       /*this.remote && */
       opts.gateway.subscribe
     ) {
-      this.onStarted(async () => {
+      this.onStarted(async (dam) => {
         this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
         opts.gateway.subscribe({ loader: this.loader }, this.url(), async ({ payload: dbMetas }) => {
           this.logger.Debug().Msg("Received message from gateway");
           await Promise.all(
-            dbMetas.map((dbMeta) => this.loader.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
+            dbMetas.map(
+              (dbMetaEv) => this.loader.taskManager?.handleEvent(
+                dbMetaEv.eventCid,
+                dbMetaEv.parents,
+                dbMetaEv.dbMeta,
+                this.loader.attachedStores.activate(dam)
+              )
+            )
           );
           this.updateParentsFromDbMetas(dbMetas);
         });
@@ -3501,8 +3938,7 @@ var MetaStoreImpl = class extends BaseStoreImpl {
   //   }
   //   return (rDbMeta.Ok() as FPEnvelopeMeta).payload;
   // }
-  async load() {
-    const branch = "main";
+  async load(branch = "main") {
     const url = await this.gateway.buildUrl({ loader: this.loader }, this.url(), branch);
     if (url.isErr()) {
       throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
@@ -3514,10 +3950,11 @@ var MetaStoreImpl = class extends BaseStoreImpl {
       }
       throw this.logger.Error().Url(url.Ok()).Err(rfpEnv).Msg("gateway get").AsError();
     }
-    const dbMetas = rfpEnv.Ok().payload;
-    await this.loader.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
-    this.updateParentsFromDbMetas(dbMetas);
-    return dbMetas.map((m) => m.dbMeta);
+    const fpMeta = rfpEnv.Ok().payload;
+    const dbMetas = fpMeta.map((m) => m.dbMeta);
+    await this.loader.handleDbMetasFromStore(dbMetas, this.loader.attachedStores.activate(url.Ok()));
+    this.updateParentsFromDbMetas(fpMeta);
+    return dbMetas;
   }
   async save(meta, branch) {
     branch = branch || "main";
@@ -3539,7 +3976,7 @@ var MetaStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close({ loader: this.loader }, this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement15.Result.Ok(void 0);
+    return import_cement16.Result.Ok(void 0);
   }
   async destroy() {
     this.logger.Debug().Msg("destroy");
@@ -3547,9 +3984,8 @@ var MetaStoreImpl = class extends BaseStoreImpl {
   }
 };
 var DataStoreImpl = class extends BaseStoreImpl {
-  constructor(sthis, url, opts) {
-    super(sthis, url, { ...opts }, ensureLogger(sthis, "DataStoreImpl"));
-    this.storeType = "data";
+  constructor(sthis, url, opts, logger) {
+    super(sthis, url, { ...opts }, logger);
   }
   async load(cid) {
     this.logger.Debug().Any("cid", cid).Msg("loading");
@@ -3564,7 +4000,6 @@ var DataStoreImpl = class extends BaseStoreImpl {
     const fpenv = res.Ok();
     switch (fpenv.type) {
       case "car":
-        return { cid, bytes: fpenv.payload };
       case "file":
         return { cid, bytes: fpenv.payload };
       default:
@@ -3580,12 +4015,11 @@ var DataStoreImpl = class extends BaseStoreImpl {
     }
     let fpMsg;
     switch (url.Ok().getParam(PARAM.STORE)) {
-      case "data":
-        if (url.Ok().getParam(PARAM.SUFFIX)) {
-          fpMsg = Car2FPMsg(car.bytes);
-        } else {
-          fpMsg = File2FPMsg(car.bytes);
-        }
+      case "car":
+        fpMsg = Car2FPMsg(car.bytes);
+        break;
+      case "file":
+        fpMsg = File2FPMsg(car.bytes);
         break;
       default:
         throw this.logger.Error().Str("store", url.Ok().getParam(PARAM.STORE)).Msg("unexpected store").AsError();
@@ -3609,20 +4043,32 @@ var DataStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close({ loader: this.loader }, this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement15.Result.Ok(void 0);
+    return import_cement16.Result.Ok(void 0);
   }
   destroy() {
     this.logger.Debug().Msg("destroy");
     return this.gateway.destroy({ loader: this.loader }, this.url());
   }
 };
+var CarStoreImpl = class extends DataStoreImpl {
+  constructor(sthis, url, opts) {
+    super(sthis, url, { ...opts }, ensureLogger(sthis, "CarStoreImpl"));
+    this.storeType = "car";
+  }
+};
+var FileStoreImpl = class extends DataStoreImpl {
+  constructor(sthis, url, opts) {
+    super(sthis, url, { ...opts }, ensureLogger(sthis, "FileStoreImpl"));
+    this.storeType = "file";
+  }
+};
 var WALStoreImpl = class extends BaseStoreImpl {
   constructor(sthis, url, opts) {
     super(sthis, url, { ...opts }, ensureLogger(sthis, "WALStoreImpl"));
     this.storeType = "wal";
     // readonly tag: string = "rwal-base";
     // readonly loader: Loadable;
-    this._ready = new import_cement15.ResolveOnce();
+    this._ready = new import_cement16.ResolveOnce();
     this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
     this.processing = void 0;
     this.processQueue = new CommitQueue();
@@ -3663,7 +4109,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
   }
   async process() {
     await this.ready();
-    if (!this.loader.remoteCarStore) return;
     await this.processQueue.enqueue(async () => {
       try {
         await this._doProcess();
@@ -3677,7 +4122,6 @@ var WALStoreImpl = class extends BaseStoreImpl {
   }
   async _doProcess() {
     if (!this.loader) return;
-    if (!this.loader.remoteCarStore) return;
     const operations = [...this.walState.operations];
     const noLoaderOps = [...this.walState.noLoaderOps];
     const fileOperations = [...this.walState.fileOperations];
@@ -3698,13 +4142,13 @@ var WALStoreImpl = class extends BaseStoreImpl {
               return;
             }
             for (const cid of dbMeta.cars) {
-              const car = await (await this.loader.carStore()).load(cid);
+              const car = await this.loader.attachedStores.local().active.car.load(cid);
               if (!car) {
-                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                if (carLogIncludesGroup(this.loader.carLog.asArray(), dbMeta.cars)) {
                   throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
                 }
               } else {
-                await throwFalsy(this.loader.remoteCarStore).save(car);
+                await this.loader.attachedStores.forRemotes((x) => x.active.car.save(car));
               }
             }
             inplaceFilter(this.walState.noLoaderOps, (op) => op !== dbMeta);
@@ -3720,13 +4164,13 @@ var WALStoreImpl = class extends BaseStoreImpl {
               return;
             }
             for (const cid of dbMeta.cars) {
-              const car = await (await this.loader.carStore()).load(cid);
+              const car = await this.loader.attachedStores.local().active.car.load(cid);
               if (!car) {
-                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                if (carLogIncludesGroup(this.loader.carLog.asArray(), dbMeta.cars)) {
                   throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
                 }
               } else {
-                await throwFalsy(this.loader.remoteCarStore).save(car);
+                await this.loader.attachedStores.forRemotes((x) => x.active.car.save(car));
               }
             }
             inplaceFilter(this.walState.operations, (op) => op !== dbMeta);
@@ -3741,11 +4185,11 @@ var WALStoreImpl = class extends BaseStoreImpl {
             if (!this.loader) {
               return;
             }
-            const fileBlock = await (await this.loader.fileStore()).load(fileCid);
+            const fileBlock = await this.loader.attachedStores.local().active.file.load(fileCid);
             if (!fileBlock) {
               throw this.logger.Error().Ref("cid", fileCid).Msg("missing file block").AsError();
             }
-            await this.loader.remoteFileStore?.save(fileBlock, { public: publicFile });
+            await this.loader.attachedStores.forRemotes((x) => x.active.file.save(fileBlock, { public: publicFile }));
             inplaceFilter(this.walState.fileOperations, (op) => op.cid !== fileCid);
           }, `fileOperation with cid=${fileCid.toString()}`);
         },
@@ -3757,7 +4201,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
           if (!this.loader) {
             return;
           }
-          await this.loader.remoteMetaStore?.save(lastOp);
+          await this.loader.attachedStores.forRemotes((x) => x.active.meta.save(lastOp));
         }, `remoteMetaStore save with dbMeta.cars=${lastOp.cars.toString()}`);
       }
     } catch (error) {
@@ -3801,7 +4245,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close({ loader: this.loader }, this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement15.Result.Ok(void 0);
+    return import_cement16.Result.Ok(void 0);
   }
   destroy() {
     this.logger.Debug().Msg("destroy");
@@ -3810,71 +4254,86 @@ var WALStoreImpl = class extends BaseStoreImpl {
 };
 
 // src/blockstore/store-factory.ts
-var onceGateway = new import_cement16.KeyedResolvOnce();
-var gatewayInstances = new import_cement16.KeyedResolvOnce();
+var onceGateway = new import_cement17.KeyedResolvOnce();
+var gatewayInstances = new import_cement17.KeyedResolvOnce();
 async function getStartedGateway(ctx, url) {
   return onceGateway.get(url.toString()).once(async () => {
     const item = getGatewayFactoryItem(url.protocol);
     if (item) {
       const ret = {
         url,
-        ...await gatewayInstances.get(url.protocol).once(async () => ({})),
-        gateway: await item.serdegateway(ctx.loader.sthis)
+        ...await gatewayInstances.get(url.protocol).once(async () => ({
+          gateway: await item.serdegateway(ctx.loader.sthis)
+        }))
       };
       const res = await ret.gateway.start(ctx, url);
       if (res.isErr()) {
-        return import_cement16.Result.Err(ctx.loader.sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
+        return import_cement17.Result.Err(ctx.loader.sthis.logger.Error().Result("start", res).Msg("start failed").AsError());
       }
       ret.url = res.Ok();
-      return import_cement16.Result.Ok(ret);
+      return import_cement17.Result.Ok(ret);
     }
-    return import_cement16.Result.Err(ctx.loader.sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
+    return import_cement17.Result.Err(ctx.loader.sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
+  });
+}
+async function carStoreFactory(ctx, uai) {
+  const storeUrl = uai.url.build().setParam(PARAM.STORE, "car").URI();
+  const rgateway = await getStartedGateway(ctx, storeUrl);
+  if (rgateway.isErr()) {
+    throw ctx.loader.sthis.logger.Error().Result("err", rgateway).Url(uai.url).Msg("notfound").AsError();
+  }
+  const gateway = rgateway.Ok();
+  const store = new CarStoreImpl(ctx.loader.sthis, gateway.url, {
+    gateway: gateway.gateway,
+    gatewayInterceptor: uai.gatewayInterceptor,
+    loader: ctx.loader
   });
+  return store;
 }
-async function dataStoreFactory(sfi) {
-  const storeUrl = sfi.url.build().setParam(PARAM.STORE, "data").URI();
-  const rgateway = await getStartedGateway(sfi, storeUrl);
+async function fileStoreFactory(ctx, uai) {
+  const storeUrl = uai.url.build().setParam(PARAM.STORE, "file").URI();
+  const rgateway = await getStartedGateway(ctx, storeUrl);
   if (rgateway.isErr()) {
-    throw sfi.loader.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
+    throw ctx.loader.sthis.logger.Error().Result("err", rgateway).Url(uai.url).Msg("notfound").AsError();
   }
   const gateway = rgateway.Ok();
-  const store = new DataStoreImpl(sfi.loader.sthis, gateway.url, {
+  const store = new FileStoreImpl(ctx.loader.sthis, gateway.url, {
     gateway: gateway.gateway,
-    gatewayInterceptor: sfi.gatewayInterceptor,
-    loader: sfi.loader
+    gatewayInterceptor: uai.gatewayInterceptor,
+    loader: ctx.loader
   });
   return store;
 }
-async function metaStoreFactory(sfi) {
-  const storeUrl = sfi.url.build().setParam(PARAM.STORE, "meta").URI();
-  const rgateway = await getStartedGateway(sfi, storeUrl);
+async function metaStoreFactory(ctx, uai) {
+  const storeUrl = uai.url.build().setParam(PARAM.STORE, "meta").URI();
+  const rgateway = await getStartedGateway(ctx, storeUrl);
   if (rgateway.isErr()) {
-    throw sfi.loader.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
+    throw ctx.loader.sthis.logger.Error().Result("err", rgateway).Url(uai.url).Msg("notfound").AsError();
   }
   const gateway = rgateway.Ok();
-  const store = new MetaStoreImpl(sfi.loader.sthis, gateway.url, {
+  const store = new MetaStoreImpl(ctx.loader.sthis, gateway.url, {
     gateway: gateway.gateway,
-    gatewayInterceptor: sfi.gatewayInterceptor,
-    loader: sfi.loader
+    gatewayInterceptor: uai.gatewayInterceptor,
+    loader: ctx.loader
   });
   return store;
 }
-async function WALStoreFactory(sfi) {
-  const storeUrl = sfi.url.build().setParam(PARAM.STORE, "wal").URI();
-  const rgateway = await getStartedGateway(sfi, storeUrl);
+async function WALStoreFactory(ctx, uai) {
+  const storeUrl = uai.url.build().setParam(PARAM.STORE, "wal").URI();
+  const rgateway = await getStartedGateway(ctx, storeUrl);
   if (rgateway.isErr()) {
-    throw sfi.loader.sthis.logger.Error().Result("err", rgateway).Url(sfi.url).Msg("notfound").AsError();
+    throw ctx.loader.sthis.logger.Error().Result("err", rgateway).Url(uai.url).Msg("notfound").AsError();
   }
   const gateway = rgateway.Ok();
-  const store = new WALStoreImpl(sfi.loader.sthis, gateway.url, {
+  const store = new WALStoreImpl(ctx.loader.sthis, gateway.url, {
     gateway: gateway.gateway,
-    gatewayInterceptor: sfi.gatewayInterceptor,
-    loader: sfi.loader
+    gatewayInterceptor: uai.gatewayInterceptor,
+    loader: ctx.loader
   });
   return store;
 }
-async function ensureStart(store) {
-  const ret = await store.start();
+async function ensureStart(store, damaw) {
+  const ret = await store.start(damaw);
   if (ret.isErr()) {
     throw store.logger.Error().Result("start", ret).Msg("start failed").AsError();
   }
@@ -3890,191 +4349,32 @@ function ensureStoreEnDeFile(ende) {
 }
 function toStoreRuntime(sthis, endeOpts = {}) {
   return {
-    makeMetaStore: async (sfi) => ensureStart(await metaStoreFactory(sfi)),
-    // async (loader: Loadable) => {
-    //   logger
-    //     .Debug()
-    //     .Str("fromOpts", "" + !!endeOpts.func?.makeMetaStore)
-    //     .Msg("makeMetaStore");
-    //   return ensureStart(await (endeOpts.func?.makeMetaStore || metaStoreFactory)(loader), logger);
-    // },
-    makeDataStore: async (sfi) => ensureStart(await dataStoreFactory(sfi)),
-    // async (loader: Loadable) => {
-    //   logger
-    //     .Debug()
-    //     .Str("fromOpts", "" + !!endeOpts.func?.makeDataStore)
-    //     .Msg("makeDataStore");
-    //   return ensureStart(await (endeOpts.func?.makeDataStore || dataStoreFactory)(loader), logger);
-    // },
-    makeWALStore: async (sfi) => ensureStart(await WALStoreFactory(sfi)),
-    // async (loader: Loadable) => {
-    //   logger
-    //     .Debug()
-    //     .Str("fromOpts", "" + !!endeOpts.func?.makeWALStore)
-    //     .Msg("makeRemoteWAL");
-    //   return ensureStart(await (endeOpts.func?.makeWALStore || remoteWalFactory)(loader), logger);
-    // },
+    makeStores: async (sfi) => {
+      const ctx = {
+        loader: sfi.loader
+      };
+      const storeSet = {};
+      storeSet.meta = await metaStoreFactory(ctx, sfi.byStore.meta);
+      storeSet.car = await carStoreFactory(ctx, sfi.byStore.car);
+      storeSet.file = await fileStoreFactory(ctx, sfi.byStore.file);
+      if (sfi.byStore.wal) {
+        storeSet.wal = await WALStoreFactory(ctx, sfi.byStore.wal);
+      }
+      await ensureStart(storeSet.meta, storeSet);
+      await ensureStart(storeSet.car, storeSet);
+      await ensureStart(storeSet.file, storeSet);
+      if (storeSet.wal) {
+        await ensureStart(storeSet.wal, storeSet);
+      }
+      return storeSet;
+    },
+    // makeMetaStore: async (sfi: StoreFactoryItem) => ensureStart(await metaStoreFactory(sfi)),
+    // makeDataStore: async (sfi: StoreFactoryItem) => ensureStart(await dataStoreFactory(sfi)),
+    // makeWALStore: async (sfi: StoreFactoryItem) => ensureStart(await WALStoreFactory(sfi)),
     ...ensureStoreEnDeFile(endeOpts)
   };
 }
 
-// src/blockstore/connection-base.ts
-var import_cement17 = require("@adviser/cement");
-
-// src/blockstore/store-remote.ts
-async function RemoteDataStore(sthis, url, opts) {
-  const ds = new DataStoreImpl(sthis, url, opts);
-  await ds.start();
-  return ds;
-}
-async function RemoteMetaStore(sthis, url, opts) {
-  const ms = new MetaStoreImpl(sthis, url, opts);
-  await ms.start();
-  return ms;
-}
-
-// src/context.ts
-var Context = class {
-  constructor() {
-    this.ctx = /* @__PURE__ */ new Map();
-  }
-  set(key, value) {
-    this.ctx.set(key, value);
-  }
-  get(key) {
-    return this.ctx.get(key);
-  }
-  delete(key) {
-    this.ctx.delete(key);
-  }
-};
-
-// src/blockstore/connection-base.ts
-function coerceLoader(ref) {
-  const refl = ref;
-  if (refl.loader) {
-    return refl.loader;
-  }
-  const refb = ref;
-  if (refb.blockstore) {
-    return coerceLoader(refb.blockstore);
-  }
-  return void 0;
-}
-var ConnectionBase = class {
-  constructor(url, logger) {
-    // loaded: Promise<void> = Promise.resolve();
-    this.context = new Context();
-    this._loaded = /* @__PURE__ */ new Set();
-    this._metaIsLoading = false;
-    this.logger = logger;
-    this.url = url;
-  }
-  loaded() {
-    const f = new import_cement17.Future();
-    if (!this._metaIsLoading) {
-      f.resolve();
-    } else {
-      this._loaded.add(f);
-    }
-    return f;
-  }
-  async refresh() {
-    await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
-    await (await throwFalsy(this.loader).WALStore()).process();
-  }
-  async connect(refl) {
-    await this.connectMeta(refl);
-    await this.connectStorage(refl);
-  }
-  async connectMeta(refl) {
-    const loader = coerceLoader(refl);
-    if (!loader) throw this.logger.Error().Msg("connectMeta: loader is required").AsError();
-    this.loader = loader;
-    await this.onConnect();
-    const metaUrl = this.url.build().defParam(PARAM.STORE, "meta").URI();
-    const rgateway = await getStartedGateway({ loader }, metaUrl);
-    if (rgateway.isErr())
-      throw this.logger.Error().Result("err", rgateway).Url(metaUrl).Msg("connectMeta: gateway is required").AsError();
-    const dbName = metaUrl.getParam(PARAM.NAME);
-    if (!dbName) {
-      throw this.logger.Error().Url(metaUrl).Msg("connectMeta: dbName is required").AsError();
-    }
-    const gateway = rgateway.Ok();
-    const remote = await RemoteMetaStore(loader.sthis, metaUrl, {
-      gateway: gateway.gateway,
-      loader
-    });
-    this.loader.remoteMetaStore = remote;
-    this._metaIsLoading = true;
-    this.loader.ready().then(async () => {
-      return remote.load().then(async () => {
-        const res = await (0, import_cement17.exception2Result)(async () => {
-          return await (await throwFalsy(this.loader).WALStore()).process();
-        });
-        this._metaIsLoading = false;
-        for (const f of this._loaded) {
-          if (res.isErr()) {
-            f.reject(res.Err());
-          } else {
-            f.resolve();
-          }
-        }
-        this._loaded.clear();
-      });
-    });
-  }
-  async connectStorage(refl) {
-    const loader = coerceLoader(refl);
-    if (!loader) throw this.logger.Error().Msg("connectStorage: loader is required").AsError();
-    this.loader = loader;
-    const dataUrl = this.url.build().defParam(PARAM.STORE, "data").URI();
-    const rgateway = await getStartedGateway({ loader }, dataUrl);
-    if (rgateway.isErr())
-      throw this.logger.Error().Result("err", rgateway).Url(dataUrl).Msg("connectStorage: gateway is required").AsError();
-    const name = dataUrl.getParam(PARAM.NAME);
-    if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage: name is required").AsError;
-    loader.remoteCarStore = await RemoteDataStore(loader.sthis, this.url, {
-      gateway: rgateway.Ok().gateway,
-      loader
-    });
-    loader.remoteFileStore = loader.remoteCarStore;
-  }
-  // move this stuff to connect
-  // async getDashboardURL(compact = true) {
-  //   const baseUrl = 'https://dashboard.fireproof.storage/'
-  //   if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
-  //   // if (compact) {
-  //   //   await this.compact()
-  //   // }
-  //   const currents = await this.loader?.metaStore?.load()
-  //   if (!currents) throw new Error("Can't sync empty ledger: save data first")
-  //   if (currents.length > 1)
-  //     throw new Error("Can't sync ledger with split heads: make an update first")
-  //   const current = currents[0]
-  //   const params = {
-  //     car: current.car.toString()
-  //   }
-  //   if (current.key) {
-  //     // @ts-ignore
-  //     params.key = current.key.toString()
-  //   }
-  //   // @ts-ignore
-  //   if (this.name) {
-  //     // @ts-ignore
-  //     params.name = this.name
-  //   }
-  //   const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
-  //   console.log('Import to dashboard: ' + url.toString())
-  //   return url
-  // }
-  // openDashboard() {
-  //   void this.getDashboardURL().then(url => {
-  //     if (url) window.open(url.toString(), '_blank')
-  //   })
-  // }
-};
-
 // src/crdt-helpers.ts
 var import_link2 = require("multiformats/link");
 var import_sha26 = require("multiformats/hashes/sha2");
@@ -4095,6 +4395,34 @@ function toString(key, logger) {
       throw logger.Error().Msg("Invalid key type").AsError();
   }
 }
+function sanitizeDocumentFields(obj) {
+  if (Array.isArray(obj)) {
+    return obj.map((item) => {
+      if (typeof item === "object" && item !== null) {
+        return sanitizeDocumentFields(item);
+      }
+      return item;
+    });
+  } else if (typeof obj === "object" && obj !== null) {
+    const typedObj = obj;
+    const result = {};
+    for (const key in typedObj) {
+      if (Object.hasOwnProperty.call(typedObj, key)) {
+        const value = typedObj[key];
+        if (value === null || !Number.isNaN(value) && value !== void 0) {
+          if (typeof value === "object" && !key.startsWith("_")) {
+            const sanitized = sanitizeDocumentFields(value);
+            result[key] = sanitized;
+          } else {
+            result[key] = value;
+          }
+        }
+      }
+    }
+    return result;
+  }
+  return obj;
+}
 async function applyBulkUpdateToCrdt(store, tblocks, head, updates, logger) {
   let result = null;
   if (updates.length > 1) {
@@ -4184,7 +4512,8 @@ async function getValueFromCrdt(blocks, head, key, logger) {
   if (!head.length) throw logger.Debug().Msg("Getting from an empty ledger").AsError();
   const link = await (0, import_crdt.get)(blocks, head, key);
   if (!link) throw logger.Error().Str("key", key).Msg(`Missing key`).AsError();
-  return await getValueFromLink(blocks, link, logger);
+  const ret = await getValueFromLink(blocks, link, logger);
+  return ret;
 }
 function readFiles(blocks, { doc }) {
   if (!doc) return;
@@ -4282,6 +4611,9 @@ async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], k
       const { key, value } = ops[i];
       if (!keys.has(key)) {
         const docValue = await getValueFromLink(blocks, value, logger);
+        if (key === PARAM.GENESIS_CID) {
+          continue;
+        }
         updates.push({ id: key, value: docValue.doc, del: docValue.del, clock: link });
         limit--;
         keys.add(key);
@@ -4295,8 +4627,10 @@ async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], k
 }
 async function* getAllEntries(blocks, head, logger) {
   for await (const [key, link] of (0, import_crdt.entries)(blocks, head)) {
-    const docValue = await getValueFromLink(blocks, link, logger);
-    yield { id: key, value: docValue.doc, del: docValue.del };
+    if (key !== PARAM.GENESIS_CID) {
+      const docValue = await getValueFromLink(blocks, link, logger);
+      yield { id: key, value: docValue.doc, del: docValue.del };
+    }
   }
 }
 async function* clockVis(blocks, head) {
@@ -4441,6 +4775,10 @@ var CRDTClockImpl = class {
     this.notifyWatchers(internalUpdates || []);
   }
   notifyWatchers(updates) {
+    updates = updates.filter((update) => update.id !== PARAM.GENESIS_CID);
+    if (!updates.length) {
+      return;
+    }
     this.emptyWatchers.forEach((fn) => fn());
     this.watchers.forEach((fn) => fn(updates || []));
   }
@@ -4520,7 +4858,7 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
     try {
       head = await (0, import_clock4.advance)(tblocks, head, cid);
     } catch (e) {
-      logger.Debug().Err(e).Msg("failed to advance head");
+      logger.Error().Err(e).Msg("failed to advance head");
       continue;
     }
   }
@@ -4579,6 +4917,17 @@ var CRDTImpl = class {
   }
   async bulk(updates) {
     await this.ready();
+    updates = updates.map((dupdate) => ({
+      ...dupdate,
+      value: sanitizeDocumentFields(dupdate.value)
+    }));
+    if (this.clock.head.length === 0) {
+      const value = { id: PARAM.GENESIS_CID, value: { _id: PARAM.GENESIS_CID } };
+      await this._bulk([value]);
+    }
+    return await this._bulk(updates);
+  }
+  async _bulk(updates) {
     const prevHead = [...this.clock.head];
     const done = await this.blockstore.transaction(async (blocks) => {
       const { head } = await applyBulkUpdateToCrdt(
@@ -4649,6 +4998,22 @@ var CRDTImpl = class {
   }
 };
 
+// src/context.ts
+var Context = class {
+  constructor() {
+    this.ctx = /* @__PURE__ */ new Map();
+  }
+  set(key, value) {
+    this.ctx.set(key, value);
+  }
+  get(key) {
+    return this.ctx.get(key);
+  }
+  delete(key) {
+    this.ctx.delete(key);
+  }
+};
+
 // src/ledger.ts
 var ledgers = new import_cement20.KeyedResolvOnce();
 function keyConfigOpts(sthis, name, opts) {
@@ -4693,6 +5058,9 @@ var LedgerShell = class {
     this.name = ref.name;
     ref.addShell(this);
   }
+  attach(a) {
+    return this.ref.attach(a);
+  }
   get opts() {
     return this.ref.opts;
   }
@@ -4791,11 +5159,16 @@ var LedgerImpl = class {
     });
     return ret;
   }
+  async attach(a) {
+    await this.ready();
+    return this.crdt.blockstore.loader.attach(a);
+  }
   // readonly _asDb = new ResolveOnce<Database>();
   // asDB(): Database {
   //   return this._asDb.once(() => new DatabaseImpl(this));
   // }
   subscribe(listener, updates) {
+    this.ready();
     this.logger.Debug().Bool("updates", updates).Msg("subscribe");
     if (updates) {
       if (!this._listening) {
@@ -4837,26 +5210,6 @@ var LedgerImpl = class {
     }
   }
 };
-function defaultURI2(sthis, name, curi, uri, store, ctx) {
-  ctx = ctx || {};
-  const ret = (curi ? import_cement20.URI.from(curi) : uri).build().setParam(PARAM.STORE, store).defParam(PARAM.NAME, name);
-  if (!ret.hasParam(PARAM.NAME)) {
-    throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Ledger name is required").AsError();
-  }
-  if (ctx.idx) {
-    ret.defParam(PARAM.INDEX, "idx");
-    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${store}-idx@`);
-  } else {
-    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${store}@`);
-  }
-  if (store === "data") {
-    if (ctx.file) {
-    } else {
-      ret.defParam(PARAM.SUFFIX, ".car");
-    }
-  }
-  return ret.URI();
-}
 function toStoreURIRuntime(sthis, name, sopts) {
   sopts = sopts || {};
   if (!sopts.base) {
@@ -4870,16 +5223,19 @@ function toStoreURIRuntime(sthis, name, sopts) {
   const base = import_cement20.URI.from(sopts.base);
   return {
     idx: {
-      data: defaultURI2(sthis, name, sopts.idx?.data, base, "data", { idx: true }),
-      file: defaultURI2(sthis, name, sopts.idx?.data, base, "data", { file: true, idx: true }),
-      meta: defaultURI2(sthis, name, sopts.idx?.meta, base, "meta", { idx: true }),
-      wal: defaultURI2(sthis, name, sopts.idx?.wal, base, "wal", { idx: true })
+      car: ensureURIDefaults(sthis, name, sopts.idx?.car ?? sopts.data?.car, base, "car", { idx: true }),
+      file: ensureURIDefaults(sthis, name, sopts.idx?.file ?? sopts.idx?.car ?? sopts.data?.file ?? sopts.data?.car, base, "file", {
+        file: true,
+        idx: true
+      }),
+      meta: ensureURIDefaults(sthis, name, sopts.idx?.meta ?? sopts.data?.meta, base, "meta", { idx: true }),
+      wal: ensureURIDefaults(sthis, name, sopts.idx?.wal ?? sopts.data?.wal, base, "wal", { idx: true })
     },
     data: {
-      data: defaultURI2(sthis, name, sopts.data?.data, base, "data"),
-      file: defaultURI2(sthis, name, sopts.data?.data, base, "data", { file: true }),
-      meta: defaultURI2(sthis, name, sopts.data?.meta, base, "meta"),
-      wal: defaultURI2(sthis, name, sopts.data?.wal, base, "wal")
+      car: ensureURIDefaults(sthis, name, sopts.data?.car, base, "car"),
+      file: ensureURIDefaults(sthis, name, sopts.data?.file ?? sopts.data?.car, base, "file", { file: true }),
+      meta: ensureURIDefaults(sthis, name, sopts.data?.meta, base, "meta"),
+      wal: ensureURIDefaults(sthis, name, sopts.data?.wal, base, "wal")
     }
   };
 }
@@ -4943,6 +5299,6 @@ __export(file_exports, {
 
 // src/version.ts
 var PACKAGE_VERSION = Object.keys({
-  "0.20.0-dev-preview-40": "xxxx"
+  "0.20.0-dev-preview-50": "xxxx"
 })[0];
 //# sourceMappingURL=index.cjs.map