@fireproof/core 0.20.0-dev-preview-41 → 0.20.0-dev-preview-50

package/index.cjs

@@ -48,6 +48,7 @@ __export(index_exports, {
   ensureLogger: () => ensureLogger,
   ensureSuperLog: () => ensureSuperLog,
   ensureSuperThis: () => ensureSuperThis,
+  ensureURIDefaults: () => ensureURIDefaults,
   exceptionWrapper: () => exceptionWrapper,
   falsyToUndef: () => falsyToUndef,
   fireproof: () => fireproof,
@@ -65,6 +66,7 @@ __export(index_exports, {
   onSuperThis: () => onSuperThis,
   rt: () => runtime_exports,
   runtime: () => runtime_exports,
+  storeType2DataMetaWal: () => storeType2DataMetaWal,
   throwFalsy: () => throwFalsy,
   toSortedArray: () => toSortedArray,
   toStoreURIRuntime: () => toStoreURIRuntime
@@ -101,8 +103,13 @@ var PARAM = {
   FRAG_LEN: "len",
   FRAG_HEAD: "headerSize",
   EXTRACTKEY: "extractKey",
-  SELF_REFLECT: "selfReflect"
+  SELF_REFLECT: "selfReflect",
   // if no subscribe in Gateway see your own META updates
+  CAR_PARALLEL: "parallel",
+  CAR_CACHE_SIZE: "carCacheSize",
+  CAR_COMPACT_CACHE_SIZE: "carCompactCacheSize",
+  CAR_META_CACHE_SIZE: "carMetaCacheSize",
+  GENESIS_CID: "baembeiarootfireproofgenesisblockaaaafireproofgenesisblocka"
   // FS = "fs",
 };
 function throwFalsy(value) {
@@ -339,21 +346,25 @@ function ensureLogger(sthis, componentName, ctx) {
   return out;
 }
 function getStore(url, sthis, joiner) {
-  const store = url.getParam(PARAM.STORE);
-  switch (store) {
-    case "data":
+  const fromUrl = url.getParam(PARAM.STORE);
+  let pathPart;
+  switch (fromUrl) {
+    case "car":
+    case "file":
+      pathPart = "data";
+      break;
     case "wal":
     case "meta":
+      pathPart = fromUrl;
       break;
     default:
       throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
-      throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
   }
-  let name = store;
+  let name = pathPart;
   if (url.hasParam("index")) {
     name = joiner(url.getParam(PARAM.INDEX) || "idx", name);
   }
-  return { store, name };
+  return { pathPart, fromUrl, name };
 }
 function getKey(url, logger) {
   const result = url.getParam(PARAM.KEY);
@@ -455,6 +466,35 @@ function makeName(fnString) {
     return found[1];
   }
 }
+function storeType2DataMetaWal(store) {
+  switch (store) {
+    case "car":
+    case "file":
+      return "data";
+    case "meta":
+    case "wal":
+      return store;
+    default:
+      throw new Error(`unknown store ${store}`);
+  }
+}
+function ensureURIDefaults(sthis, name, curi, uri, store, ctx) {
+  ctx = ctx || {};
+  const ret = (curi ? import_cement.URI.from(curi) : uri).build().setParam(PARAM.STORE, store).defParam(PARAM.NAME, name);
+  if (!ret.hasParam(PARAM.NAME)) {
+    throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Ledger name is required").AsError();
+  }
+  if (ctx.idx) {
+    ret.defParam(PARAM.INDEX, "idx");
+    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${storeType2DataMetaWal(store)}-idx@`);
+  } else {
+    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${storeType2DataMetaWal(store)}@`);
+  }
+  if (store === "car") {
+    ret.defParam(PARAM.SUFFIX, ".car");
+  }
+  return ret.URI();
+}
 
 // src/write-queue.ts
 var import_cement2 = require("@adviser/cement");
@@ -1136,8 +1176,10 @@ function getFileName(url, sthis) {
   const key = url.getParam("key");
   if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
   const res = (0, import_core.getStore)(url, sthis, (...a) => a.join("-"));
-  switch (res.store) {
-    case "data":
+  switch (res.fromUrl) {
+    case "file":
+      return sthis.pathOps.join(res.name, key);
+    case "car":
       return sthis.pathOps.join(res.name, key + ".car");
     case "wal":
     case "meta":
@@ -1285,8 +1327,13 @@ var MemoryGateway = class {
   close(baseUrl) {
     return Promise.resolve(import_cement6.Result.Ok(void 0));
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   destroy(baseUrl) {
+    const keyUrl = baseUrl.toString();
+    for (const key of this.memorys.keys()) {
+      if (key.startsWith(keyUrl)) {
+        this.memorys.delete(key);
+      }
+    }
     this.memorys.clear();
     return Promise.resolve(import_cement6.Result.Ok(void 0));
   }
@@ -1298,7 +1345,7 @@ var MemoryGateway = class {
   get(url) {
     const x = this.memorys.get(url.toString());
     if (!x) {
-      return Promise.resolve(import_cement6.Result.Err(new NotFoundError(`not found: ${url.getParam(PARAM.STORE)}`)));
+      return Promise.resolve(import_cement6.Result.Err(new NotFoundError(`not found: ${url.toString()}`)));
     }
     return Promise.resolve(import_cement6.Result.Ok(x));
   }
@@ -1468,10 +1515,9 @@ async function fpDeserialize(sthis, url, intoRaw, pdecoder) {
     ...pdecoder
   };
   switch (url.getParam(PARAM.STORE)) {
-    case "data":
-      if (url.getParam(PARAM.SUFFIX) === ".car") {
-        return makeFPEnvelope(FPEnvelopeTypes.CAR, await decoder.car(sthis, raw2));
-      }
+    case "car":
+      return makeFPEnvelope(FPEnvelopeTypes.CAR, await decoder.car(sthis, raw2));
+    case "file":
       return makeFPEnvelope(FPEnvelopeTypes.FILE, await decoder.file(sthis, raw2));
     case "wal":
       return makeFPEnvelope(FPEnvelopeTypes.WAL, await decode2WalState(sthis, await decoder.wal(sthis, raw2)));
@@ -1533,7 +1579,7 @@ var DefSerdeGateway = class {
       const urlWithoutKey = url.build().delParam(PARAM.KEY).delParam(PARAM.SELF_REFLECT).toString();
       this.subscribeFn.set(urlWithoutKey, rawCallback);
       return import_cement9.Result.Ok(() => {
-        this.subscribeFn.delete(url.toString());
+        this.subscribeFn.delete(urlWithoutKey);
       });
     }
     const unreg = await this.gw.subscribe(url, rawCallback, sthis);
@@ -1662,37 +1708,49 @@ var Block = import_block.Block;
 async function decode({
   bytes,
   codec: codec3,
-  hasher: hasher7
+  hasher: hasher6
 }) {
   if (bytes == null) throw new Error('Missing required argument "bytes"');
-  if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
+  if (codec3 == null || hasher6 == null) throw new Error("Missing required argument: codec or hasher");
   const value = await Promise.resolve(codec3.decode(bytes));
-  const hash = await hasher7.digest(bytes);
+  let toHash = bytes;
+  if (codec3.valueToHashBytes) {
+    toHash = await Promise.resolve(codec3.valueToHashBytes(value));
+  }
+  const hash = await hasher6.digest(toHash);
   const cid = import_multiformats2.CID.create(1, codec3.code, hash);
-  return new import_block.Block({ value, bytes, cid });
+  return new import_block.Block({ value, bytes: toHash, cid });
 }
 async function encode({
   value,
   codec: codec3,
-  hasher: hasher7
+  hasher: hasher6
 }) {
   if (typeof value === "undefined") throw new Error('Missing required argument "value"');
-  if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
-  const bytes = await Promise.resolve(codec3.encode(value));
-  const hash = await hasher7.digest(bytes);
+  if (codec3 == null || hasher6 == null) throw new Error("Missing required argument: codec or hasher");
+  let bytes;
+  let hash;
+  if (codec3.bytesToHash) {
+    const hashable = await Promise.resolve(codec3.bytesToHash(value));
+    hash = await hasher6.digest(hashable);
+    bytes = await Promise.resolve(codec3.encode(value));
+  } else {
+    bytes = await Promise.resolve(codec3.encode(value));
+    hash = await hasher6.digest(bytes);
+  }
   const cid = import_multiformats2.CID.create(1, codec3.code, hash);
-  return new import_block.Block({ value, bytes, cid });
+  return new Block({ value, bytes, cid });
 }
 async function create({
   bytes,
   cid,
-  hasher: hasher7,
+  hasher: hasher6,
   codec: codec3
 }) {
   if (bytes == null) throw new Error('Missing required argument "bytes"');
-  if (hasher7 == null) throw new Error('Missing required argument "hasher"');
+  if (hasher6 == null) throw new Error('Missing required argument "hasher"');
   const value = await Promise.resolve(codec3.decode(bytes));
-  const hash = await hasher7.digest(bytes);
+  const hash = await hasher6.digest(bytes);
   if (!import_multiformats2.bytes.equals(cid.multihash.bytes, hash.bytes)) {
     throw new Error("CID hash does not match bytes");
   }
@@ -2189,8 +2247,11 @@ var import_cement19 = require("@adviser/cement");
 var blockstore_exports = {};
 __export(blockstore_exports, {
   AttachedRemotesImpl: () => AttachedRemotesImpl,
+  BaseActiveStore: () => BaseActiveStore,
   BaseBlockstoreImpl: () => BaseBlockstoreImpl,
   Car2FPMsg: () => Car2FPMsg,
+  CarActiveStore: () => CarActiveStore,
+  CarLog: () => CarLog,
   CarTransactionImpl: () => CarTransactionImpl,
   CompactionFetcher: () => CompactionFetcher,
   DbMetaEventEqual: () => DbMetaEventEqual,
@@ -2198,9 +2259,13 @@ __export(blockstore_exports, {
   EncryptedBlockstore: () => EncryptedBlockstore,
   FPEnvelopeTypes: () => FPEnvelopeTypes,
   File2FPMsg: () => File2FPMsg,
+  FileActiveStore: () => FileActiveStore,
   InterceptorGateway: () => InterceptorGateway,
   Loader: () => Loader,
+  MetaActiveStore: () => MetaActiveStore,
   PassThroughGateway: () => PassThroughGateway,
+  TaskManager: () => TaskManager,
+  WALActiveStore: () => WALActiveStore,
   createAttachedStores: () => createAttachedStores,
   createDbMetaEvent: () => createDbMetaEvent,
   defaultGatewayFactoryItem: () => defaultGatewayFactoryItem,
@@ -2215,6 +2280,37 @@ __export(blockstore_exports, {
 });
 
 // src/blockstore/types.ts
+var CarLog = class {
+  constructor() {
+    this._logs = [];
+  }
+  get length() {
+    return this._logs.length;
+  }
+  last() {
+    const x = [...this._logs[this._logs.length - 1]];
+    Object.freeze(x);
+    return x;
+  }
+  xunshift(logs) {
+    this._logs.unshift(logs);
+  }
+  update(logs) {
+    this._logs.length = 0;
+    this._logs.push(...logs);
+  }
+  asArray() {
+    const a = [
+      ...this._logs.map((l) => {
+        const x = [...l];
+        Object.freeze(x);
+        return x;
+      })
+    ];
+    Object.freeze(a);
+    return a;
+  }
+};
 function toCIDBlock(block) {
   return block;
 }
@@ -2224,6 +2320,16 @@ function DbMetaEventEqual(a, b) {
 function DbMetaEventsEqual(a, b) {
   return a.length === b.length && a.every((e, i) => DbMetaEventEqual(e, b[i]));
 }
+var BaseActiveStore = class {
+};
+var CarActiveStore = class extends BaseActiveStore {
+};
+var FileActiveStore = class extends BaseActiveStore {
+};
+var MetaActiveStore = class extends BaseActiveStore {
+};
+var WALActiveStore = class extends BaseActiveStore {
+};
 
 // src/blockstore/store-factory.ts
 var import_cement17 = require("@adviser/cement");
@@ -2240,8 +2346,8 @@ var import_cement14 = require("@adviser/cement");
 var import_sha22 = require("multiformats/hashes/sha2");
 var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
 async function parseCarFile(reader, logger) {
-  const roots = await reader.getRoots();
-  const header = await reader.get(roots[0]);
+  const roots = await reader.roots;
+  const header = reader.blocks.find((i) => i.cid.equals(roots[0]));
   if (!header) throw logger.Error().Msg("missing header block").AsError();
   const dec = await decode({ bytes: header.bytes, hasher: import_sha22.sha256, codec: dagCodec });
   const fpvalue = dec.value;
@@ -2256,6 +2362,7 @@ var import_block4 = require("@fireproof/vendor/@web3-storage/pail/block");
 var import_cement11 = require("@adviser/cement");
 var CarTransactionImpl = class {
   #memblock = new import_block4.MemoryBlockstore();
+  #hackUnshift;
   constructor(parent, opts = { add: true, noLoader: false }) {
     if (opts.add) {
       parent.transactions.add(this);
@@ -2263,7 +2370,7 @@ var CarTransactionImpl = class {
     this.parent = parent;
   }
   async get(cid) {
-    return await this.superGet(cid) || falsyToUndef(await this.parent.get(cid));
+    return await this.superGet(cid) ?? falsyToUndef(await this.parent.get(cid));
   }
   async superGet(cid) {
     return this.#memblock.get(cid);
@@ -2274,7 +2381,16 @@ var CarTransactionImpl = class {
   putSync(cid, bytes) {
     this.#memblock.putSync(cid, bytes);
   }
+  unshift(cid, bytes) {
+    if (this.#hackUnshift) {
+      throw new Error("unshift already called");
+    }
+    this.#hackUnshift = { cid, bytes };
+  }
   async *entries() {
+    if (this.#hackUnshift) {
+      yield this.#hackUnshift;
+    }
     for await (const blk of this.#memblock.entries()) {
       yield blk;
     }
@@ -2300,6 +2416,11 @@ function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
     keyBag: opts.keyBag || {},
     crypto: (0, import_cement11.toCryptoRuntime)(opts.crypto),
     storeUrls: opts.storeUrls,
+    taskManager: {
+      removeAfter: 3,
+      retryTimeout: 50,
+      ...opts.taskManager
+    },
     // storeEnDeFile: ensureStoreEnDeFile(opts.storeEnDeFile),
     // store,
     storeRuntime: toStoreRuntime(sthis, ensureStoreEnDeFile(opts.storeEnDeFile))
@@ -2392,10 +2513,8 @@ var EncryptedBlockstore = class extends BaseBlockstoreImpl {
   async get(cid) {
     const got = await super.get(cid);
     if (got) return got;
-    if (!this.loader) {
-      return;
-    }
-    return falsyToUndef(await this.loader.getBlock(cid, this.loader.attachedStores.local()));
+    const ret = falsyToUndef(await this.loader.getBlock(cid, this.loader.attachedStores.local()));
+    return ret;
   }
   async transaction(fn, opts = { noLoader: false }) {
     this.logger.Debug().Msg("enter transaction");
@@ -2416,7 +2535,7 @@ var EncryptedBlockstore = class extends BaseBlockstoreImpl {
     await this.ready();
     if (!this.loader) throw this.logger.Error().Msg("loader required to get file, ledger must be named").AsError();
     const reader = await this.loader.loadFileCar(car, this.loader.attachedStores.local());
-    const block = await reader.get(cid);
+    const block = await reader.blocks.find((i) => i.cid.equals(cid));
     if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
     return block.bytes;
   }
@@ -2523,8 +2642,8 @@ var CommitQueue = class {
 
 // src/blockstore/commitor.ts
 var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
-var import_sha23 = require("multiformats/hashes/sha2");
 var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
+var import_sha23 = require("multiformats/hashes/sha2");
 async function encodeCarFile(roots, t, codec3) {
   let size = 0;
   const headerSize = CBW.headerLength({ roots });
@@ -2549,7 +2668,7 @@ async function createCarFile(encoder, cid, t) {
 async function commitFiles(fileStore, walStore, t, done) {
   const { files: roots } = makeFileCarHeader(done);
   const cids = [];
-  const codec3 = (await fileStore.keyedCrypto()).codec();
+  const codec3 = await fileStore.keyedCrypto().then((i) => i.codec());
   const cars = await prepareCarFilesFiles(codec3, roots, t);
   for (const car of cars) {
     const { cid, bytes } = car;
@@ -2575,7 +2694,7 @@ async function prepareCarFilesFiles(encoder, roots, t) {
   return [await encodeCarFile(roots, t, encoder)];
 }
 function makeCarHeader(meta, cars, compact = false) {
-  const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
+  const coreHeader = compact ? { cars: [], compact: cars.asArray() } : { cars: cars.asArray(), compact: [] };
   return { ...coreHeader, meta };
 }
 async function encodeCarHeader(fp) {
@@ -2602,7 +2721,7 @@ async function commit(params, t, done, opts = { noLoader: false, compact: false
 }
 async function prepareCarFiles(encoder, threshold, rootBlock, t) {
   const carFiles = [];
-  threshold = threshold || 128e3 * 8;
+  threshold = threshold || 16 * 65536;
   let clonedt = new CarTransactionImpl(t.parent, { add: false, noLoader: false });
   clonedt.putSync(rootBlock.cid, rootBlock.bytes);
   let newsize = CBW.blockLength(toCIDBlock(rootBlock));
@@ -2628,13 +2747,14 @@ var import_sha24 = require("multiformats/hashes/sha2");
 
 // src/blockstore/task-manager.ts
 var TaskManager = class {
-  constructor(sthis, callback) {
+  constructor(sthis, callback, params) {
     // we need to remove the events after some time
     this.eventsWeHandled = /* @__PURE__ */ new Set();
     this.queue = [];
     this.isProcessing = false;
     this.logger = ensureLogger(sthis, "TaskManager");
     this.callback = callback;
+    this.params = params;
   }
   async handleEvent(cid, parents, dbMeta, store) {
     for (const parent of parents) {
@@ -2663,7 +2783,7 @@ var TaskManager = class {
         this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
       }
       await new Promise((resolve) => setTimeout(resolve, 50));
-      throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
+      this.logger.Warn().Err(err).Msg("retry to process event block");
     } finally {
       this.isProcessing = false;
       if (this.queue.length > 0) {
@@ -2693,11 +2813,32 @@ var AttachedImpl = class {
     return "attached";
   }
 };
-var DataActiveStoreImpl = class {
+var FileActiveStoreImpl = class extends FileActiveStore {
   constructor(ref, active, attached) {
+    super();
     this.ref = ref;
     this.active = active;
-    this.attached = attached;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return this.xattached.remotes();
+  }
+};
+var CarActiveStoreImpl = class extends CarActiveStore {
+  constructor(ref, active, attached) {
+    super();
+    this.ref = ref;
+    this.active = active;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return [this.active, ...this.xattached.remotes().filter((i) => i !== this.active)];
   }
 };
 var CarAttachedStoresImpl = class {
@@ -2722,11 +2863,18 @@ var FileAttachedStoresImpl = class {
     return this.attached.remotes().map(({ active }) => active.file);
   }
 };
-var MetaActiveStoreImpl = class {
+var MetaActiveStoreImpl = class extends MetaActiveStore {
   constructor(ref, active, attached) {
+    super();
     this.ref = ref;
     this.active = active;
-    this.attached = attached;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return [this.active, ...this.xattached.remotes().filter((i) => i !== this.active)];
   }
 };
 var MetaAttachedStoresImpl = class {
@@ -2740,11 +2888,18 @@ var MetaAttachedStoresImpl = class {
     return this.attached.remotes().map(({ active }) => active.meta);
   }
 };
-var WALActiveStoreImpl = class {
+var WALActiveStoreImpl = class extends WALActiveStore {
   constructor(ref, active, attached) {
+    super();
     this.ref = ref;
     this.active = active;
-    this.attached = attached;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return this.xattached.remotes();
   }
 };
 var WALAttachedStoresImpl = class {
@@ -2761,7 +2916,13 @@ var WALAttachedStoresImpl = class {
 var ActiveStoreImpl = class {
   constructor(active, attached) {
     this.active = active;
-    this.attached = attached;
+    this.xattached = attached;
+  }
+  local() {
+    return this.xattached.local();
+  }
+  remotes() {
+    return this.xattached.remotes();
   }
   baseStores() {
     const bs = [this.active.car, this.active.file, this.active.meta];
@@ -2771,19 +2932,19 @@ var ActiveStoreImpl = class {
     return bs;
   }
   carStore() {
-    return new DataActiveStoreImpl(this, this.active.car, new CarAttachedStoresImpl(this.attached));
+    return new CarActiveStoreImpl(this, this.active.car, new CarAttachedStoresImpl(this.xattached));
   }
   fileStore() {
-    return new DataActiveStoreImpl(this, this.active.file, new FileAttachedStoresImpl(this.attached));
+    return new FileActiveStoreImpl(this, this.active.file, new FileAttachedStoresImpl(this.xattached));
   }
   metaStore() {
-    return new MetaActiveStoreImpl(this, this.active.meta, new MetaAttachedStoresImpl(this.attached));
+    return new MetaActiveStoreImpl(this, this.active.meta, new MetaAttachedStoresImpl(this.xattached));
   }
   walStore() {
     if (!this.active.wal) {
-      throw this.attached.loadable.sthis.logger.Error().Msg("wal store not set").AsError();
+      throw this.xattached.loadable.sthis.logger.Error().Msg("wal store not set").AsError();
     }
-    return new WALActiveStoreImpl(this, this.active.wal, new WALAttachedStoresImpl(this.attached));
+    return new WALActiveStoreImpl(this, this.active.wal, new WALAttachedStoresImpl(this.xattached));
   }
 };
 function isLoadable(unknown) {
@@ -2835,7 +2996,25 @@ var AttachedRemotesImpl = class {
   }
   activate(store) {
     if ((0, import_cement13.isCoerceURI)(store)) {
-      throw this.loadable.sthis.logger.Error().Msg("store must be an object").AsError();
+      const activateUrl = import_cement13.URI.from(store);
+      let maxScore = 0;
+      let maxStore;
+      for (const { value } of this._remotes.values()) {
+        if (value.isErr()) {
+          continue;
+        }
+        for (const url of value.Ok().stores.baseStores.map((i) => i.url())) {
+          const mr = url.match(activateUrl);
+          if (mr.score > maxScore) {
+            maxScore = mr.score;
+            maxStore = value.Ok().stores;
+          }
+        }
+      }
+      if (!maxStore) {
+        throw this.loadable.sthis.logger.Error().Url(activateUrl).Msg("no store found").AsError();
+      }
+      store = maxStore;
     }
     return new ActiveStoreImpl(store, this);
   }
@@ -2853,19 +3032,19 @@ var AttachedRemotesImpl = class {
     const gws = {
       car: {
         ...gwp.car,
-        url: import_cement13.BuildURI.from(gwp.car.url).defParam("name", attached.name).URI()
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, gwp.car.url, import_cement13.URI.from(gwp.car.url), "car")
       },
       file: {
         ...gwp.file,
-        url: import_cement13.BuildURI.from(gwp.file.url).defParam("name", attached.name).URI()
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, void 0, import_cement13.URI.from(gwp.file.url), "file", { file: true })
       },
       meta: {
         ...gwp.meta,
-        url: import_cement13.BuildURI.from(gwp.meta.url).defParam("name", attached.name).URI()
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, void 0, import_cement13.URI.from(gwp.meta.url), "meta")
       },
       wal: gwp.wal ? {
         ...gwp.wal,
-        url: import_cement13.BuildURI.from(gwp.wal.url).defParam("name", attached.name).URI()
+        url: ensureURIDefaults(this.loadable.sthis, attached.name, void 0, import_cement13.URI.from(gwp.wal.url), "wal")
       } : void 0
     };
     const key = JSON.stringify(
@@ -2884,11 +3063,6 @@ var AttachedRemotesImpl = class {
           byStore: gws,
           loader: this.loadable
         }),
-        // {
-        //   car: await rt.makeDataStore({ url: gws.carUrl, loader: this.loadable }),
-        //   file: await rt.makeDataStore({ url: gws.filesUrl, loader: this.loadable }),
-        //   meta: await rt.makeMetaStore({ url: gws.metaUrl, loader: this.loadable }),
-        // },
         () => {
           this._remotes.unget(key);
         }
@@ -2906,11 +3080,12 @@ var AttachedRemotesImpl = class {
 
 // src/blockstore/loader.ts
 function carLogIncludesGroup(list, cids) {
-  return list.some((arr) => {
-    return arr.toString() === cids.toString();
-  });
+  const cidSet = cids.map((cid) => cid.toString()).sort().join(",");
+  return list.some(
+    (arr) => cidSet === arr.map((cid) => cid.toString()).sort().join(",")
+  );
 }
-function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
+function uniqueCids(list, remove = new import_cement14.LRUSet()) {
   const byString = /* @__PURE__ */ new Map();
   for (const cid of list) {
     if (remove.has(cid.toString())) continue;
@@ -2922,13 +3097,8 @@ var Loader = class {
   constructor(sthis, ebOpts) {
     this.commitQueue = new CommitQueue();
     this.isCompacting = false;
-    this.carReaders = /* @__PURE__ */ new Map();
-    this.seenCompacted = /* @__PURE__ */ new Set();
-    this.processedCars = /* @__PURE__ */ new Set();
-    this.carLog = [];
-    this.getBlockCache = /* @__PURE__ */ new Map();
-    this.seenMeta = /* @__PURE__ */ new Set();
-    this.writeLimit = (0, import_p_limit.default)(1);
+    this.maxConcurrentWrite = (0, import_p_limit.default)(1);
+    this.carLog = new CarLog();
     this.onceReady = new import_cement14.ResolveOnce();
     this.sthis = sthis;
     this.ebOpts = defaultedBlockstoreRuntime(
@@ -2939,63 +3109,44 @@ var Loader = class {
       },
       "Loader"
     );
-    this.logger = this.ebOpts.logger;
-    this.taskManager = new TaskManager(sthis, async (dbMeta, activeStore) => {
-      await this.handleDbMetasFromStore([dbMeta], activeStore);
+    this.logger = ensureLogger(sthis, "Loader");
+    this.cidCache = new import_cement14.KeyedResolvOnce({
+      lru: {
+        maxEntries: parseInt(this.ebOpts.storeUrls.car.getParam(PARAM.CAR_CACHE_SIZE, "1000"), 10)
+      }
+    });
+    this.seenMeta = new import_cement14.LRUSet({
+      maxEntries: parseInt(this.ebOpts.storeUrls.meta.getParam(PARAM.CAR_META_CACHE_SIZE, "1000"), 10)
+    });
+    this.seenCompacted = new import_cement14.LRUSet({
+      maxEntries: parseInt(this.ebOpts.storeUrls.car.getParam(PARAM.CAR_COMPACT_CACHE_SIZE, "1000"), 10)
     });
+    this.maxConcurrentCarReader = (0, import_p_limit.default)(parseInt(this.ebOpts.storeUrls.car.getParam(PARAM.CAR_PARALLEL, "5"), 10));
+    this.taskManager = new TaskManager(
+      sthis,
+      async (dbMeta, activeStore) => {
+        await this.handleDbMetasFromStore([dbMeta], activeStore);
+      },
+      this.ebOpts.taskManager
+    );
     this.attachedStores = new AttachedRemotesImpl(this);
   }
-  attach(attached) {
-    return this.attachedStores.attach(attached);
-  }
-  // private readonly _carStore = new ResolveOnce<DataStore>();
-  // async carStore(): Promise<DataStore> {
-  //   return this._carStore.once(async () =>
-  //     this.ebOpts.storeRuntime.makeDataStore({
-  //       // sthis: this.sthis,
-  //       gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-  //       url: this.ebOpts.storeUrls.data,
-  //       // keybag: await this.keyBag(),
-  //       loader: this,
-  //     }),
-  //   );
-  // }
-  // private readonly _fileStore = new ResolveOnce<DataStore>();
-  // async fileStore(): Promise<DataStore> {
-  //   return this._fileStore.once(async () =>
-  //     this.ebOpts.storeRuntime.makeDataStore({
-  //       // sthis: this.sthis,
-  //       gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-  //       url: this.ebOpts.storeUrls.file,
-  //       // keybag: await this.keyBag(),
-  //       loader: this,
-  //     }),
-  //   );
-  // }
-  // private readonly _WALStore = new ResolveOnce<WALStore>();
-  // async WALStore(): Promise<WALStore> {
-  //   return this._WALStore.once(async () =>
-  //     this.ebOpts.storeRuntime.makeWALStore({
-  //       // sthis: this.sthis,
-  //       gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-  //       url: this.ebOpts.storeUrls.wal,
-  //       // keybag: await this.keyBag(),
-  //       loader: this,
-  //     }),
-  //   );
-  // }
-  // private readonly _metaStore = new ResolveOnce<MetaStore>();
-  // async metaStore(): Promise<MetaStore> {
-  //   return this._metaStore.once(async () =>
-  //     this.ebOpts.storeRuntime.makeMetaStore({
-  //       // sthis: this.sthis,
-  //       gatewayInterceptor: this.ebOpts.gatewayInterceptor,
-  //       url: this.ebOpts.storeUrls.meta,
-  //       // keybag: await this.keyBag(),
-  //       loader: this,
-  //     }),
-  //   );
-  // }
+  async attach(attached) {
+    const at = await this.attachedStores.attach(attached);
+    if (!at.stores.wal) {
+      try {
+        const dbMeta = await at.stores.meta.load();
+        if (!Array.isArray(dbMeta)) {
+          throw this.logger.Error().Msg("missing dbMeta").AsError();
+        }
+        await this.handleDbMetasFromStore(dbMeta, this.attachedStores.activate(at.stores));
+      } catch (e) {
+        this.logger.Error().Err(e).Msg("error attaching store");
+        at.detach();
+      }
+    }
+    return at;
+  }
   keyBag() {
     return getKeyBag(this.sthis, this.ebOpts.keyBag);
   }
@@ -3039,9 +3190,9 @@ var Loader = class {
   //   await this._applyCarHeader(carHeader, true)
   // }
   async handleDbMetasFromStore(metas, activeStore) {
-    this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
+    this.logger.Debug().Any("metas", metas).Url(activeStore.active.car.url()).Msg("handleDbMetasFromStore");
     for (const meta of metas) {
-      await this.writeLimit(async () => {
+      await this.maxConcurrentWrite(async () => {
         await this.mergeDbMetaIntoClock(meta, activeStore);
       });
     }
@@ -3050,16 +3201,26 @@ var Loader = class {
     if (this.isCompacting) {
       throw this.logger.Error().Msg("cannot merge while compacting").AsError();
     }
-    if (this.seenMeta.has(meta.cars.toString())) return;
-    this.seenMeta.add(meta.cars.toString());
-    if (carLogIncludesGroup(this.carLog, meta.cars)) {
-      return;
+    try {
+      this.isCompacting = true;
+      const metaKey = meta.cars.map((i) => i.toString()).sort().join(",");
+      if (this.seenMeta.has(metaKey)) return;
+      this.seenMeta.add(metaKey);
+      if (carLogIncludesGroup(this.carLog.asArray(), meta.cars)) {
+        return;
+      }
+      const carHeader = await this.loadCarHeaderFromMeta(meta, activeStore);
+      carHeader.compact.map((c) => c.toString()).forEach((k) => this.seenCompacted.add(k), this.seenCompacted);
+      try {
+        await this.getMoreReaders(carHeader.cars.flat(), activeStore);
+      } catch (e) {
+        this.logger.Error().Err(e).Msg("error getting more readers");
+      }
+      this.carLog.update(uniqueCids([meta.cars, ...this.carLog.asArray(), ...carHeader.cars], this.seenCompacted));
+      await this.ebOpts.applyMeta?.(carHeader.meta);
+    } finally {
+      this.isCompacting = false;
     }
-    const carHeader = await this.loadCarHeaderFromMeta(meta, activeStore);
-    carHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
-    await this.getMoreReaders(carHeader.cars.flat(), activeStore);
-    this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
-    await this.ebOpts.applyMeta?.(carHeader.meta);
   }
   // protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
   //   const { key } = meta;
@@ -3090,13 +3251,13 @@ var Loader = class {
   }
   async commit(t, done, opts = { noLoader: false, compact: false }) {
     await this.ready();
-    const carStore = await this.attachedStores.local().active.car;
+    const carStore = this.attachedStores.local().active.car;
     const params = {
       encoder: (await carStore.keyedCrypto()).codec(),
       carLog: this.carLog,
       carStore,
-      WALStore: await this.attachedStores.local().active.wal,
-      metaStore: await this.attachedStores.local().active.meta,
+      WALStore: this.attachedStores.local().active.wal,
+      metaStore: this.attachedStores.local().active.meta,
       threshold: this.ebOpts.threshold
     };
     return this.commitQueue.enqueue(async () => {
@@ -3106,34 +3267,44 @@ var Loader = class {
       return ret.cgrp;
     });
   }
-  async updateCarLog(cids, fp, compact) {
+  async updateCarLog(cids, cHeader, compact) {
     if (compact) {
-      const previousCompactCid = fp.compact[fp.compact.length - 1];
-      fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
-      this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
+      const previousCompactCid = cHeader.compact[cHeader.compact.length - 1];
+      cHeader.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
+      this.carLog.update(uniqueCids([...this.carLog.asArray(), ...cHeader.cars, cids], this.seenCompacted));
       await this.removeCidsForCompact(previousCompactCid[0], this.attachedStores.local()).catch((e) => e);
     } else {
-      this.carLog.unshift(cids);
+      this.carLog.xunshift(cids);
     }
   }
   async cacheTransaction(t) {
     for await (const block of t.entries()) {
       const sBlock = block.cid.toString();
-      if (!this.getBlockCache.has(sBlock)) {
-        this.getBlockCache.set(sBlock, block);
-      }
-    }
-  }
-  async cacheCarReader(carCidStr, reader) {
-    if (this.processedCars.has(carCidStr)) return;
-    this.processedCars.add(carCidStr);
-    for await (const block of reader.blocks()) {
-      const sBlock = block.cid.toString();
-      if (!this.getBlockCache.has(sBlock)) {
-        this.getBlockCache.set(sBlock, block);
-      }
+      this.cidCache.get(sBlock).once(
+        () => ({
+          type: "block",
+          cid: block.cid,
+          blocks: [block],
+          roots: []
+        })
+      );
     }
   }
+  // /**
+  //  *
+  //  * @returns the list of blocks which was read from the car file
+  //  */
+  // private async readCar(reader: CarReader): Promise<AnyBlock[]> {
+  //   const blocks: AnyBlock[] = [];
+  //   for await (const block of reader.blocks()) {
+  //     const sBlock = block.cid.toString();
+  //     this.cidCache.get(sBlock).once(() => {
+  //       blocks.push(block);
+  //       return [block];
+  //     });
+  //   }
+  //   return blocks;
+  // }
   async removeCidsForCompact(cid, store) {
     const carHeader = await this.loadCarHeaderFromMeta(
       {
@@ -3155,149 +3326,134 @@ var Loader = class {
   //     await this.remoteWAL!.enqueue(dbMeta, { public: false })
   //   }
   // }
-  async *entries(cache2 = true) {
+  async *entries() {
     await this.ready();
-    if (cache2) {
-      for (const [, block] of this.getBlockCache) {
-        yield block;
-      }
-    } else {
-      for (const [, block] of this.getBlockCache) {
-        yield block;
-      }
-      for (const cids of this.carLog) {
-        for (const cid of cids) {
-          const reader = await this.loadCar(cid, this.attachedStores.local());
-          if (!reader) throw this.logger.Error().Ref("cid", cid).Msg("missing car reader").AsError();
-          for await (const block of reader.blocks()) {
-            const sCid = block.cid.toString();
-            if (!this.getBlockCache.has(sCid)) {
-              yield block;
-            }
-          }
+    const seen = /* @__PURE__ */ new Set();
+    for (const carCids of this.carLog.asArray()) {
+      for (const carCid of carCids) {
+        const reader = await this.loadCar(carCid, this.attachedStores.local());
+        if (!reader || reader.type !== "car") {
+          throw this.logger.Error().Any("reader", reader.type).Str("cid", carCid.toString()).Msg("missing car reader").AsError();
+        }
+        for (const block of reader.blocks) {
+          const cidStr = block.cid.toString();
+          if (seen.has(cidStr)) continue;
+          seen.add(cidStr);
+          yield block;
         }
       }
     }
   }
   async getBlock(cid, store) {
     await this.ready();
-    const sCid = cid.toString();
-    if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-    const getCarCid = async (carCid) => {
-      if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-      const reader = await this.loadCar(carCid, store);
-      if (!reader) {
-        throw this.logger.Error().Ref("cid", carCid).Msg("missing car reader").AsError();
-      }
-      await this.cacheCarReader(carCid.toString(), reader).catch(() => {
-        return;
-      });
-      if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-      throw this.logger.Error().Str("cid", sCid).Msg("block not in reader").AsError();
-    };
-    const getCompactCarCids = async (carCid) => {
-      const reader = await this.loadCar(carCid, store);
-      if (!reader) {
-        throw this.logger.Error().Str("cid", carCid.toString()).Msg("missing car reader").AsError();
-      }
-      const header = await parseCarFile(reader, this.logger);
-      const compacts = header.compact;
-      let got2;
-      const batchSize2 = 5;
-      for (let i = 0; i < compacts.length; i += batchSize2) {
-        const promises = [];
-        for (let j = i; j < Math.min(i + batchSize2, compacts.length); j++) {
-          for (const cid2 of compacts[j]) {
-            promises.push(getCarCid(cid2));
+    const cidStr = cid.toString();
+    const ci = await this.cidCache.get(cidStr).once(async () => {
+      const getCompactCarCids = async (carCid) => {
+        const sCid = carCid.toString();
+        const reader = await this.loadCar(carCid, store);
+        const header = await parseCarFile(reader, this.logger);
+        const compacts = header.compact;
+        const got2 = await Promise.allSettled(compacts.map((compact) => compact.map((cid2) => this.loadCar(cid2, store)).flat()));
+        got2.filter((result) => result.status === "rejected").forEach((result) => {
+          this.logger.Error().Err(result.reason).Str("cid", sCid).Msg("error getting compacted block");
+        });
+      };
+      let got;
+      for (const carCids of this.carLog.asArray()) {
+        for (const carCid of carCids) {
+          const ci2 = await this.loadCar(carCid, store);
+          if (!ci2) {
+            this.logger.Error().Str("cid", carCid.toString()).Msg("missing CarCID");
+            continue;
+          }
+          got = ci2.blocks.find((block) => block.cid.equals(cid));
+          if (got) {
+            break;
           }
         }
-        try {
-          got2 = await Promise.any(promises);
-        } catch {
-        }
-        if (got2) break;
       }
-      if (this.getBlockCache.has(sCid)) return this.getBlockCache.get(sCid);
-      throw this.logger.Error().Str("cid", sCid).Msg("block not in compact reader").AsError();
-    };
-    let got;
-    const batchSize = 5;
-    for (let i = 0; i < this.carLog.length; i += batchSize) {
-      const batch = this.carLog.slice(i, i + batchSize);
-      const promises = batch.flatMap((slice) => slice.map(getCarCid));
-      try {
-        got = await Promise.any(promises);
-      } catch {
-      }
-      if (got) break;
-    }
-    if (!got) {
-      try {
-        got = await getCompactCarCids(this.carLog[this.carLog.length - 1][0]);
-      } catch {
+      if (!got) {
+        await getCompactCarCids(this.carLog.last()[0]);
       }
+      return {
+        type: "block",
+        cid,
+        blocks: got ? [got] : [],
+        roots: []
+      };
+    });
+    if (!(ci.type === "block" && ci.blocks.length === 1)) {
+      throw this.logger.Error().Str("cid", cidStr).Any("block", ci).Msg("missing block").AsError();
     }
-    return got;
+    return ci.blocks[0];
   }
   async loadCar(cid, store) {
     const loaded = await this.storesLoadCar(cid, store.carStore());
     return loaded;
   }
-  async makeDecoderAndCarReader(cid, store) {
-    const cidsString = cid.toString();
+  async makeDecoderAndCarReader(carCid, store) {
+    const carCidStr = carCid.toString();
     let loadedCar = void 0;
-    let activeStore = store.attached.local();
+    let activeStore = store.local();
     try {
-      this.logger.Debug().Any("cid", cidsString).Msg("loading car");
-      loadedCar = await store.attached.local().load(cid);
+      this.logger.Debug().Any("cid", carCidStr).Msg("loading car");
+      loadedCar = await store.local().load(carCid);
       this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
     } catch (e) {
       if (!isNotFoundError(e)) {
-        throw this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
+        throw this.logger.Error().Str("cid", carCidStr).Err(e).Msg("loading car");
       }
-      for (const remote of store.attached.remotes()) {
-        const remoteCar = await remote.load(cid);
-        if (remoteCar) {
-          this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
-          await store.attached.local().save(remoteCar);
-          loadedCar = remoteCar;
-          activeStore = remote;
-          break;
-        } else {
-          this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
+      for (const remote of store.remotes()) {
+        try {
+          const remoteCar = await remote.load(carCid);
+          if (remoteCar) {
+            this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
+            await store.local().save(remoteCar);
+            loadedCar = remoteCar;
+            activeStore = remote;
+            break;
+          } else {
+            this.logger.Error().Str("cid", carCidStr).Err(e).Msg("loading car");
+          }
+        } catch (e2) {
+          this.logger.Warn().Str("cid", carCidStr).Url(remote.url()).Err(e2).Msg("loading car");
         }
       }
     }
     if (!loadedCar) {
-      throw this.logger.Error().Url(store.attached.local().url()).Str("cid", cidsString).Msg("missing car files").AsError();
+      throw this.logger.Error().Url(store.local().url()).Str("cid", carCidStr).Msg("missing car files").AsError();
     }
     const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha24.sha256, codec: (await activeStore.keyedCrypto()).codec() });
-    const rawReader = await import_reader.CarReader.fromBytes(bytes.value);
-    const readerP = Promise.resolve(rawReader);
-    const cachedReaderP = readerP.then(async (reader) => {
-      await this.cacheCarReader(cidsString, reader).catch((e) => {
-        this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
-        return;
-      });
-      return reader;
-    });
-    this.carReaders.set(cidsString, cachedReaderP);
-    return readerP;
+    const rawReader = await import_reader.CarReader.fromBytes(bytes.value.data);
+    const blocks = [];
+    for await (const block of rawReader.blocks()) {
+      const sBlock = block.cid.toString();
+      blocks.push(block);
+      this.cidCache.get(sBlock).once(() => ({
+        type: "block",
+        cid: block.cid,
+        blocks: [block],
+        roots: []
+      }));
+    }
+    return {
+      type: "car",
+      cid: carCid,
+      blocks,
+      roots: await rawReader.getRoots()
+    };
   }
   //What if instead it returns an Array of CarHeader
-  async storesLoadCar(cid, store) {
-    const cidsString = cid.toString();
-    let dacr = this.carReaders.get(cidsString);
-    if (!dacr) {
-      dacr = this.makeDecoderAndCarReader(cid, store);
-      this.carReaders.set(cidsString, dacr);
-    }
-    return dacr;
+  async storesLoadCar(carCid, store) {
+    const carCidStr = carCid.toString();
+    return this.cidCache.get(carCidStr).once(async () => {
+      return this.maxConcurrentCarReader(() => this.makeDecoderAndCarReader(carCid, store));
+    });
   }
   async getMoreReaders(cids, store) {
-    const limit = (0, import_p_limit.default)(5);
-    const missing = cids.filter((cid) => !this.carReaders.has(cid.toString()));
-    await Promise.all(missing.map((cid) => limit(() => this.loadCar(cid, store))));
+    for (const cid of cids) {
+      await this.loadCar(cid, store);
+    }
   }
 };
 
@@ -3342,12 +3498,21 @@ function getGenerateIVFn(url, opts) {
 }
 var BlockIvKeyIdCodec = class {
   constructor(ko, iv, opts) {
-    this.code = 3147065;
+    this.code = 24;
     this.name = "Fireproof@encrypted-block:aes-gcm";
     this.ko = ko;
     this.iv = iv;
     this.opts = opts || {};
   }
+  // hashAsBytes(data: IvKeyIdData): AsyncHashAsBytes<Uint8Array<ArrayBufferLike>> {
+  //   return data;
+  // }
+  valueToHashBytes(value) {
+    return Promise.resolve(value.data);
+  }
+  bytesToHash(data) {
+    return Promise.resolve(data);
+  }
   async encode(data) {
     const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
     const { iv } = this.ko.algo(calcIv);
@@ -3379,11 +3544,16 @@ var BlockIvKeyIdCodec = class {
     if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
       throw this.ko.logger.Error().Msg("iv missmatch").AsError();
     }
-    return result;
+    return {
+      iv,
+      keyId,
+      data: result
+    };
   }
 };
 var cryptoAction = class {
   constructor(url, key, cyopt, sthis) {
+    this.code = 24;
     this.ivLength = 12;
     this.isEncrypting = true;
     this.logger = ensureLogger(sthis, "cryptoAction");
@@ -3417,14 +3587,19 @@ var cryptoAction = class {
 };
 var nullCodec = class {
   constructor() {
-    this.code = 0;
+    this.code = 24;
     this.name = "Fireproof@unencrypted-block";
+    this.empty = new Uint8Array();
   }
-  encode(data) {
+  async encode(data) {
     return data;
   }
-  decode(data) {
-    return data;
+  async decode(data) {
+    return {
+      iv: this.empty,
+      keyId: this.empty,
+      data
+    };
   }
 };
 var noCrypto = class {
@@ -3679,13 +3854,8 @@ var BaseStoreImpl = class {
     this._url = res.Ok();
     const kb = await this.loader.keyBag();
     const skRes = await kb.ensureKeyFromUrl(this._url, () => {
-      const idx = this._url.getParam(PARAM.INDEX);
-      const storeKeyName = [this.url().getParam(PARAM.NAME)];
-      if (idx) {
-        storeKeyName.push(idx);
-      }
-      storeKeyName.push(this.storeType);
-      return storeKeyName.join(":");
+      const key = this._url.getParam(PARAM.KEY);
+      return key;
     });
     if (skRes.isErr()) {
       return skRes;
@@ -3768,8 +3938,7 @@ var MetaStoreImpl = class extends BaseStoreImpl {
   //   }
   //   return (rDbMeta.Ok() as FPEnvelopeMeta).payload;
   // }
-  async load() {
-    const branch = "main";
+  async load(branch = "main") {
     const url = await this.gateway.buildUrl({ loader: this.loader }, this.url(), branch);
     if (url.isErr()) {
       throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
@@ -3783,7 +3952,7 @@ var MetaStoreImpl = class extends BaseStoreImpl {
     }
     const fpMeta = rfpEnv.Ok().payload;
     const dbMetas = fpMeta.map((m) => m.dbMeta);
-    await this.loader.handleDbMetasFromStore(dbMetas, this.loader.attachedStores.local());
+    await this.loader.handleDbMetasFromStore(dbMetas, this.loader.attachedStores.activate(url.Ok()));
     this.updateParentsFromDbMetas(fpMeta);
     return dbMetas;
   }
@@ -3815,9 +3984,8 @@ var MetaStoreImpl = class extends BaseStoreImpl {
   }
 };
 var DataStoreImpl = class extends BaseStoreImpl {
-  constructor(sthis, url, opts) {
-    super(sthis, url, { ...opts }, ensureLogger(sthis, "DataStoreImpl"));
-    this.storeType = "data";
+  constructor(sthis, url, opts, logger) {
+    super(sthis, url, { ...opts }, logger);
   }
   async load(cid) {
     this.logger.Debug().Any("cid", cid).Msg("loading");
@@ -3832,7 +4000,6 @@ var DataStoreImpl = class extends BaseStoreImpl {
     const fpenv = res.Ok();
     switch (fpenv.type) {
       case "car":
-        return { cid, bytes: fpenv.payload };
       case "file":
         return { cid, bytes: fpenv.payload };
       default:
@@ -3848,12 +4015,11 @@ var DataStoreImpl = class extends BaseStoreImpl {
     }
     let fpMsg;
     switch (url.Ok().getParam(PARAM.STORE)) {
-      case "data":
-        if (url.Ok().getParam(PARAM.SUFFIX)) {
-          fpMsg = Car2FPMsg(car.bytes);
-        } else {
-          fpMsg = File2FPMsg(car.bytes);
-        }
+      case "car":
+        fpMsg = Car2FPMsg(car.bytes);
+        break;
+      case "file":
+        fpMsg = File2FPMsg(car.bytes);
         break;
       default:
         throw this.logger.Error().Str("store", url.Ok().getParam(PARAM.STORE)).Msg("unexpected store").AsError();
@@ -3884,6 +4050,18 @@ var DataStoreImpl = class extends BaseStoreImpl {
     return this.gateway.destroy({ loader: this.loader }, this.url());
   }
 };
+var CarStoreImpl = class extends DataStoreImpl {
+  constructor(sthis, url, opts) {
+    super(sthis, url, { ...opts }, ensureLogger(sthis, "CarStoreImpl"));
+    this.storeType = "car";
+  }
+};
+var FileStoreImpl = class extends DataStoreImpl {
+  constructor(sthis, url, opts) {
+    super(sthis, url, { ...opts }, ensureLogger(sthis, "FileStoreImpl"));
+    this.storeType = "file";
+  }
+};
 var WALStoreImpl = class extends BaseStoreImpl {
   constructor(sthis, url, opts) {
     super(sthis, url, { ...opts }, ensureLogger(sthis, "WALStoreImpl"));
@@ -3966,7 +4144,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
             for (const cid of dbMeta.cars) {
               const car = await this.loader.attachedStores.local().active.car.load(cid);
               if (!car) {
-                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                if (carLogIncludesGroup(this.loader.carLog.asArray(), dbMeta.cars)) {
                   throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
                 }
               } else {
@@ -3988,7 +4166,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
             for (const cid of dbMeta.cars) {
               const car = await this.loader.attachedStores.local().active.car.load(cid);
               if (!car) {
-                if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars)) {
+                if (carLogIncludesGroup(this.loader.carLog.asArray(), dbMeta.cars)) {
                   throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
                 }
               } else {
@@ -4084,8 +4262,9 @@ async function getStartedGateway(ctx, url) {
     if (item) {
       const ret = {
         url,
-        ...await gatewayInstances.get(url.protocol).once(async () => ({})),
-        gateway: await item.serdegateway(ctx.loader.sthis)
+        ...await gatewayInstances.get(url.protocol).once(async () => ({
+          gateway: await item.serdegateway(ctx.loader.sthis)
+        }))
       };
       const res = await ret.gateway.start(ctx, url);
       if (res.isErr()) {
@@ -4097,14 +4276,28 @@ async function getStartedGateway(ctx, url) {
     return import_cement17.Result.Err(ctx.loader.sthis.logger.Warn().Url(url).Msg("unsupported protocol").AsError());
   });
 }
-async function dataStoreFactory(ctx, uai) {
-  const storeUrl = uai.url.build().setParam(PARAM.STORE, "data").URI();
+async function carStoreFactory(ctx, uai) {
+  const storeUrl = uai.url.build().setParam(PARAM.STORE, "car").URI();
   const rgateway = await getStartedGateway(ctx, storeUrl);
   if (rgateway.isErr()) {
     throw ctx.loader.sthis.logger.Error().Result("err", rgateway).Url(uai.url).Msg("notfound").AsError();
   }
   const gateway = rgateway.Ok();
-  const store = new DataStoreImpl(ctx.loader.sthis, gateway.url, {
+  const store = new CarStoreImpl(ctx.loader.sthis, gateway.url, {
+    gateway: gateway.gateway,
+    gatewayInterceptor: uai.gatewayInterceptor,
+    loader: ctx.loader
+  });
+  return store;
+}
+async function fileStoreFactory(ctx, uai) {
+  const storeUrl = uai.url.build().setParam(PARAM.STORE, "file").URI();
+  const rgateway = await getStartedGateway(ctx, storeUrl);
+  if (rgateway.isErr()) {
+    throw ctx.loader.sthis.logger.Error().Result("err", rgateway).Url(uai.url).Msg("notfound").AsError();
+  }
+  const gateway = rgateway.Ok();
+  const store = new FileStoreImpl(ctx.loader.sthis, gateway.url, {
     gateway: gateway.gateway,
     gatewayInterceptor: uai.gatewayInterceptor,
     loader: ctx.loader
@@ -4162,8 +4355,8 @@ function toStoreRuntime(sthis, endeOpts = {}) {
       };
       const storeSet = {};
       storeSet.meta = await metaStoreFactory(ctx, sfi.byStore.meta);
-      storeSet.car = await dataStoreFactory(ctx, sfi.byStore.car);
-      storeSet.file = await dataStoreFactory(ctx, sfi.byStore.file);
+      storeSet.car = await carStoreFactory(ctx, sfi.byStore.car);
+      storeSet.file = await fileStoreFactory(ctx, sfi.byStore.file);
       if (sfi.byStore.wal) {
         storeSet.wal = await WALStoreFactory(ctx, sfi.byStore.wal);
       }
@@ -4202,6 +4395,34 @@ function toString(key, logger) {
       throw logger.Error().Msg("Invalid key type").AsError();
   }
 }
+function sanitizeDocumentFields(obj) {
+  if (Array.isArray(obj)) {
+    return obj.map((item) => {
+      if (typeof item === "object" && item !== null) {
+        return sanitizeDocumentFields(item);
+      }
+      return item;
+    });
+  } else if (typeof obj === "object" && obj !== null) {
+    const typedObj = obj;
+    const result = {};
+    for (const key in typedObj) {
+      if (Object.hasOwnProperty.call(typedObj, key)) {
+        const value = typedObj[key];
+        if (value === null || !Number.isNaN(value) && value !== void 0) {
+          if (typeof value === "object" && !key.startsWith("_")) {
+            const sanitized = sanitizeDocumentFields(value);
+            result[key] = sanitized;
+          } else {
+            result[key] = value;
+          }
+        }
+      }
+    }
+    return result;
+  }
+  return obj;
+}
 async function applyBulkUpdateToCrdt(store, tblocks, head, updates, logger) {
   let result = null;
   if (updates.length > 1) {
@@ -4291,7 +4512,8 @@ async function getValueFromCrdt(blocks, head, key, logger) {
   if (!head.length) throw logger.Debug().Msg("Getting from an empty ledger").AsError();
   const link = await (0, import_crdt.get)(blocks, head, key);
   if (!link) throw logger.Error().Str("key", key).Msg(`Missing key`).AsError();
-  return await getValueFromLink(blocks, link, logger);
+  const ret = await getValueFromLink(blocks, link, logger);
+  return ret;
 }
 function readFiles(blocks, { doc }) {
   if (!doc) return;
@@ -4389,6 +4611,9 @@ async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], k
       const { key, value } = ops[i];
       if (!keys.has(key)) {
         const docValue = await getValueFromLink(blocks, value, logger);
+        if (key === PARAM.GENESIS_CID) {
+          continue;
+        }
         updates.push({ id: key, value: docValue.doc, del: docValue.del, clock: link });
         limit--;
         keys.add(key);
@@ -4402,8 +4627,10 @@ async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], k
 }
 async function* getAllEntries(blocks, head, logger) {
   for await (const [key, link] of (0, import_crdt.entries)(blocks, head)) {
-    const docValue = await getValueFromLink(blocks, link, logger);
-    yield { id: key, value: docValue.doc, del: docValue.del };
+    if (key !== PARAM.GENESIS_CID) {
+      const docValue = await getValueFromLink(blocks, link, logger);
+      yield { id: key, value: docValue.doc, del: docValue.del };
+    }
   }
 }
 async function* clockVis(blocks, head) {
@@ -4548,6 +4775,10 @@ var CRDTClockImpl = class {
     this.notifyWatchers(internalUpdates || []);
   }
   notifyWatchers(updates) {
+    updates = updates.filter((update) => update.id !== PARAM.GENESIS_CID);
+    if (!updates.length) {
+      return;
+    }
     this.emptyWatchers.forEach((fn) => fn());
     this.watchers.forEach((fn) => fn(updates || []));
   }
@@ -4627,7 +4858,7 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
     try {
       head = await (0, import_clock4.advance)(tblocks, head, cid);
     } catch (e) {
-      logger.Debug().Err(e).Msg("failed to advance head");
+      logger.Error().Err(e).Msg("failed to advance head");
       continue;
     }
   }
@@ -4686,6 +4917,17 @@ var CRDTImpl = class {
   }
   async bulk(updates) {
     await this.ready();
+    updates = updates.map((dupdate) => ({
+      ...dupdate,
+      value: sanitizeDocumentFields(dupdate.value)
+    }));
+    if (this.clock.head.length === 0) {
+      const value = { id: PARAM.GENESIS_CID, value: { _id: PARAM.GENESIS_CID } };
+      await this._bulk([value]);
+    }
+    return await this._bulk(updates);
+  }
+  async _bulk(updates) {
     const prevHead = [...this.clock.head];
     const done = await this.blockstore.transaction(async (blocks) => {
       const { head } = await applyBulkUpdateToCrdt(
@@ -4917,7 +5159,8 @@ var LedgerImpl = class {
     });
     return ret;
   }
-  attach(a) {
+  async attach(a) {
+    await this.ready();
     return this.crdt.blockstore.loader.attach(a);
   }
   // readonly _asDb = new ResolveOnce<Database>();
@@ -4925,6 +5168,7 @@ var LedgerImpl = class {
   //   return this._asDb.once(() => new DatabaseImpl(this));
   // }
   subscribe(listener, updates) {
+    this.ready();
     this.logger.Debug().Bool("updates", updates).Msg("subscribe");
     if (updates) {
       if (!this._listening) {
@@ -4966,26 +5210,6 @@ var LedgerImpl = class {
     }
   }
 };
-function defaultURI2(sthis, name, curi, uri, store, ctx) {
-  ctx = ctx || {};
-  const ret = (curi ? import_cement20.URI.from(curi) : uri).build().setParam(PARAM.STORE, store).defParam(PARAM.NAME, name);
-  if (!ret.hasParam(PARAM.NAME)) {
-    throw sthis.logger.Error().Url(ret).Any("ctx", ctx).Msg("Ledger name is required").AsError();
-  }
-  if (ctx.idx) {
-    ret.defParam(PARAM.INDEX, "idx");
-    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${store}-idx@`);
-  } else {
-    ret.defParam(PARAM.STORE_KEY, `@${ret.getParam(PARAM.NAME)}-${store}@`);
-  }
-  if (store === "data") {
-    if (ctx.file) {
-    } else {
-      ret.defParam(PARAM.SUFFIX, ".car");
-    }
-  }
-  return ret.URI();
-}
 function toStoreURIRuntime(sthis, name, sopts) {
   sopts = sopts || {};
   if (!sopts.base) {
@@ -4999,16 +5223,19 @@ function toStoreURIRuntime(sthis, name, sopts) {
   const base = import_cement20.URI.from(sopts.base);
   return {
     idx: {
-      car: defaultURI2(sthis, name, sopts.idx?.data, base, "data", { idx: true }),
-      file: defaultURI2(sthis, name, sopts.idx?.data, base, "data", { file: true, idx: true }),
-      meta: defaultURI2(sthis, name, sopts.idx?.meta, base, "meta", { idx: true }),
-      wal: defaultURI2(sthis, name, sopts.idx?.wal, base, "wal", { idx: true })
+      car: ensureURIDefaults(sthis, name, sopts.idx?.car ?? sopts.data?.car, base, "car", { idx: true }),
+      file: ensureURIDefaults(sthis, name, sopts.idx?.file ?? sopts.idx?.car ?? sopts.data?.file ?? sopts.data?.car, base, "file", {
+        file: true,
+        idx: true
+      }),
+      meta: ensureURIDefaults(sthis, name, sopts.idx?.meta ?? sopts.data?.meta, base, "meta", { idx: true }),
+      wal: ensureURIDefaults(sthis, name, sopts.idx?.wal ?? sopts.data?.wal, base, "wal", { idx: true })
     },
     data: {
-      car: defaultURI2(sthis, name, sopts.data?.data, base, "data"),
-      file: defaultURI2(sthis, name, sopts.data?.data, base, "data", { file: true }),
-      meta: defaultURI2(sthis, name, sopts.data?.meta, base, "meta"),
-      wal: defaultURI2(sthis, name, sopts.data?.wal, base, "wal")
+      car: ensureURIDefaults(sthis, name, sopts.data?.car, base, "car"),
+      file: ensureURIDefaults(sthis, name, sopts.data?.file ?? sopts.data?.car, base, "file", { file: true }),
+      meta: ensureURIDefaults(sthis, name, sopts.data?.meta, base, "meta"),
+      wal: ensureURIDefaults(sthis, name, sopts.data?.wal, base, "wal")
     }
   };
 }
@@ -5072,6 +5299,6 @@ __export(file_exports, {
 
 // src/version.ts
 var PACKAGE_VERSION = Object.keys({
-  "0.20.0-dev-preview-41": "xxxx"
+  "0.20.0-dev-preview-50": "xxxx"
 })[0];
 //# sourceMappingURL=index.cjs.map