@fireproof/core 0.19.9-dev-frag → 0.19.11-dev-dryrun2

package/index.cjs

@@ -117,6 +117,21 @@ function ensureLogger(sthis, componentName, ctx) {
     sthis.env.onSet(
       (key, value) => {
         switch (key) {
+          case "FP_FORMAT": {
+            switch (value) {
+              case "jsonice":
+                logger.SetFormatter(new import_cement.JSONFormatter(logger.TxtEnDe(), 2));
+                break;
+              case "yaml":
+                logger.SetFormatter(new import_cement.YAMLFormatter(logger.TxtEnDe(), 2));
+                break;
+              case "json":
+              default:
+                logger.SetFormatter(new import_cement.JSONFormatter(logger.TxtEnDe()));
+                break;
+            }
+            break;
+          }
           case "FP_DEBUG":
             logger.SetDebug(value || []);
             break;
@@ -125,6 +140,7 @@ function ensureLogger(sthis, componentName, ctx) {
             break;
         }
       },
+      "FP_FORMAT",
       "FP_DEBUG",
       "FP_STACK"
     );
@@ -170,9 +186,6 @@ function getName(sthis, url) {
   }
   return result;
 }
-function exception2Result(fn) {
-  return fn().then((value) => import_cement.Result.Ok(value)).catch((e) => import_cement.Result.Err(e));
-}
 async function exceptionWrapper(fn) {
   return fn().catch((e) => import_cement.Result.Err(e));
 }
@@ -230,6 +243,16 @@ var init_utils = __esm({
           bin
         };
       }
+      timeOrderedNextId(now) {
+        now = typeof now === "number" ? now : (/* @__PURE__ */ new Date()).getTime();
+        const t = (281474976710656 + now).toString(16).replace(/^1/, "");
+        const bin = this.crypto.randomBytes(10);
+        bin[1] = bin[1] & 240 | (bin[1] | 8 && 11);
+        const hex = Array.from(bin).map((i) => i.toString(16).padStart(2, "0")).join("");
+        return {
+          str: `${t.slice(0, 8)}-${t.slice(8)}-7${hex.slice(0, 3)}-${hex.slice(3, 7)}-${hex.slice(7, 19)}`
+        };
+      }
       start() {
         return Promise.resolve();
       }
@@ -318,11 +341,12 @@ var node_filesystem_exports = {};
 __export(node_filesystem_exports, {
   NodeFileSystem: () => NodeFileSystem
 });
-var NodeFileSystem;
+var import_cement4, NodeFileSystem;
 var init_node_filesystem = __esm({
   "src/runtime/gateways/file/node-filesystem.ts"() {
     "use strict";
     init_utils2();
+    import_cement4 = require("@adviser/cement");
     NodeFileSystem = class {
       async start() {
         this.fs = await import("fs/promises");
@@ -351,6 +375,9 @@ var init_node_filesystem = __esm({
         return this.fs?.unlink(path);
       }
       async writefile(path, data) {
+        if ((0, import_cement4.runtimeFn)().isDeno) {
+          return this.fs?.writeFile(path, data);
+        }
         return this.fs?.writeFile(path, Buffer.from(data));
       }
     };
@@ -375,11 +402,19 @@ async function getFileSystem(url) {
         fs2 = new MemFileSystem2();
       }
       break;
-    case "node":
-    case "sys":
-    default: {
+    // case 'deno': {
+    //   const { DenoFileSystem } = await import("./deno-filesystem.js");
+    //   fs = new DenoFileSystem();
+    //   break;
+    // }
+    case "node": {
       const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
       fs2 = new NodeFileSystem2();
+      break;
+    }
+    case "sys":
+    default: {
+      return getFileSystem(url.build().setParam("fs", "node").URI());
     }
   }
   return fs2.start();
@@ -489,16 +524,16 @@ var key_bag_indexdb_exports = {};
 __export(key_bag_indexdb_exports, {
   KeyBagProviderIndexDB: () => KeyBagProviderIndexDB
 });
-var import_idb, import_cement4, KeyBagProviderIndexDB;
+var import_idb, import_cement5, KeyBagProviderIndexDB;
 var init_key_bag_indexdb = __esm({
   "src/runtime/key-bag-indexdb.ts"() {
     "use strict";
     import_idb = require("idb");
     init_utils2();
-    import_cement4 = require("@adviser/cement");
+    import_cement5 = require("@adviser/cement");
     KeyBagProviderIndexDB = class {
       constructor(url, sthis) {
-        this._db = new import_cement4.ResolveOnce();
+        this._db = new import_cement5.ResolveOnce();
         this.sthis = sthis;
         this.logger = sthis.logger;
         this.url = url;
@@ -552,15 +587,15 @@ __export(gateway_exports, {
   FileGateway: () => FileGateway,
   FileTestStore: () => FileTestStore
 });
-var import_cement8, versionFiles, FileGateway, FileTestStore;
+var import_cement11, versionFiles, FileGateway, FileTestStore;
 var init_gateway = __esm({
   "src/runtime/gateways/file/gateway.ts"() {
     "use strict";
     init_version();
-    import_cement8 = require("@adviser/cement");
+    import_cement11 = require("@adviser/cement");
     init_utils();
     init_utils2();
-    versionFiles = new import_cement8.KeyedResolvOnce();
+    versionFiles = new import_cement11.KeyedResolvOnce();
     FileGateway = class {
       get fs() {
         if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
@@ -590,7 +625,7 @@ var init_gateway = __esm({
         });
       }
       start(baseURL) {
-        return exception2Result(async () => {
+        return (0, import_cement11.exception2Result)(async () => {
           this._fs = await getFileSystem(baseURL);
           await this.fs.start();
           const url = baseURL.build();
@@ -605,10 +640,10 @@ var init_gateway = __esm({
         });
       }
       async buildUrl(baseUrl, key) {
-        return import_cement8.Result.Ok(baseUrl.build().setParam("key", key).URI());
+        return import_cement11.Result.Ok(baseUrl.build().setParam("key", key).URI());
       }
       async close() {
-        return import_cement8.Result.Ok(void 0);
+        return import_cement11.Result.Ok(void 0);
       }
       // abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
       getFilePath(url) {
@@ -617,7 +652,7 @@ var init_gateway = __esm({
         return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
       }
       async put(url, body) {
-        return exception2Result(async () => {
+        return (0, import_cement11.exception2Result)(async () => {
           const file = await this.getFilePath(url);
           this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
           await this.fs.writefile(file, body);
@@ -629,17 +664,17 @@ var init_gateway = __esm({
           try {
             const res = await this.fs.readfile(file);
             this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get");
-            return import_cement8.Result.Ok(new Uint8Array(res));
+            return import_cement11.Result.Ok(new Uint8Array(res));
           } catch (e) {
             if (isNotFoundError(e)) {
-              return import_cement8.Result.Err(new NotFoundError(`file not found: ${file}`));
+              return import_cement11.Result.Err(new NotFoundError(`file not found: ${file}`));
             }
-            return import_cement8.Result.Err(e);
+            return import_cement11.Result.Err(e);
           }
         });
       }
       async delete(url) {
-        return exception2Result(async () => {
+        return (0, import_cement11.exception2Result)(async () => {
           await this.fs.unlink(this.getFilePath(url));
         });
       }
@@ -665,7 +700,7 @@ var init_gateway = __esm({
             }
           }
         }
-        return import_cement8.Result.Ok(void 0);
+        return import_cement11.Result.Ok(void 0);
       }
     };
     FileTestStore = class {
@@ -754,15 +789,15 @@ function getIndexDBName(iurl, sthis) {
     dbName
   };
 }
-var import_idb2, import_cement9, onceIndexDB, IndexDBGateway, IndexDBTestStore;
+var import_idb2, import_cement12, onceIndexDB, IndexDBGateway, IndexDBTestStore;
 var init_gateway2 = __esm({
   "src/runtime/gateways/indexdb/gateway.ts"() {
     "use strict";
     import_idb2 = require("idb");
-    import_cement9 = require("@adviser/cement");
+    import_cement12 = require("@adviser/cement");
     init_version2();
     init_utils();
-    onceIndexDB = new import_cement9.KeyedResolvOnce();
+    onceIndexDB = new import_cement12.KeyedResolvOnce();
     IndexDBGateway = class {
       constructor(sthis) {
         this._db = {};
@@ -770,7 +805,7 @@ var init_gateway2 = __esm({
         this.sthis = sthis;
       }
       async start(baseURL) {
-        return exception2Result(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           this.logger.Debug().Url(baseURL).Msg("starting");
           await this.sthis.start();
           const ic = await connectIdb(baseURL, this.sthis);
@@ -780,10 +815,10 @@ var init_gateway2 = __esm({
         });
       }
       async close() {
-        return import_cement9.Result.Ok(void 0);
+        return import_cement12.Result.Ok(void 0);
       }
       async destroy(baseUrl) {
-        return exception2Result(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           const type = getStore(baseUrl, this.sthis, joinDBName).name;
           const idb = this._db;
           const trans = idb.transaction(type, "readwrite");
@@ -799,7 +834,7 @@ var init_gateway2 = __esm({
         });
       }
       buildUrl(baseUrl, key) {
-        return Promise.resolve(import_cement9.Result.Ok(baseUrl.build().setParam("key", key).URI()));
+        return Promise.resolve(import_cement12.Result.Ok(baseUrl.build().setParam("key", key).URI()));
       }
       async get(url) {
         return exceptionWrapper(async () => {
@@ -810,13 +845,13 @@ var init_gateway2 = __esm({
           const bytes = await tx.objectStore(store).get(sanitzeKey(key));
           await tx.done;
           if (!bytes) {
-            return import_cement9.Result.Err(new NotFoundError(`missing ${key}`));
+            return import_cement12.Result.Err(new NotFoundError(`missing ${key}`));
           }
-          return import_cement9.Result.Ok(bytes);
+          return import_cement12.Result.Ok(bytes);
         });
       }
       async put(url, value) {
-        return exception2Result(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           const key = getKey(url, this.logger);
           const store = getStore(url, this.sthis, joinDBName).name;
           this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
@@ -826,14 +861,14 @@ var init_gateway2 = __esm({
         });
       }
       async delete(url) {
-        return exception2Result(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           const key = getKey(url, this.logger);
           const store = getStore(url, this.sthis, joinDBName).name;
           this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
           const tx = this._db.transaction([store], "readwrite");
           await tx.objectStore(store).delete(sanitzeKey(key));
           await tx.done;
-          return import_cement9.Result.Ok(void 0);
+          return import_cement12.Result.Ok(void 0);
         });
       }
     };
@@ -873,7 +908,6 @@ __export(src_exports, {
   ensureLogger: () => ensureLogger,
   ensureSuperLog: () => ensureSuperLog,
   ensureSuperThis: () => ensureSuperThis,
-  exception2Result: () => exception2Result,
   exceptionWrapper: () => exceptionWrapper,
   falsyToUndef: () => falsyToUndef,
   fireproof: () => fireproof,
@@ -890,7 +924,7 @@ __export(src_exports, {
 module.exports = __toCommonJS(src_exports);
 
 // src/database.ts
-var import_cement13 = require("@adviser/cement");
+var import_cement16 = require("@adviser/cement");
 
 // src/write-queue.ts
 function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -933,7 +967,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
 }
 
 // src/crdt.ts
-var import_cement12 = require("@adviser/cement");
+var import_cement15 = require("@adviser/cement");
 
 // src/runtime/wait-pr-multiformats/block.ts
 var block_exports = {};
@@ -1024,10 +1058,12 @@ __export(blockstore_exports, {
   EncryptedBlockstore: () => EncryptedBlockstore,
   FragmentGateway: () => FragmentGateway,
   Loader: () => Loader,
+  addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
   ensureStart: () => ensureStart,
   getGatewayFromURL: () => getGatewayFromURL,
   parseCarFile: () => parseCarFile,
   registerStoreProtocol: () => registerStoreProtocol,
+  setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
   testStoreFactory: () => testStoreFactory,
   toCIDBlock: () => toCIDBlock,
   toStoreRuntime: () => toStoreRuntime
@@ -1039,7 +1075,7 @@ function toCIDBlock(block) {
 }
 
 // src/blockstore/store-factory.ts
-var import_cement10 = require("@adviser/cement");
+var import_cement13 = require("@adviser/cement");
 
 // src/runtime/files.ts
 var files_exports = {};
@@ -1114,8 +1150,8 @@ var UnixFSFileBuilder = class {
 
 // src/blockstore/store.ts
 var import_p_limit2 = __toESM(require("p-limit"), 1);
-var import_dag_json = require("@ipld/dag-json");
-var import_cement7 = require("@adviser/cement");
+var import_dag_json2 = require("@ipld/dag-json");
+var import_cement10 = require("@adviser/cement");
 
 // src/types.ts
 function isFalsy(value) {
@@ -1140,7 +1176,7 @@ init_utils();
 // src/blockstore/loader.ts
 var import_p_limit = __toESM(require("p-limit"), 1);
 var import_car = require("@ipld/car");
-var import_cement6 = require("@adviser/cement");
+var import_cement7 = require("@adviser/cement");
 
 // src/blockstore/loader-helpers.ts
 var import_sha2 = require("multiformats/hashes/sha2");
@@ -1411,34 +1447,38 @@ __export(key_bag_exports, {
   getKeyBag: () => getKeyBag,
   registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
 });
-var import_cement5 = require("@adviser/cement");
+var import_cement6 = require("@adviser/cement");
 init_utils();
 var import_base582 = require("multiformats/bases/base58");
 var KeyBag = class {
   constructor(rt) {
     this.rt = rt;
-    this._seq = new import_cement5.ResolveSeq();
-    this.logger = ensureLogger(rt.sthis, "KeyBag", {
-      id: rt.id()
-    });
+    this._warnOnce = new import_cement6.ResolveOnce();
+    this._seq = new import_cement6.ResolveSeq();
+    this.logger = ensureLogger(rt.sthis, "KeyBag");
     this.logger.Debug().Msg("KeyBag created");
   }
   async subtleKey(key) {
+    const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
+    if (extractable) {
+      this._warnOnce.once(
+        () => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
+      );
+    }
     return await this.rt.crypto.importKey(
       "raw",
       // raw or jwk
       import_base582.base58btc.decode(key),
       // hexStringToUint8Array(key), // raw data
       "AES-GCM",
-      false,
-      // extractable
+      extractable,
       ["encrypt", "decrypt"]
     );
   }
   async ensureKeyFromUrl(url, keyFactory) {
     const storeKey = url.getParam("storekey");
     if (storeKey === "insecure") {
-      return import_cement5.Result.Ok(url);
+      return import_cement6.Result.Ok(url);
     }
     if (!storeKey) {
       const keyName = `@${keyFactory()}@`;
@@ -1447,7 +1487,7 @@ var KeyBag = class {
         return ret;
       }
       const urb = url.build().setParam("storekey", keyName);
-      return import_cement5.Result.Ok(urb.URI());
+      return import_cement6.Result.Ok(urb.URI());
     }
     if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
       const ret = await this.getNamedKey(storeKey);
@@ -1455,13 +1495,13 @@ var KeyBag = class {
         return ret;
       }
     }
-    return import_cement5.Result.Ok(url);
+    return import_cement6.Result.Ok(url);
   }
   async toKeyWithFingerPrint(keyStr) {
     const material = import_base582.base58btc.decode(keyStr);
     const key = await this.subtleKey(keyStr);
     const fpr = await this.rt.crypto.digestSHA256(material);
-    return import_cement5.Result.Ok({
+    return import_cement6.Result.Ok({
       key,
       fingerPrint: import_base582.base58btc.encode(new Uint8Array(fpr))
     });
@@ -1480,6 +1520,23 @@ var KeyBag = class {
     await bag.set(name, item);
     return await this.toKeyWithFingerPrint(item.key);
   }
+  async getNamedExtractableKey(name, failIfNotFound = false) {
+    const ret = await this.getNamedKey(name, failIfNotFound);
+    if (ret.isErr()) {
+      return ret;
+    }
+    const named = ret.Ok();
+    return import_cement6.Result.Ok({
+      ...named,
+      extract: async () => {
+        const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
+        return {
+          key: ext,
+          keyStr: import_base582.base58btc.encode(ext)
+        };
+      }
+    });
+  }
   async getNamedKey(name, failIfNotFound = false) {
     const id = this.rt.sthis.nextId(4).str;
     return this._seq.add(async () => {
@@ -1492,7 +1549,7 @@ var KeyBag = class {
       }
       if (failIfNotFound) {
         this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
-        return import_cement5.Result.Err(new Error(`Key not found: ${name}`));
+        return import_cement6.Result.Err(new Error(`Key not found: ${name}`));
       }
       const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
       this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
@@ -1532,56 +1589,44 @@ function defaultKeyBagOpts(sthis, kbo) {
   const logger = ensureLogger(sthis, "KeyBag");
   let url;
   if (kbo.url) {
-    url = import_cement5.URI.from(kbo.url);
+    url = import_cement6.URI.from(kbo.url);
     logger.Debug().Url(url).Msg("from opts");
   } else {
     let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
-    if ((0, import_cement5.runtimeFn)().isBrowser) {
-      url = import_cement5.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
+    if ((0, import_cement6.runtimeFn)().isBrowser) {
+      url = import_cement6.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
     } else {
       if (!bagFnameOrUrl) {
         const home = sthis.env.get("HOME");
         bagFnameOrUrl = `${home}/.fireproof/keybag`;
-        url = import_cement5.URI.from(`file://${bagFnameOrUrl}`);
+        url = import_cement6.URI.from(`file://${bagFnameOrUrl}`);
       } else {
-        url = import_cement5.URI.from(bagFnameOrUrl);
+        url = import_cement6.URI.from(bagFnameOrUrl);
       }
     }
     logger.Debug().Url(url).Msg("from env");
   }
-  let keyProviderFactory;
-  switch (url.protocol) {
-    case "file:":
-      keyProviderFactory = async () => {
-        const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
-        return new KeyBagProviderFile2(url, sthis);
-      };
-      break;
-    case "indexdb:":
-      keyProviderFactory = async () => {
-        const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
-        return new KeyBagProviderIndexDB2(url, sthis);
-      };
-      break;
-    default:
-      throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
+  const kitem = keyBagProviderFactories.get(url.protocol);
+  if (!kitem) {
+    throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
   }
+  const getBag = async () => kitem.factory(url, sthis);
   if (url.hasParam("masterkey")) {
     throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
   }
   return {
     url,
-    crypto: kbo.crypto || (0, import_cement5.toCryptoRuntime)({}),
+    crypto: kbo.crypto || (0, import_cement6.toCryptoRuntime)({}),
     sthis,
     logger,
     keyLength: kbo.keyLength || 16,
-    getBag: keyProviderFactory,
+    getBag,
     id: () => {
       return url.toString();
     }
   };
 }
-var _keyBags = new import_cement5.KeyedResolvOnce();
+var _keyBags = new import_cement6.KeyedResolvOnce();
 async function getKeyBag(sthis, kbo = {}) {
   await sthis.start();
   const rt = defaultKeyBagOpts(sthis, kbo);
@@ -1669,7 +1714,7 @@ async function commit(params, t, done, opts = { noLoader: false, compact: false
 }
 async function prepareCarFiles(encoder, threshold, rootBlock, t) {
   const carFiles = [];
-  threshold = threshold || 1e3 * 1e3;
+  threshold = threshold || 128e3 * 8;
   let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
   clonedt.putSync(rootBlock.cid, rootBlock.bytes);
   let newsize = CBW.blockLength(toCIDBlock(rootBlock));
@@ -1692,6 +1737,54 @@ async function prepareCarFiles(encoder, threshold, rootBlock, t) {
 
 // src/blockstore/loader.ts
 var import_sha23 = require("multiformats/hashes/sha2");
+
+// src/blockstore/task-manager.ts
+init_utils();
+var TaskManager = class {
+  constructor(sthis, callback) {
+    this.eventsWeHandled = /* @__PURE__ */ new Set();
+    this.queue = [];
+    this.isProcessing = false;
+    this.logger = ensureLogger(sthis, "TaskManager");
+    this.callback = callback;
+  }
+  async handleEvent(cid, parents, dbMeta) {
+    for (const parent of parents) {
+      this.eventsWeHandled.add(parent.toString());
+    }
+    this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
+    this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
+    void this.processQueue();
+  }
+  async processQueue() {
+    if (this.isProcessing) return;
+    this.isProcessing = true;
+    const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
+    const first = filteredQueue[0];
+    if (!first) {
+      return;
+    }
+    try {
+      await this.callback(first.dbMeta);
+      this.eventsWeHandled.add(first.cid);
+      this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
+    } catch (err) {
+      if (first.retries++ > 3) {
+        this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
+        this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
+    } finally {
+      this.isProcessing = false;
+      if (this.queue.length > 0) {
+        void this.processQueue();
+      }
+    }
+  }
+};
+
+// src/blockstore/loader.ts
 function carLogIncludesGroup(list, cids) {
   return list.some((arr) => {
     return arr.toString() === cids.toString();
@@ -1716,7 +1809,7 @@ var Loader = class {
     this.getBlockCache = /* @__PURE__ */ new Map();
     this.seenMeta = /* @__PURE__ */ new Set();
     this.writeLimit = (0, import_p_limit.default)(1);
-    this.onceReady = new import_cement6.ResolveOnce();
+    this.onceReady = new import_cement7.ResolveOnce();
     this.name = name;
     this.sthis = sthis;
     this.ebOpts = defaultedBlockstoreRuntime(
@@ -1728,6 +1821,9 @@ var Loader = class {
       "Loader"
     );
     this.logger = this.ebOpts.logger;
+    this.taskManager = new TaskManager(sthis, async (dbMeta) => {
+      await this.handleDbMetasFromStore([dbMeta]);
+    });
   }
   // readonly id = uuidv4();
   async keyBag() {
@@ -1747,8 +1843,10 @@ var Loader = class {
   }
   async ready() {
     return this.onceReady.once(async () => {
-      const metas = this.ebOpts.meta ? [this.ebOpts.meta] : await (await this.metaStore()).load("main");
-      if (metas) {
+      const metas = await (await this.metaStore()).load();
+      if (this.ebOpts.meta) {
+        await this.handleDbMetasFromStore([this.ebOpts.meta]);
+      } else if (metas) {
         await this.handleDbMetasFromStore(metas);
       }
     });
@@ -1829,7 +1927,8 @@ var Loader = class {
       carLog: this.carLog,
       carStore: fstore,
       WALStore: await this.WALStore(),
-      metaStore: await this.metaStore()
+      metaStore: await this.metaStore(),
+      threshold: this.ebOpts.threshold
     };
     return this.commitQueue.enqueue(async () => {
       await this.cacheTransaction(t);
@@ -2054,7 +2153,7 @@ var generateIV = {
     calc: async (ko, crypto, data) => {
       const hash = await import_sha24.sha256.digest(data);
       const hashBytes = new Uint8Array(hash.bytes);
-      const hashArray = new Uint8Array(ko.ivLength * 8);
+      const hashArray = new Uint8Array(ko.ivLength);
       for (let i = 0; i < hashBytes.length; i++) {
         hashArray[i % ko.ivLength] ^= hashBytes[i];
       }
@@ -2132,7 +2231,7 @@ var keyedCrypto = class {
     };
   }
   async _decrypt(data) {
-    this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("decrypting");
+    this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
     return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
   }
   async _encrypt(data) {
@@ -2203,8 +2302,10 @@ async function keyedCryptoFactory(url, kb, sthis) {
 }
 
 // src/blockstore/fragment-gateway.ts
+var import_cement8 = require("@adviser/cement");
 var import_base584 = require("multiformats/bases/base58");
 var import_cborg = require("cborg");
+init_utils();
 function getFragSize(url) {
   const fragSize = url.getParam("fragSize");
   let ret = 0;
@@ -2225,7 +2326,7 @@ async function getFrags(url, innerGW, headerSize, logger) {
     }
     const data = res.unwrap();
     return [
-      import_cement.Result.Ok({
+      import_cement8.Result.Ok({
         fid: new Uint8Array(0),
         ofs: 0,
         len: data.length,
@@ -2239,7 +2340,7 @@ async function getFrags(url, innerGW, headerSize, logger) {
   }
   const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
   const blockSize = firstFragment.data.length;
-  const ops = [Promise.resolve(import_cement.Result.Ok(firstFragment))];
+  const ops = [Promise.resolve(import_cement8.Result.Ok(firstFragment))];
   const fidStr = import_base584.base58btc.encode(firstFragment.fid);
   const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
   for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
@@ -2251,12 +2352,12 @@ async function getFrags(url, innerGW, headerSize, logger) {
         }
         const fragment = (0, import_cborg.decode)(raw2.unwrap());
         if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
-          return import_cement.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
+          return import_cement8.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
         }
         if (fragment.ofs !== ofs2) {
-          return import_cement.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
+          return import_cement8.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
         }
-        return import_cement.Result.Ok(fragment);
+        return import_cement8.Result.Ok(fragment);
       })(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
     );
   }
@@ -2318,42 +2419,156 @@ var FragmentGateway = class {
   }
   async put(url, body) {
     await Promise.all(this.slicer(url, body));
-    return import_cement.Result.Ok(void 0);
+    return import_cement8.Result.Ok(void 0);
   }
   async get(url) {
     const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
     let buffer = void 0;
     for (const rfrag of rfrags) {
       if (rfrag.isErr()) {
-        return import_cement.Result.Err(rfrag.Err());
+        return import_cement8.Result.Err(rfrag.Err());
       }
       const frag = rfrag.Ok();
       buffer = buffer || new Uint8Array(frag.len);
       buffer.set(frag.data, frag.ofs);
     }
-    return import_cement.Result.Ok(buffer || new Uint8Array(0));
+    return import_cement8.Result.Ok(buffer || new Uint8Array(0));
+  }
+  async subscribe(url, callback) {
+    if (this.innerGW.subscribe) {
+      return this.innerGW.subscribe(url, callback);
+    } else {
+      return import_cement8.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
+    }
   }
   async delete(url) {
     const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
     for (const rfrag of rfrags) {
       if (rfrag.isErr()) {
-        return import_cement.Result.Err(rfrag.Err());
+        return import_cement8.Result.Err(rfrag.Err());
       }
       const frag = rfrag.Ok();
       const fidStr = import_base584.base58btc.encode(frag.fid);
       const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
       await this.innerGW.delete(fragUrl);
     }
-    return import_cement.Result.Ok(void 0);
+    return import_cement8.Result.Ok(void 0);
   }
 };
 
+// src/blockstore/meta-key-helper.ts
+var import_dag_json = require("@ipld/dag-json");
+var import_clock = require("@web3-storage/pail/clock");
+var import_multiformats2 = require("multiformats");
+var import_base64 = require("multiformats/bases/base64");
+var import_cement9 = require("@adviser/cement");
+async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
+  const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
+  if (!crdtEntries.length) {
+    sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
+    return [];
+  }
+  if (!crdtEntries.map) {
+    sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
+    return [];
+  }
+  return Promise.all(
+    crdtEntries.map(async (crdtEntry) => {
+      const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
+      const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
+      return {
+        eventCid: eventBlock.cid,
+        parents: crdtEntry.parents,
+        dbMeta
+      };
+    })
+  );
+}
+async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
+  try {
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
+    const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
+    if (keyInfo.length) {
+      const dbMeta = keyInfo[0].dbMeta;
+      if (dbMeta.key) {
+        const kb = await getKeyBag(sthis);
+        const keyName = getStoreKeyName(uri);
+        const res = await kb.setNamedKey(keyName, dbMeta.key);
+        if (res.isErr()) {
+          sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
+          throw res.Err();
+        }
+      }
+      sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
+      return import_cement9.Result.Ok(dbMeta);
+    }
+    sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
+    return import_cement9.Result.Ok(void 0);
+  } catch (error) {
+    sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
+    return import_cement9.Result.Err(error);
+  }
+}
+async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
+  try {
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
+    const keyName = getStoreKeyName(uri);
+    const kb = await getKeyBag(sthis);
+    const res = await kb.getNamedExtractableKey(keyName, true);
+    if (res.isErr()) {
+      sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
+      throw res.Err();
+    }
+    const keyData = await res.Ok().extract();
+    const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
+    const { dbMeta, parents } = dbMetas[0];
+    const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
+    dbMeta.key = keyData.keyStr;
+    const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
+    const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
+    return import_cement9.Result.Ok(encoded);
+  } catch (error) {
+    sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
+    return import_cement9.Result.Err(error);
+  }
+}
+function getStoreKeyName(url) {
+  const storeKeyName = [url.getParam("localName") || url.getParam("name")];
+  const idx = url.getParam("index");
+  if (idx) {
+    storeKeyName.push(idx);
+  }
+  storeKeyName.push("data");
+  return `@${storeKeyName.join(":")}@`;
+}
+async function createDbMetaEventBlock(sthis, dbMeta, parents) {
+  const event = await import_clock.EventBlock.create(
+    {
+      dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
+    },
+    parents
+  );
+  return event;
+}
+async function encodeEventsWithParents(sthis, events, parents) {
+  const crdtEntries = events.map((event) => {
+    const base64String = import_base64.base64pad.encode(event.bytes);
+    return {
+      cid: event.cid.toString(),
+      data: base64String,
+      parents: parents.map((p) => p.toString())
+    };
+  });
+  return sthis.txt.encode(JSON.stringify(crdtEntries));
+}
+
 // src/blockstore/store.ts
 function guardVersion(url) {
   if (!url.hasParam("version")) {
-    return import_cement7.Result.Err(`missing version: ${url.toString()}`);
+    return import_cement10.Result.Err(`missing version: ${url.toString()}`);
   }
-  return import_cement7.Result.Ok(url);
+  return import_cement10.Result.Ok(url);
 }
 var BaseStoreImpl = class {
   constructor(name, url, opts, sthis, logger) {
@@ -2365,6 +2580,7 @@ var BaseStoreImpl = class {
     this.sthis = sthis;
     this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
     this.gateway = new FragmentGateway(this.sthis, opts.gateway);
+    this.loader = opts.loader;
   }
   url() {
     return this._url;
@@ -2375,6 +2591,9 @@ var BaseStoreImpl = class {
   onClosed(fn) {
     this._onClosed.push(fn);
   }
+  async ready() {
+    return;
+  }
   async keyedCrypto() {
     return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
   }
@@ -2409,7 +2628,7 @@ var BaseStoreImpl = class {
     }
     if (this.ready) {
       const fn = this.ready.bind(this);
-      const ready = await exception2Result(fn);
+      const ready = await (0, import_cement10.exception2Result)(fn);
       if (ready.isErr()) {
         await this.close();
         return ready;
@@ -2421,67 +2640,44 @@ var BaseStoreImpl = class {
   }
 };
 var MetaStoreImpl = class extends BaseStoreImpl {
+  // remote: boolean;
   constructor(sthis, name, url, opts) {
-    super(
-      name,
-      url,
-      {
-        ...opts
-      },
-      sthis,
-      ensureLogger(sthis, "MetaStoreImpl")
-    );
+    super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
     this.storeType = "meta";
     this.subscribers = /* @__PURE__ */ new Map();
-  }
-  onLoad(branch, loadHandler) {
-    const subscribers = this.subscribers.get(branch) || [];
-    subscribers.push(loadHandler);
-    this.subscribers.set(branch, subscribers);
-    return () => {
-      const subscribers2 = this.subscribers.get(branch) || [];
-      const idx = subscribers2.indexOf(loadHandler);
-      if (idx > -1) subscribers2.splice(idx, 1);
-    };
-  }
-  makeHeader({ cars }) {
-    const toEncode = { cars };
-    return (0, import_dag_json.format)(toEncode);
-  }
-  parseHeader(headerData) {
-    const got = (0, import_dag_json.parse)(headerData);
-    return got;
-  }
-  async handleSubscribers(dbMetas, branch) {
-    try {
-      const subscribers = this.subscribers.get(branch) || [];
-      await Promise.all(subscribers.map((subscriber) => subscriber(dbMetas)));
-    } catch (e) {
-      this.logger.Error().Err(e).Msg("handleSubscribers").AsError();
+    this.parents = [];
+    if (
+      /*this.remote && */
+      opts.gateway.subscribe
+    ) {
+      this.onStarted(async () => {
+        this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
+        opts.gateway.subscribe?.(this.url(), async (message) => {
+          this.logger.Debug().Msg("Received message from gateway");
+          const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
+          await Promise.all(
+            dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
+          );
+          this.updateParentsFromDbMetas(dbMetas);
+        });
+      });
     }
   }
-  async handleByteHeads(byteHeads, branch = "main") {
-    let dbMetas;
-    try {
-      dbMetas = this.dbMetasForByteHeads(byteHeads);
-    } catch (e) {
-      throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
-    }
-    await this.handleSubscribers(dbMetas, branch);
-    return dbMetas;
+  updateParentsFromDbMetas(dbMetas) {
+    const cids = dbMetas.map((m) => m.eventCid);
+    const dbMetaParents = dbMetas.flatMap((m) => m.parents);
+    const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
+    const dbMetaParentsSet = new Set(dbMetaParents.map((p) => p.toString()));
+    this.parents = Array.from(uniqueParentsMap.values()).filter((p) => !dbMetaParentsSet.has(p.toString()));
   }
-  dbMetasForByteHeads(byteHeads) {
-    return byteHeads.map((bytes) => {
-      const txt = this.sthis.txt.decode(bytes);
-      return this.parseHeader(txt);
-    });
+  async handleByteHeads(byteHeads) {
+    return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
   }
-  async load(branch) {
-    branch = branch || "main";
-    this.logger.Debug().Str("branch", branch).Msg("loading");
+  async load() {
+    const branch = "main";
     const url = await this.gateway.buildUrl(this.url(), branch);
     if (url.isErr()) {
-      throw this.logger.Error().Result("buidUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
+      throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
     }
     const bytes = await this.gateway.get(url.Ok());
     if (bytes.isErr()) {
@@ -2490,27 +2686,31 @@ var MetaStoreImpl = class extends BaseStoreImpl {
       }
       throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
     }
-    return this.handleByteHeads([bytes.Ok()], branch);
+    const dbMetas = await this.handleByteHeads(bytes.Ok());
+    await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
+    this.updateParentsFromDbMetas(dbMetas);
+    return dbMetas.map((m) => m.dbMeta);
   }
   async save(meta, branch) {
     branch = branch || "main";
     this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
-    const bytes = this.makeHeader(meta);
+    const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
+    const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
     const url = await this.gateway.buildUrl(this.url(), branch);
     if (url.isErr()) {
       throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
     }
-    const res = await this.gateway.put(url.Ok(), this.sthis.txt.encode(bytes));
+    this.parents = [event.cid];
+    const res = await this.gateway.put(url.Ok(), bytes);
     if (res.isErr()) {
       throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
     }
-    await this.handleSubscribers([meta], branch);
     return res;
   }
   async close() {
     await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement7.Result.Ok(void 0);
+    return import_cement10.Result.Ok(void 0);
   }
   async destroy() {
     return this.gateway.destroy(this.url());
@@ -2519,15 +2719,7 @@ var MetaStoreImpl = class extends BaseStoreImpl {
 var DataStoreImpl = class extends BaseStoreImpl {
   // readonly tag: string = "car-base";
   constructor(sthis, name, url, opts) {
-    super(
-      name,
-      url,
-      {
-        ...opts
-      },
-      sthis,
-      ensureLogger(sthis, "DataStoreImpl")
-    );
+    super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
     this.storeType = "data";
   }
   async load(cid) {
@@ -2565,7 +2757,7 @@ var DataStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement7.Result.Ok(void 0);
+    return import_cement10.Result.Ok(void 0);
   }
   destroy() {
     return this.gateway.destroy(this.url());
@@ -2573,46 +2765,43 @@ var DataStoreImpl = class extends BaseStoreImpl {
 };
 var WALStoreImpl = class extends BaseStoreImpl {
   constructor(loader, url, opts) {
-    super(
-      loader.name,
-      url,
-      {
-        ...opts
-      },
-      loader.sthis,
-      ensureLogger(loader.sthis, "WALStoreImpl")
-    );
+    super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
     this.storeType = "wal";
-    this._ready = new import_cement7.ResolveOnce();
+    this._ready = new import_cement10.ResolveOnce();
     this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
     this.processing = void 0;
     this.processQueue = new CommitQueue();
-    this.ready = async () => {
-      return this._ready.once(async () => {
-        const walState = await this.load().catch((e) => {
-          this.logger.Error().Any("error", e).Msg("error loading wal");
-          return void 0;
-        });
-        if (!walState) {
-          this.walState.operations = [];
-          this.walState.fileOperations = [];
-        } else {
-          this.walState.operations = walState.operations || [];
-          this.walState.fileOperations = walState.fileOperations || [];
-        }
-      });
-    };
     this.loader = loader;
   }
+  async ready() {
+    return this._ready.once(async () => {
+      const walState = await this.load().catch((e) => {
+        this.logger.Error().Any("error", e).Msg("error loading wal");
+        return void 0;
+      });
+      if (!walState) {
+        this.walState.operations = [];
+        this.walState.fileOperations = [];
+      } else {
+        this.walState.operations = walState.operations || [];
+        this.walState.fileOperations = walState.fileOperations || [];
+      }
+    });
+  }
   async enqueue(dbMeta, opts) {
     await this.ready();
-    if (opts.noLoader) {
+    if (opts.compact) {
+      this.walState.operations = [];
+      this.walState.noLoaderOps = [dbMeta];
+    } else if (opts.noLoader) {
       this.walState.noLoaderOps.push(dbMeta);
     } else {
       this.walState.operations.push(dbMeta);
     }
     await this.save(this.walState);
-    void this.process();
+    if (!opts.noLoader) {
+      void this.process();
+    }
   }
   async enqueueFile(fileCid, publicFile = false) {
     await this.ready();
@@ -2639,7 +2828,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
       const fileOperations = [...this.walState.fileOperations];
       const uploads = [];
       const noLoaderOps = [...this.walState.noLoaderOps];
-      const limit = (0, import_p_limit2.default)(5);
+      const limit = (0, import_p_limit2.default)(3);
       if (operations.length + fileOperations.length + noLoaderOps.length === 0) return;
       for (const dbMeta of noLoaderOps) {
         const uploadP = limit(async () => {
@@ -2715,7 +2904,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
       throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
     }
     try {
-      return bytes && (0, import_dag_json.parse)(this.sthis.txt.decode(bytes.Ok()));
+      return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
     } catch (e) {
       throw this.logger.Error().Err(e).Msg("error parse").AsError();
     }
@@ -2727,7 +2916,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
     }
     let encoded;
     try {
-      encoded = (0, import_dag_json.format)(state);
+      encoded = (0, import_dag_json2.format)(state);
     } catch (e) {
       throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
     }
@@ -2739,7 +2928,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement7.Result.Ok(void 0);
+    return import_cement10.Result.Ok(void 0);
   }
   destroy() {
     return this.gateway.destroy(this.url());
@@ -2766,15 +2955,15 @@ function buildURL(optURL, loader) {
   const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
   let obuUrl;
   if (obuItem && obuItem.overrideBaseURL) {
-    obuUrl = import_cement10.URI.from(obuItem.overrideBaseURL);
+    obuUrl = import_cement13.URI.from(obuItem.overrideBaseURL);
   }
   const ret = ensureIsIndex(
-    import_cement10.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
+    import_cement13.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
     storeOpts.isIndex
   );
   return ret;
 }
-var onceGateway = new import_cement10.KeyedResolvOnce();
+var onceGateway = new import_cement13.KeyedResolvOnce();
 async function getGatewayFromURL(url, sthis) {
   return onceGateway.get(url.toString()).once(async () => {
     const item = storeFactory.get(url.protocol);
@@ -2816,52 +3005,61 @@ function registerStoreProtocol(item) {
     storeFactory.delete(protocol);
   };
 }
+var onceDataStoreFactory = new import_cement13.KeyedResolvOnce();
 async function dataStoreFactory(loader) {
   const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
   const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
-  const gateway = await getGatewayFromURL(url, sthis);
-  if (!gateway) {
-    throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
-  }
-  const store = new DataStoreImpl(sthis, loader.name, url, {
-    gateway: gateway.gateway,
-    keybag: () => getKeyBag(loader.sthis, {
-      ...loader.ebOpts.keyBag
-    })
+  return onceDataStoreFactory.get(url.toString()).once(async () => {
+    const gateway = await getGatewayFromURL(url, sthis);
+    if (!gateway) {
+      throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+    }
+    const store = new DataStoreImpl(sthis, loader.name, url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, {
+        ...loader.ebOpts.keyBag
+      })
+    });
+    return store;
   });
-  return store;
 }
+var onceMetaStoreFactory = new import_cement13.KeyedResolvOnce();
 async function metaStoreFactory(loader) {
   const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
   const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
-  sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
-  const gateway = await getGatewayFromURL(url, sthis);
-  if (!gateway) {
-    throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
-  }
-  const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
-    gateway: gateway.gateway,
-    keybag: () => getKeyBag(loader.sthis, {
-      ...loader.ebOpts.keyBag
-    })
+  return onceMetaStoreFactory.get(url.toString()).once(async () => {
+    sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
+    const gateway = await getGatewayFromURL(url, sthis);
+    if (!gateway) {
+      throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+    }
+    const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, {
+        ...loader.ebOpts.keyBag
+      })
+    });
+    return store;
   });
-  return store;
 }
+var onceRemoteWalFactory = new import_cement13.KeyedResolvOnce();
 async function remoteWalFactory(loader) {
   const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
   const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
-  const gateway = await getGatewayFromURL(url, sthis);
-  if (!gateway) {
-    throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
-  }
-  sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
-  const store = new WALStoreImpl(loader, url, {
-    gateway: gateway.gateway,
-    keybag: () => getKeyBag(loader.sthis, {
-      ...loader.ebOpts.keyBag
-    })
+  return onceRemoteWalFactory.get(url.toString()).once(async () => {
+    const gateway = await getGatewayFromURL(url, sthis);
+    if (!gateway) {
+      throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
+    }
+    sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
+    const store = new WALStoreImpl(loader, url, {
+      gateway: gateway.gateway,
+      keybag: () => getKeyBag(loader.sthis, {
+        ...loader.ebOpts.keyBag
+      })
+    });
+    return store;
   });
-  return store;
 }
 async function testStoreFactory(url, sthis) {
   sthis = ensureSuperLog(sthis, "testStoreFactory");
@@ -2900,79 +3098,27 @@ function toStoreRuntime(opts, sthis) {
 }
 registerStoreProtocol({
   protocol: "file:",
-  gateway: async (logger) => {
+  gateway: async (sthis) => {
     const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
-    return new FileGateway2(logger);
+    return new FileGateway2(sthis);
   },
-  test: async (logger) => {
+  test: async (sthis) => {
     const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
-    return new FileTestStore2(logger);
+    return new FileTestStore2(sthis);
   }
 });
 registerStoreProtocol({
   protocol: "indexdb:",
-  gateway: async (logger) => {
+  gateway: async (sthis) => {
     const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
-    return new IndexDBGateway2(logger);
+    return new IndexDBGateway2(sthis);
   },
-  test: async (logger) => {
+  test: async (sthis) => {
     const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
-    return new IndexDBTestStore2(logger);
+    return new IndexDBTestStore2(sthis);
   }
 });
 
-// src/blockstore/connection-base.ts
-var import_clock = require("@web3-storage/pail/clock");
-var import_block6 = require("@web3-storage/pail/block");
-
-// src/blockstore/task-manager.ts
-init_utils();
-var TaskManager = class {
-  constructor(loader) {
-    this.eventsWeHandled = /* @__PURE__ */ new Set();
-    this.queue = [];
-    this.isProcessing = false;
-    this.loader = loader;
-    this.logger = ensureLogger(loader.sthis, "TaskManager");
-  }
-  async handleEvent(eventBlock) {
-    const cid = eventBlock.cid.toString();
-    const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
-    for (const parent of parents) {
-      this.eventsWeHandled.add(parent);
-    }
-    this.queue.push({ cid, eventBlock, retries: 0 });
-    this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
-    void this.processQueue();
-  }
-  async processQueue() {
-    if (this.isProcessing) return;
-    this.isProcessing = true;
-    const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
-    const first = filteredQueue[0];
-    if (!first) {
-      return;
-    }
-    try {
-      this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
-      this.eventsWeHandled.add(first.cid);
-      this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
-    } catch (err) {
-      if (first.retries++ > 3) {
-        this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
-        this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
-      }
-      await new Promise((resolve) => setTimeout(resolve, 50));
-      throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
-    } finally {
-      this.isProcessing = false;
-      if (this.queue.length > 0) {
-        void this.processQueue();
-      }
-    }
-  }
-};
-
 // src/blockstore/store-remote.ts
 async function RemoteDataStore(sthis, name, url, opts) {
   const ds = new DataStoreImpl(sthis, name, url, opts);
@@ -2980,7 +3126,13 @@ async function RemoteDataStore(sthis, name, url, opts) {
   return ds;
 }
 async function RemoteMetaStore(sthis, name, url, opts) {
-  const ms = new MetaStoreImpl(sthis, name, url, opts);
+  const ms = new MetaStoreImpl(
+    sthis,
+    name,
+    url,
+    opts
+    /* , true*/
+  );
   await ms.start();
   return ms;
 }
@@ -2988,16 +3140,12 @@ async function RemoteMetaStore(sthis, name, url, opts) {
 // src/blockstore/connection-base.ts
 var ConnectionBase = class {
   constructor(url, logger) {
-    // readonly ready: Promise<unknown>;
-    // todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
-    this.eventBlocks = new import_block6.MemoryBlockstore();
-    this.parents = [];
     this.loaded = Promise.resolve();
     this.logger = logger;
     this.url = url;
   }
   async refresh() {
-    await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load("main");
+    await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
     await (await throwFalsy(this.loader).WALStore()).process();
   }
   async connect_X({ loader }) {
@@ -3008,26 +3156,20 @@ var ConnectionBase = class {
   async connectMeta_X({ loader }) {
     if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
     this.loader = loader;
-    this.taskManager = new TaskManager(loader);
     await this.onConnect();
     const metaUrl = this.url.build().defParam("store", "meta").URI();
     const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
     if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
-    const name = metaUrl.toString();
-    const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
+    const dbName = metaUrl.getParam("name");
+    if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
+    const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
       gateway: gateway.gateway,
-      keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag)
-    });
-    remote.onLoad("main", async (metas) => {
-      if (metas) {
-        this.logger.Debug().Any("metas", metas).Bool("loader", this.loader).Msg("connectMeta_X: handleDbMetasFromStore pre");
-        await throwFalsy(this.loader).handleDbMetasFromStore(metas);
-        this.logger.Debug().Any("metas", metas).Msg("connectMeta_X: handleDbMetasFromStore post");
-      }
+      keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
+      loader
     });
     this.loader.remoteMetaStore = remote;
     this.loaded = this.loader.ready().then(async () => {
-      remote.load("main").then(async () => {
+      remote.load().then(async () => {
         (await throwFalsy(this.loader).WALStore()).process();
       });
     });
@@ -3038,28 +3180,14 @@ var ConnectionBase = class {
     const dataUrl = this.url.build().defParam("store", "data").URI();
     const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
     if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
-    const name = dataUrl.toString();
+    const name = dataUrl.getParam("name");
+    if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
     loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
       gateway: gateway.gateway,
       keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
     });
     loader.remoteFileStore = loader.remoteCarStore;
   }
-  async createEventBlock(bytes) {
-    const data = {
-      dbMeta: bytes
-    };
-    const event = await import_clock.EventBlock.create(
-      data,
-      this.parents
-    );
-    await this.eventBlocks.put(event.cid, event.bytes);
-    return event;
-  }
-  async decodeEventBlock(bytes) {
-    const event = await (0, import_clock.decodeEventBlock)(bytes);
-    return event;
-  }
   // move this stuff to connect
   // async getDashboardURL(compact = true) {
   //   const baseUrl = 'https://dashboard.fireproof.storage/'
@@ -3364,7 +3492,7 @@ var import_sha26 = require("multiformats/hashes/sha2");
 var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
 var import_charwise = __toESM(require("charwise"), 1);
 var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
-var import_utils15 = require("prolly-trees/utils");
+var import_utils16 = require("prolly-trees/utils");
 var import_cache = require("prolly-trees/cache");
 var IndexTree = class {
 };
@@ -3372,17 +3500,17 @@ function refCompare(aRef, bRef) {
   if (Number.isNaN(aRef)) return -1;
   if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
   if (aRef === Infinity) return 1;
-  return (0, import_utils15.simpleCompare)(aRef, bRef);
+  return (0, import_utils16.simpleCompare)(aRef, bRef);
 }
 function compare(a, b) {
   const [aKey, aRef] = a;
   const [bKey, bRef] = b;
-  const comp = (0, import_utils15.simpleCompare)(aKey, bKey);
+  const comp = (0, import_utils16.simpleCompare)(aKey, bKey);
   if (comp !== 0) return comp;
   return refCompare(aRef, bRef);
 }
-var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
-var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils15.simpleCompare };
+var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
+var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils16.simpleCompare };
 function indexEntriesForChanges(changes, mapFn) {
   const indexEntries = [];
   changes.forEach(({ id: key, value, del }) => {
@@ -3690,7 +3818,7 @@ var Index = class {
 // src/crdt-clock.ts
 var import_clock3 = require("@web3-storage/pail/clock");
 var import_crdt2 = require("@web3-storage/pail/crdt");
-var import_cement11 = require("@adviser/cement");
+var import_cement14 = require("@adviser/cement");
 
 // src/apply-head-queue.ts
 function applyHeadQueue(worker, logger) {
@@ -3705,7 +3833,7 @@ function applyHeadQueue(worker, logger) {
         queue.sort((a, b) => b.updates ? 1 : -1);
         const task = queue.shift();
         if (!task) continue;
-        await worker(task.newHead, task.prevHead, task.updates !== null).catch((e) => {
+        await worker(task.newHead, task.prevHead, task.updates !== void 0).catch((e) => {
           throw logger.Error().Err(e).Msg("int_applyHead worker error").AsError();
         });
         if (task.updates) {
@@ -3748,7 +3876,7 @@ var CRDTClock = class {
     this.zoomers = /* @__PURE__ */ new Set();
     this.watchers = /* @__PURE__ */ new Set();
     this.emptyWatchers = /* @__PURE__ */ new Set();
-    this._ready = new import_cement11.ResolveOnce();
+    this._ready = new import_cement14.ResolveOnce();
     this.blockstore = blockstore;
     this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
     this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
@@ -3796,6 +3924,7 @@ var CRDTClock = class {
     this.zoomers.add(fn);
   }
   async int_applyHead(newHead, prevHead, localUpdates) {
+    const noLoader = !localUpdates;
     const ogHead = sortClockHead(this.head);
     newHead = sortClockHead(newHead);
     if (compareClockHeads(ogHead, newHead)) {
@@ -3806,7 +3935,6 @@ var CRDTClock = class {
       this.setHead(newHead);
       return;
     }
-    const noLoader = !localUpdates;
     if (!this.blockstore) {
       throw this.logger.Error().Msg("missing blockstore").AsError();
     }
@@ -3860,7 +3988,7 @@ init_utils();
 var CRDT = class {
   constructor(sthis, name, opts = {}) {
     this.indexers = /* @__PURE__ */ new Map();
-    this.onceReady = new import_cement12.ResolveOnce();
+    this.onceReady = new import_cement15.ResolveOnce();
     this.sthis = sthis;
     this.name = name;
     this.logger = ensureLogger(sthis, "CRDT");
@@ -3926,7 +4054,7 @@ var CRDT = class {
       try {
         await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
       } catch (e) {
-        throw this.logger.Error().Err(e).Msg("CRDT not ready").AsError();
+        throw this.logger.Error().Err(e).Msg(`CRDT is not ready`).AsError();
       }
     });
   }
@@ -3981,7 +4109,7 @@ var Database = class {
     this._listening = false;
     this._listeners = /* @__PURE__ */ new Set();
     this._noupdate_listeners = /* @__PURE__ */ new Set();
-    this._ready = new import_cement13.ResolveOnce();
+    this._ready = new import_cement16.ResolveOnce();
     this.name = name;
     this.opts = opts || this.opts;
     this.sthis = ensureSuperThis(this.opts);
@@ -4030,7 +4158,7 @@ var Database = class {
     await this.ready();
     this.logger.Debug().Str("id", doc._id).Msg("put");
     const { _id, ...value } = doc;
-    const docId = _id || this.sthis.nextId().str;
+    const docId = _id || this.sthis.timeOrderedNextId().str;
     const result = await this._writeQueue.push({
       id: docId,
       value: {
@@ -4171,6 +4299,7 @@ __export(runtime_exports, {
   kb: () => key_bag_exports,
   kc: () => keyed_crypto_exports,
   mf: () => wait_pr_multiformats_exports,
+  runtimeFn: () => import_cement17.runtimeFn,
   toArrayBuffer: () => toArrayBuffer
 });
 init_utils2();
@@ -4186,6 +4315,7 @@ __export(wait_pr_multiformats_exports, {
 var codec_interface_exports = {};
 
 // src/runtime/index.ts
+var import_cement17 = require("@adviser/cement");
 init_version();
 init_version2();
 
@@ -4194,6 +4324,6 @@ init_utils();
 
 // src/version.ts
 var PACKAGE_VERSION = Object.keys({
-  "0.19.9-dev-frag": "xxxx"
+  "0.19.11-dev-dryrun2": "xxxx"
 })[0];
 //# sourceMappingURL=index.cjs.map