@fireproof/core 0.19.11-dev-dryrun → 0.19.11-dev-dryrun2

package/index.cjs

@@ -243,6 +243,16 @@ var init_utils = __esm({
           bin
         };
       }
+      timeOrderedNextId(now) {
+        now = typeof now === "number" ? now : (/* @__PURE__ */ new Date()).getTime();
+        const t = (281474976710656 + now).toString(16).replace(/^1/, "");
+        const bin = this.crypto.randomBytes(10);
+        bin[1] = bin[1] & 240 | (bin[1] | 8 && 11);
+        const hex = Array.from(bin).map((i) => i.toString(16).padStart(2, "0")).join("");
+        return {
+          str: `${t.slice(0, 8)}-${t.slice(8)}-7${hex.slice(0, 3)}-${hex.slice(3, 7)}-${hex.slice(7, 19)}`
+        };
+      }
       start() {
         return Promise.resolve();
       }
@@ -571,30 +581,21 @@ var init_version = __esm({
   }
 });
 
-// src/runtime/gateways/indexdb/version.ts
-var INDEXDB_VERSION;
-var init_version2 = __esm({
-  "src/runtime/gateways/indexdb/version.ts"() {
-    "use strict";
-    INDEXDB_VERSION = "v0.19-indexdb";
-  }
-});
-
 // src/runtime/gateways/file/gateway.ts
 var gateway_exports = {};
 __export(gateway_exports, {
   FileGateway: () => FileGateway,
   FileTestStore: () => FileTestStore
 });
-var import_cement12, versionFiles, FileGateway, FileTestStore;
+var import_cement11, versionFiles, FileGateway, FileTestStore;
 var init_gateway = __esm({
   "src/runtime/gateways/file/gateway.ts"() {
     "use strict";
     init_version();
-    import_cement12 = require("@adviser/cement");
+    import_cement11 = require("@adviser/cement");
     init_utils();
     init_utils2();
-    versionFiles = new import_cement12.KeyedResolvOnce();
+    versionFiles = new import_cement11.KeyedResolvOnce();
     FileGateway = class {
       get fs() {
         if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
@@ -624,7 +625,7 @@ var init_gateway = __esm({
         });
       }
       start(baseURL) {
-        return (0, import_cement12.exception2Result)(async () => {
+        return (0, import_cement11.exception2Result)(async () => {
           this._fs = await getFileSystem(baseURL);
           await this.fs.start();
           const url = baseURL.build();
@@ -639,10 +640,10 @@ var init_gateway = __esm({
         });
       }
       async buildUrl(baseUrl, key) {
-        return import_cement12.Result.Ok(baseUrl.build().setParam("key", key).URI());
+        return import_cement11.Result.Ok(baseUrl.build().setParam("key", key).URI());
       }
       async close() {
-        return import_cement12.Result.Ok(void 0);
+        return import_cement11.Result.Ok(void 0);
       }
       // abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
       getFilePath(url) {
@@ -651,7 +652,7 @@ var init_gateway = __esm({
         return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
       }
       async put(url, body) {
-        return (0, import_cement12.exception2Result)(async () => {
+        return (0, import_cement11.exception2Result)(async () => {
           const file = await this.getFilePath(url);
           this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
           await this.fs.writefile(file, body);
@@ -663,17 +664,17 @@ var init_gateway = __esm({
           try {
             const res = await this.fs.readfile(file);
             this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get");
-            return import_cement12.Result.Ok(new Uint8Array(res));
+            return import_cement11.Result.Ok(new Uint8Array(res));
           } catch (e) {
             if (isNotFoundError(e)) {
-              return import_cement12.Result.Err(new NotFoundError(`file not found: ${file}`));
+              return import_cement11.Result.Err(new NotFoundError(`file not found: ${file}`));
             }
-            return import_cement12.Result.Err(e);
+            return import_cement11.Result.Err(e);
           }
         });
       }
       async delete(url) {
-        return (0, import_cement12.exception2Result)(async () => {
+        return (0, import_cement11.exception2Result)(async () => {
           await this.fs.unlink(this.getFilePath(url));
         });
       }
@@ -699,7 +700,7 @@ var init_gateway = __esm({
             }
           }
         }
-        return import_cement12.Result.Ok(void 0);
+        return import_cement11.Result.Ok(void 0);
       }
     };
     FileTestStore = class {
@@ -719,6 +720,15 @@ var init_gateway = __esm({
   }
 });
 
+// src/runtime/gateways/indexdb/version.ts
+var INDEXDB_VERSION;
+var init_version2 = __esm({
+  "src/runtime/gateways/indexdb/version.ts"() {
+    "use strict";
+    INDEXDB_VERSION = "v0.19-indexdb";
+  }
+});
+
 // src/runtime/gateways/indexdb/gateway.ts
 var gateway_exports2 = {};
 __export(gateway_exports2, {
@@ -779,15 +789,15 @@ function getIndexDBName(iurl, sthis) {
     dbName
   };
 }
-var import_idb2, import_cement13, onceIndexDB, IndexDBGateway, IndexDBTestStore;
+var import_idb2, import_cement12, onceIndexDB, IndexDBGateway, IndexDBTestStore;
 var init_gateway2 = __esm({
   "src/runtime/gateways/indexdb/gateway.ts"() {
     "use strict";
     import_idb2 = require("idb");
-    import_cement13 = require("@adviser/cement");
+    import_cement12 = require("@adviser/cement");
     init_version2();
     init_utils();
-    onceIndexDB = new import_cement13.KeyedResolvOnce();
+    onceIndexDB = new import_cement12.KeyedResolvOnce();
     IndexDBGateway = class {
       constructor(sthis) {
         this._db = {};
@@ -795,7 +805,7 @@ var init_gateway2 = __esm({
         this.sthis = sthis;
       }
       async start(baseURL) {
-        return (0, import_cement13.exception2Result)(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           this.logger.Debug().Url(baseURL).Msg("starting");
           await this.sthis.start();
           const ic = await connectIdb(baseURL, this.sthis);
@@ -805,10 +815,10 @@ var init_gateway2 = __esm({
         });
       }
       async close() {
-        return import_cement13.Result.Ok(void 0);
+        return import_cement12.Result.Ok(void 0);
       }
       async destroy(baseUrl) {
-        return (0, import_cement13.exception2Result)(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           const type = getStore(baseUrl, this.sthis, joinDBName).name;
           const idb = this._db;
           const trans = idb.transaction(type, "readwrite");
@@ -824,7 +834,7 @@ var init_gateway2 = __esm({
         });
       }
       buildUrl(baseUrl, key) {
-        return Promise.resolve(import_cement13.Result.Ok(baseUrl.build().setParam("key", key).URI()));
+        return Promise.resolve(import_cement12.Result.Ok(baseUrl.build().setParam("key", key).URI()));
       }
       async get(url) {
         return exceptionWrapper(async () => {
@@ -835,13 +845,13 @@ var init_gateway2 = __esm({
           const bytes = await tx.objectStore(store).get(sanitzeKey(key));
           await tx.done;
           if (!bytes) {
-            return import_cement13.Result.Err(new NotFoundError(`missing ${key}`));
+            return import_cement12.Result.Err(new NotFoundError(`missing ${key}`));
           }
-          return import_cement13.Result.Ok(bytes);
+          return import_cement12.Result.Ok(bytes);
         });
       }
       async put(url, value) {
-        return (0, import_cement13.exception2Result)(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           const key = getKey(url, this.logger);
           const store = getStore(url, this.sthis, joinDBName).name;
           this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
@@ -851,14 +861,14 @@ var init_gateway2 = __esm({
         });
       }
       async delete(url) {
-        return (0, import_cement13.exception2Result)(async () => {
+        return (0, import_cement12.exception2Result)(async () => {
           const key = getKey(url, this.logger);
           const store = getStore(url, this.sthis, joinDBName).name;
           this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
           const tx = this._db.transaction([store], "readwrite");
           await tx.objectStore(store).delete(sanitzeKey(key));
           await tx.done;
-          return import_cement13.Result.Ok(void 0);
+          return import_cement12.Result.Ok(void 0);
         });
       }
     };
@@ -914,7 +924,7 @@ __export(src_exports, {
 module.exports = __toCommonJS(src_exports);
 
 // src/database.ts
-var import_cement17 = require("@adviser/cement");
+var import_cement16 = require("@adviser/cement");
 
 // src/write-queue.ts
 function writeQueue(worker, payload = Infinity, unbounded = false) {
@@ -957,7 +967,7 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
 }
 
 // src/crdt.ts
-var import_cement16 = require("@adviser/cement");
+var import_cement15 = require("@adviser/cement");
 
 // src/runtime/wait-pr-multiformats/block.ts
 var block_exports = {};
@@ -1065,7 +1075,7 @@ function toCIDBlock(block) {
 }
 
 // src/blockstore/store-factory.ts
-var import_cement14 = require("@adviser/cement");
+var import_cement13 = require("@adviser/cement");
 
 // src/runtime/files.ts
 var files_exports = {};
@@ -1140,8 +1150,8 @@ var UnixFSFileBuilder = class {
 
 // src/blockstore/store.ts
 var import_p_limit2 = __toESM(require("p-limit"), 1);
-var import_dag_json = require("@ipld/dag-json");
-var import_cement9 = require("@adviser/cement");
+var import_dag_json2 = require("@ipld/dag-json");
+var import_cement10 = require("@adviser/cement");
 
 // src/types.ts
 function isFalsy(value) {
@@ -1704,7 +1714,7 @@ async function commit(params, t, done, opts = { noLoader: false, compact: false
 }
 async function prepareCarFiles(encoder, threshold, rootBlock, t) {
   const carFiles = [];
-  threshold = threshold || 1e3 * 1e3;
+  threshold = threshold || 128e3 * 8;
   let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
   clonedt.putSync(rootBlock.cid, rootBlock.bytes);
   let newsize = CBW.blockLength(toCIDBlock(rootBlock));
@@ -1917,7 +1927,8 @@ var Loader = class {
       carLog: this.carLog,
       carStore: fstore,
       WALStore: await this.WALStore(),
-      metaStore: await this.metaStore()
+      metaStore: await this.metaStore(),
+      threshold: this.ebOpts.threshold
     };
     return this.commitQueue.enqueue(async () => {
       await this.cacheTransaction(t);
@@ -2427,26 +2438,7 @@ var FragmentGateway = class {
     if (this.innerGW.subscribe) {
       return this.innerGW.subscribe(url, callback);
     } else {
-      let lastData = void 0;
-      let interval = 100;
-      const fetchData = async () => {
-        const result = await this.innerGW.get(url);
-        if (result.isOk()) {
-          const data = result.Ok();
-          if (!lastData || !data.every((value, index2) => lastData && value === lastData[index2])) {
-            lastData = data;
-            callback(data);
-            interval = 100;
-          } else {
-            interval *= 2;
-          }
-        }
-        timeoutId = setTimeout(fetchData, interval);
-      };
-      let timeoutId = setTimeout(fetchData, interval);
-      return import_cement8.Result.Ok(() => {
-        clearTimeout(timeoutId);
-      });
+      return import_cement8.Result.Err(this.logger.Error().Url(url).Msg("subscribe not supported").AsError());
     }
   }
   async delete(url) {
@@ -2464,12 +2456,119 @@ var FragmentGateway = class {
   }
 };
 
+// src/blockstore/meta-key-helper.ts
+var import_dag_json = require("@ipld/dag-json");
+var import_clock = require("@web3-storage/pail/clock");
+var import_multiformats2 = require("multiformats");
+var import_base64 = require("multiformats/bases/base64");
+var import_cement9 = require("@adviser/cement");
+async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
+  const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
+  if (!crdtEntries.length) {
+    sthis.logger.Debug().Str("byteHeads", new TextDecoder().decode(byteHeads)).Msg("No CRDT entries found");
+    return [];
+  }
+  if (!crdtEntries.map) {
+    sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries");
+    return [];
+  }
+  return Promise.all(
+    crdtEntries.map(async (crdtEntry) => {
+      const eventBlock = await (0, import_clock.decodeEventBlock)(import_base64.base64pad.decode(crdtEntry.data));
+      const dbMeta = (0, import_dag_json.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
+      return {
+        eventCid: eventBlock.cid,
+        parents: crdtEntry.parents,
+        dbMeta
+      };
+    })
+  );
+}
+async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
+  try {
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
+    const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
+    if (keyInfo.length) {
+      const dbMeta = keyInfo[0].dbMeta;
+      if (dbMeta.key) {
+        const kb = await getKeyBag(sthis);
+        const keyName = getStoreKeyName(uri);
+        const res = await kb.setNamedKey(keyName, dbMeta.key);
+        if (res.isErr()) {
+          sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
+          throw res.Err();
+        }
+      }
+      sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
+      return import_cement9.Result.Ok(dbMeta);
+    }
+    sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
+    return import_cement9.Result.Ok(void 0);
+  } catch (error) {
+    sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
+    return import_cement9.Result.Err(error);
+  }
+}
+async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
+  try {
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
+    const keyName = getStoreKeyName(uri);
+    const kb = await getKeyBag(sthis);
+    const res = await kb.getNamedExtractableKey(keyName, true);
+    if (res.isErr()) {
+      sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
+      throw res.Err();
+    }
+    const keyData = await res.Ok().extract();
+    const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
+    const { dbMeta, parents } = dbMetas[0];
+    const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
+    dbMeta.key = keyData.keyStr;
+    const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
+    const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
+    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
+    return import_cement9.Result.Ok(encoded);
+  } catch (error) {
+    sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
+    return import_cement9.Result.Err(error);
+  }
+}
+function getStoreKeyName(url) {
+  const storeKeyName = [url.getParam("localName") || url.getParam("name")];
+  const idx = url.getParam("index");
+  if (idx) {
+    storeKeyName.push(idx);
+  }
+  storeKeyName.push("data");
+  return `@${storeKeyName.join(":")}@`;
+}
+async function createDbMetaEventBlock(sthis, dbMeta, parents) {
+  const event = await import_clock.EventBlock.create(
+    {
+      dbMeta: sthis.txt.encode((0, import_dag_json.format)(dbMeta))
+    },
+    parents
+  );
+  return event;
+}
+async function encodeEventsWithParents(sthis, events, parents) {
+  const crdtEntries = events.map((event) => {
+    const base64String = import_base64.base64pad.encode(event.bytes);
+    return {
+      cid: event.cid.toString(),
+      data: base64String,
+      parents: parents.map((p) => p.toString())
+    };
+  });
+  return sthis.txt.encode(JSON.stringify(crdtEntries));
+}
+
 // src/blockstore/store.ts
 function guardVersion(url) {
   if (!url.hasParam("version")) {
-    return import_cement9.Result.Err(`missing version: ${url.toString()}`);
+    return import_cement10.Result.Err(`missing version: ${url.toString()}`);
   }
-  return import_cement9.Result.Ok(url);
+  return import_cement10.Result.Ok(url);
 }
 var BaseStoreImpl = class {
   constructor(name, url, opts, sthis, logger) {
@@ -2492,6 +2591,9 @@ var BaseStoreImpl = class {
   onClosed(fn) {
     this._onClosed.push(fn);
   }
+  async ready() {
+    return;
+  }
   async keyedCrypto() {
     return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
   }
@@ -2526,7 +2628,7 @@ var BaseStoreImpl = class {
     }
     if (this.ready) {
       const fn = this.ready.bind(this);
-      const ready = await (0, import_cement9.exception2Result)(fn);
+      const ready = await (0, import_cement10.exception2Result)(fn);
       if (ready.isErr()) {
         await this.close();
         return ready;
@@ -2537,18 +2639,87 @@ var BaseStoreImpl = class {
     return version;
   }
 };
+var MetaStoreImpl = class extends BaseStoreImpl {
+  // remote: boolean;
+  constructor(sthis, name, url, opts) {
+    super(name, url, { ...opts }, sthis, ensureLogger(sthis, "MetaStoreImpl"));
+    this.storeType = "meta";
+    this.subscribers = /* @__PURE__ */ new Map();
+    this.parents = [];
+    if (
+      /*this.remote && */
+      opts.gateway.subscribe
+    ) {
+      this.onStarted(async () => {
+        this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
+        opts.gateway.subscribe?.(this.url(), async (message) => {
+          this.logger.Debug().Msg("Received message from gateway");
+          const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
+          await Promise.all(
+            dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
+          );
+          this.updateParentsFromDbMetas(dbMetas);
+        });
+      });
+    }
+  }
+  updateParentsFromDbMetas(dbMetas) {
+    const cids = dbMetas.map((m) => m.eventCid);
+    const dbMetaParents = dbMetas.flatMap((m) => m.parents);
+    const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
+    const dbMetaParentsSet = new Set(dbMetaParents.map((p) => p.toString()));
+    this.parents = Array.from(uniqueParentsMap.values()).filter((p) => !dbMetaParentsSet.has(p.toString()));
+  }
+  async handleByteHeads(byteHeads) {
+    return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
+  }
+  async load() {
+    const branch = "main";
+    const url = await this.gateway.buildUrl(this.url(), branch);
+    if (url.isErr()) {
+      throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
+    }
+    const bytes = await this.gateway.get(url.Ok());
+    if (bytes.isErr()) {
+      if (isNotFoundError(bytes)) {
+        return void 0;
+      }
+      throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
+    }
+    const dbMetas = await this.handleByteHeads(bytes.Ok());
+    await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
+    this.updateParentsFromDbMetas(dbMetas);
+    return dbMetas.map((m) => m.dbMeta);
+  }
+  async save(meta, branch) {
+    branch = branch || "main";
+    this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
+    const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
+    const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
+    const url = await this.gateway.buildUrl(this.url(), branch);
+    if (url.isErr()) {
+      throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
+    }
+    this.parents = [event.cid];
+    const res = await this.gateway.put(url.Ok(), bytes);
+    if (res.isErr()) {
+      throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
+    }
+    return res;
+  }
+  async close() {
+    await this.gateway.close(this.url());
+    this._onClosed.forEach((fn) => fn());
+    return import_cement10.Result.Ok(void 0);
+  }
+  async destroy() {
+    return this.gateway.destroy(this.url());
+  }
+};
 var DataStoreImpl = class extends BaseStoreImpl {
   // readonly tag: string = "car-base";
   constructor(sthis, name, url, opts) {
-    super(
-      name,
-      url,
-      {
-        ...opts
-      },
-      sthis,
-      ensureLogger(sthis, "DataStoreImpl")
-    );
+    super(name, url, { ...opts }, sthis, ensureLogger(sthis, "DataStoreImpl"));
     this.storeType = "data";
   }
   async load(cid) {
@@ -2586,7 +2757,7 @@ var DataStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement9.Result.Ok(void 0);
+    return import_cement10.Result.Ok(void 0);
   }
   destroy() {
     return this.gateway.destroy(this.url());
@@ -2594,46 +2765,43 @@ var DataStoreImpl = class extends BaseStoreImpl {
 };
 var WALStoreImpl = class extends BaseStoreImpl {
   constructor(loader, url, opts) {
-    super(
-      loader.name,
-      url,
-      {
-        ...opts
-      },
-      loader.sthis,
-      ensureLogger(loader.sthis, "WALStoreImpl")
-    );
+    super(loader.name, url, { ...opts }, loader.sthis, ensureLogger(loader.sthis, "WALStoreImpl"));
     this.storeType = "wal";
-    this._ready = new import_cement9.ResolveOnce();
+    this._ready = new import_cement10.ResolveOnce();
     this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
     this.processing = void 0;
     this.processQueue = new CommitQueue();
-    this.ready = async () => {
-      return this._ready.once(async () => {
-        const walState = await this.load().catch((e) => {
-          this.logger.Error().Any("error", e).Msg("error loading wal");
-          return void 0;
-        });
-        if (!walState) {
-          this.walState.operations = [];
-          this.walState.fileOperations = [];
-        } else {
-          this.walState.operations = walState.operations || [];
-          this.walState.fileOperations = walState.fileOperations || [];
-        }
-      });
-    };
     this.loader = loader;
   }
+  async ready() {
+    return this._ready.once(async () => {
+      const walState = await this.load().catch((e) => {
+        this.logger.Error().Any("error", e).Msg("error loading wal");
+        return void 0;
+      });
+      if (!walState) {
+        this.walState.operations = [];
+        this.walState.fileOperations = [];
+      } else {
+        this.walState.operations = walState.operations || [];
+        this.walState.fileOperations = walState.fileOperations || [];
+      }
+    });
+  }
   async enqueue(dbMeta, opts) {
     await this.ready();
-    if (opts.noLoader) {
+    if (opts.compact) {
+      this.walState.operations = [];
+      this.walState.noLoaderOps = [dbMeta];
+    } else if (opts.noLoader) {
       this.walState.noLoaderOps.push(dbMeta);
     } else {
       this.walState.operations.push(dbMeta);
     }
     await this.save(this.walState);
-    void this.process();
+    if (!opts.noLoader) {
+      void this.process();
+    }
   }
   async enqueueFile(fileCid, publicFile = false) {
     await this.ready();
@@ -2660,7 +2828,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
       const fileOperations = [...this.walState.fileOperations];
       const uploads = [];
       const noLoaderOps = [...this.walState.noLoaderOps];
-      const limit = (0, import_p_limit2.default)(5);
+      const limit = (0, import_p_limit2.default)(3);
       if (operations.length + fileOperations.length + noLoaderOps.length === 0) return;
       for (const dbMeta of noLoaderOps) {
         const uploadP = limit(async () => {
@@ -2736,7 +2904,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
       throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
     }
     try {
-      return bytes && (0, import_dag_json.parse)(this.sthis.txt.decode(bytes.Ok()));
+      return bytes && (0, import_dag_json2.parse)(this.sthis.txt.decode(bytes.Ok()));
     } catch (e) {
       throw this.logger.Error().Err(e).Msg("error parse").AsError();
     }
@@ -2748,7 +2916,7 @@ var WALStoreImpl = class extends BaseStoreImpl {
     }
     let encoded;
     try {
-      encoded = (0, import_dag_json.format)(state);
+      encoded = (0, import_dag_json2.format)(state);
     } catch (e) {
       throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
     }
@@ -2760,265 +2928,13 @@ var WALStoreImpl = class extends BaseStoreImpl {
   async close() {
     await this.gateway.close(this.url());
     this._onClosed.forEach((fn) => fn());
-    return import_cement9.Result.Ok(void 0);
+    return import_cement10.Result.Ok(void 0);
   }
   destroy() {
     return this.gateway.destroy(this.url());
   }
 };
 
-// src/blockstore/store-meta.ts
-var import_dag_json2 = require("@ipld/dag-json");
-var import_clock = require("@web3-storage/pail/clock");
-var import_multiformats2 = require("multiformats");
-var import_cement11 = require("@adviser/cement");
-init_utils();
-
-// src/runtime/index.ts
-var runtime_exports = {};
-__export(runtime_exports, {
-  FILESTORE_VERSION: () => FILESTORE_VERSION,
-  INDEXDB_VERSION: () => INDEXDB_VERSION,
-  files: () => files_exports,
-  getFileName: () => getFileName,
-  getFileSystem: () => getFileSystem,
-  getPath: () => getPath,
-  kb: () => key_bag_exports,
-  kc: () => keyed_crypto_exports,
-  mf: () => wait_pr_multiformats_exports,
-  runtimeFn: () => import_cement10.runtimeFn,
-  toArrayBuffer: () => toArrayBuffer
-});
-init_utils2();
-
-// src/runtime/wait-pr-multiformats/index.ts
-var wait_pr_multiformats_exports = {};
-__export(wait_pr_multiformats_exports, {
-  block: () => block_exports,
-  codec: () => codec_interface_exports
-});
-
-// src/runtime/wait-pr-multiformats/codec-interface.ts
-var codec_interface_exports = {};
-
-// src/runtime/index.ts
-var import_cement10 = require("@adviser/cement");
-init_version();
-init_version2();
-
-// src/blockstore/store-meta.ts
-async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
-  const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
-  return Promise.all(
-    crdtEntries.map(async (crdtEntry) => {
-      const eventBlock = await (0, import_clock.decodeEventBlock)(decodeFromBase64(crdtEntry.data));
-      const dbMeta = (0, import_dag_json2.parse)(sthis.txt.decode(eventBlock.value.data.dbMeta));
-      return {
-        eventCid: eventBlock.cid,
-        parents: crdtEntry.parents,
-        dbMeta
-      };
-    })
-  );
-}
-async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
-  try {
-    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
-    const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
-    if (keyInfo.length) {
-      const dbMeta = keyInfo[0].dbMeta;
-      if (dbMeta.key) {
-        const kb = await key_bag_exports.getKeyBag(sthis);
-        const keyName = getStoreKeyName(uri);
-        const res = await kb.setNamedKey(keyName, dbMeta.key);
-        if (res.isErr()) {
-          sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
-          throw res.Err();
-        }
-      }
-      sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
-      return import_cement11.Result.Ok(dbMeta);
-    }
-    sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
-    return import_cement11.Result.Ok(void 0);
-  } catch (error) {
-    sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
-    return import_cement11.Result.Err(error);
-  }
-}
-async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
-  try {
-    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
-    const keyName = getStoreKeyName(uri);
-    const kb = await key_bag_exports.getKeyBag(sthis);
-    const res = await kb.getNamedExtractableKey(keyName, true);
-    if (res.isErr()) {
-      sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
-      throw res.Err();
-    }
-    const keyData = await res.Ok().extract();
-    const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
-    const { dbMeta, parents } = dbMetas[0];
-    const parentLinks = parents.map((p) => import_multiformats2.CID.parse(p));
-    dbMeta.key = keyData.keyStr;
-    const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
-    const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
-    sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
-    return import_cement11.Result.Ok(encoded);
-  } catch (error) {
-    sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
-    return import_cement11.Result.Err(error);
-  }
-}
-function getStoreKeyName(url) {
-  const storeKeyName = [url.getParam("localName") || url.getParam("name")];
-  const idx = url.getParam("index");
-  if (idx) {
-    storeKeyName.push(idx);
-  }
-  storeKeyName.push("data");
-  return `@${storeKeyName.join(":")}@`;
-}
-async function createDbMetaEventBlock(sthis, dbMeta, parents) {
-  const event = await import_clock.EventBlock.create(
-    {
-      dbMeta: sthis.txt.encode((0, import_dag_json2.format)(dbMeta))
-    },
-    parents
-  );
-  return event;
-}
-async function encodeEventsWithParents(sthis, events, parents) {
-  const crdtEntries = events.map((event) => {
-    const base64String = encodeToBase64(event.bytes);
-    return {
-      cid: event.cid.toString(),
-      data: base64String,
-      parents: parents.map((p) => p.toString())
-    };
-  });
-  return sthis.txt.encode(JSON.stringify(crdtEntries));
-}
-var MetaStoreImpl = class extends BaseStoreImpl {
-  constructor(sthis, name, url, opts, remote) {
-    super(
-      name,
-      url,
-      {
-        ...opts
-      },
-      sthis,
-      ensureLogger(sthis, "MetaStoreImpl")
-    );
-    this.storeType = "meta";
-    this.subscribers = /* @__PURE__ */ new Map();
-    this.parents = [];
-    if (remote && opts.gateway.subscribe) {
-      this.onStarted(async () => {
-        this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
-        opts.gateway.subscribe?.(this.url(), async (message) => {
-          this.logger.Debug().Msg("Received message from gateway");
-          const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
-          await Promise.all(
-            dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
-          );
-        });
-      });
-    }
-  }
-  async handleByteHeads(byteHeads) {
-    return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
-  }
-  async load() {
-    const branch = "main";
-    const url = await this.gateway.buildUrl(this.url(), branch);
-    if (url.isErr()) {
-      throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
-    }
-    const bytes = await this.gateway.get(url.Ok());
-    if (bytes.isErr()) {
-      if (isNotFoundError(bytes)) {
-        return void 0;
-      }
-      throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
-    }
-    const dbMetas = await this.handleByteHeads(bytes.Ok());
-    await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
-    const cids = dbMetas.map((m) => m.eventCid);
-    const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
-    this.parents = Array.from(uniqueParentsMap.values());
-    return dbMetas.map((m) => m.dbMeta);
-  }
-  async save(meta, branch) {
-    branch = branch || "main";
-    this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
-    const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
-    const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
-    const url = await this.gateway.buildUrl(this.url(), branch);
-    if (url.isErr()) {
-      throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
-    }
-    const res = await this.gateway.put(url.Ok(), bytes);
-    if (res.isErr()) {
-      throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
-    }
-    await this.loader?.handleDbMetasFromStore([meta]);
-    this.parents = [event.cid];
-    return res;
-  }
-  async close() {
-    await this.gateway.close(this.url());
-    this._onClosed.forEach((fn) => fn());
-    return import_cement11.Result.Ok(void 0);
-  }
-  async destroy() {
-    return this.gateway.destroy(this.url());
-  }
-};
-function encodeToBase64(bytes) {
-  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-  let base64 = "";
-  let i;
-  for (i = 0; i < bytes.length - 2; i += 3) {
-    base64 += chars[bytes[i] >> 2];
-    base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
-    base64 += chars[(bytes[i + 1] & 15) << 2 | bytes[i + 2] >> 6];
-    base64 += chars[bytes[i + 2] & 63];
-  }
-  if (i < bytes.length) {
-    base64 += chars[bytes[i] >> 2];
-    if (i === bytes.length - 1) {
-      base64 += chars[(bytes[i] & 3) << 4];
-      base64 += "==";
-    } else {
-      base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
-      base64 += chars[(bytes[i + 1] & 15) << 2];
-      base64 += "=";
-    }
-  }
-  return base64;
-}
-function decodeFromBase64(base64) {
-  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-  const bytes = new Uint8Array(base64.length * 3 / 4);
-  let i;
-  let j = 0;
-  for (i = 0; i < base64.length; i += 4) {
-    const a = chars.indexOf(base64[i]);
-    const b = chars.indexOf(base64[i + 1]);
-    const c = chars.indexOf(base64[i + 2]);
-    const d = chars.indexOf(base64[i + 3]);
-    bytes[j++] = a << 2 | b >> 4;
-    if (base64[i + 2] !== "=") {
-      bytes[j++] = (b & 15) << 4 | c >> 2;
-    }
-    if (base64[i + 3] !== "=") {
-      bytes[j++] = (c & 3) << 6 | d;
-    }
-  }
-  return bytes.slice(0, j);
-}
-
 // src/blockstore/store-factory.ts
 init_utils();
 function ensureIsIndex(url, isIndex) {
@@ -3039,15 +2955,15 @@ function buildURL(optURL, loader) {
   const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
   let obuUrl;
   if (obuItem && obuItem.overrideBaseURL) {
-    obuUrl = import_cement14.URI.from(obuItem.overrideBaseURL);
+    obuUrl = import_cement13.URI.from(obuItem.overrideBaseURL);
   }
   const ret = ensureIsIndex(
-    import_cement14.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
+    import_cement13.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
     storeOpts.isIndex
   );
   return ret;
 }
-var onceGateway = new import_cement14.KeyedResolvOnce();
+var onceGateway = new import_cement13.KeyedResolvOnce();
 async function getGatewayFromURL(url, sthis) {
   return onceGateway.get(url.toString()).once(async () => {
     const item = storeFactory.get(url.protocol);
@@ -3089,7 +3005,7 @@ function registerStoreProtocol(item) {
     storeFactory.delete(protocol);
   };
 }
-var onceDataStoreFactory = new import_cement14.KeyedResolvOnce();
+var onceDataStoreFactory = new import_cement13.KeyedResolvOnce();
 async function dataStoreFactory(loader) {
   const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
   const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
@@ -3107,7 +3023,7 @@ async function dataStoreFactory(loader) {
     return store;
   });
 }
-var onceMetaStoreFactory = new import_cement14.KeyedResolvOnce();
+var onceMetaStoreFactory = new import_cement13.KeyedResolvOnce();
 async function metaStoreFactory(loader) {
   const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
   const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
@@ -3126,7 +3042,7 @@ async function metaStoreFactory(loader) {
     return store;
   });
 }
-var onceRemoteWalFactory = new import_cement14.KeyedResolvOnce();
+var onceRemoteWalFactory = new import_cement13.KeyedResolvOnce();
 async function remoteWalFactory(loader) {
   const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
   const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
@@ -3210,7 +3126,13 @@ async function RemoteDataStore(sthis, name, url, opts) {
   return ds;
 }
 async function RemoteMetaStore(sthis, name, url, opts) {
-  const ms = new MetaStoreImpl(sthis, name, url, opts, true);
+  const ms = new MetaStoreImpl(
+    sthis,
+    name,
+    url,
+    opts
+    /* , true*/
+  );
   await ms.start();
   return ms;
 }
@@ -3570,7 +3492,7 @@ var import_sha26 = require("multiformats/hashes/sha2");
 var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
 var import_charwise = __toESM(require("charwise"), 1);
 var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
-var import_utils17 = require("prolly-trees/utils");
+var import_utils16 = require("prolly-trees/utils");
 var import_cache = require("prolly-trees/cache");
 var IndexTree = class {
 };
@@ -3578,17 +3500,17 @@ function refCompare(aRef, bRef) {
   if (Number.isNaN(aRef)) return -1;
   if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
   if (aRef === Infinity) return 1;
-  return (0, import_utils17.simpleCompare)(aRef, bRef);
+  return (0, import_utils16.simpleCompare)(aRef, bRef);
 }
 function compare(a, b) {
   const [aKey, aRef] = a;
   const [bKey, bRef] = b;
-  const comp = (0, import_utils17.simpleCompare)(aKey, bKey);
+  const comp = (0, import_utils16.simpleCompare)(aKey, bKey);
   if (comp !== 0) return comp;
   return refCompare(aRef, bRef);
 }
-var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils17.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
-var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils17.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils17.simpleCompare };
+var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
+var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils16.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils16.simpleCompare };
 function indexEntriesForChanges(changes, mapFn) {
   const indexEntries = [];
   changes.forEach(({ id: key, value, del }) => {
@@ -3896,7 +3818,7 @@ var Index = class {
 // src/crdt-clock.ts
 var import_clock3 = require("@web3-storage/pail/clock");
 var import_crdt2 = require("@web3-storage/pail/crdt");
-var import_cement15 = require("@adviser/cement");
+var import_cement14 = require("@adviser/cement");
 
 // src/apply-head-queue.ts
 function applyHeadQueue(worker, logger) {
@@ -3911,7 +3833,7 @@ function applyHeadQueue(worker, logger) {
         queue.sort((a, b) => b.updates ? 1 : -1);
         const task = queue.shift();
         if (!task) continue;
-        await worker(task.newHead, task.prevHead, task.updates !== null).catch((e) => {
+        await worker(task.newHead, task.prevHead, task.updates !== void 0).catch((e) => {
           throw logger.Error().Err(e).Msg("int_applyHead worker error").AsError();
         });
         if (task.updates) {
@@ -3954,7 +3876,7 @@ var CRDTClock = class {
     this.zoomers = /* @__PURE__ */ new Set();
     this.watchers = /* @__PURE__ */ new Set();
     this.emptyWatchers = /* @__PURE__ */ new Set();
-    this._ready = new import_cement15.ResolveOnce();
+    this._ready = new import_cement14.ResolveOnce();
     this.blockstore = blockstore;
     this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
     this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
@@ -4002,6 +3924,7 @@ var CRDTClock = class {
     this.zoomers.add(fn);
   }
   async int_applyHead(newHead, prevHead, localUpdates) {
+    const noLoader = !localUpdates;
     const ogHead = sortClockHead(this.head);
     newHead = sortClockHead(newHead);
     if (compareClockHeads(ogHead, newHead)) {
@@ -4012,7 +3935,6 @@ var CRDTClock = class {
       this.setHead(newHead);
       return;
     }
-    const noLoader = !localUpdates;
     if (!this.blockstore) {
       throw this.logger.Error().Msg("missing blockstore").AsError();
     }
@@ -4066,7 +3988,7 @@ init_utils();
 var CRDT = class {
   constructor(sthis, name, opts = {}) {
     this.indexers = /* @__PURE__ */ new Map();
-    this.onceReady = new import_cement16.ResolveOnce();
+    this.onceReady = new import_cement15.ResolveOnce();
     this.sthis = sthis;
     this.name = name;
     this.logger = ensureLogger(sthis, "CRDT");
@@ -4132,8 +4054,7 @@ var CRDT = class {
       try {
         await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
       } catch (e) {
-        const ee = e;
-        throw this.logger.Error().Err(e).Msg(`CRDT is not ready: ${ee.stack}`).AsError();
+        throw this.logger.Error().Err(e).Msg(`CRDT is not ready`).AsError();
       }
     });
   }
@@ -4188,7 +4109,7 @@ var Database = class {
     this._listening = false;
     this._listeners = /* @__PURE__ */ new Set();
     this._noupdate_listeners = /* @__PURE__ */ new Set();
-    this._ready = new import_cement17.ResolveOnce();
+    this._ready = new import_cement16.ResolveOnce();
     this.name = name;
     this.opts = opts || this.opts;
     this.sthis = ensureSuperThis(this.opts);
@@ -4237,7 +4158,7 @@ var Database = class {
     await this.ready();
     this.logger.Debug().Str("id", doc._id).Msg("put");
     const { _id, ...value } = doc;
-    const docId = _id || this.sthis.nextId().str;
+    const docId = _id || this.sthis.timeOrderedNextId().str;
     const result = await this._writeQueue.push({
       id: docId,
       value: {
@@ -4366,11 +4287,43 @@ function makeName(fnString) {
   }
 }
 
+// src/runtime/index.ts
+var runtime_exports = {};
+__export(runtime_exports, {
+  FILESTORE_VERSION: () => FILESTORE_VERSION,
+  INDEXDB_VERSION: () => INDEXDB_VERSION,
+  files: () => files_exports,
+  getFileName: () => getFileName,
+  getFileSystem: () => getFileSystem,
+  getPath: () => getPath,
+  kb: () => key_bag_exports,
+  kc: () => keyed_crypto_exports,
+  mf: () => wait_pr_multiformats_exports,
+  runtimeFn: () => import_cement17.runtimeFn,
+  toArrayBuffer: () => toArrayBuffer
+});
+init_utils2();
+
+// src/runtime/wait-pr-multiformats/index.ts
+var wait_pr_multiformats_exports = {};
+__export(wait_pr_multiformats_exports, {
+  block: () => block_exports,
+  codec: () => codec_interface_exports
+});
+
+// src/runtime/wait-pr-multiformats/codec-interface.ts
+var codec_interface_exports = {};
+
+// src/runtime/index.ts
+var import_cement17 = require("@adviser/cement");
+init_version();
+init_version2();
+
 // src/index.ts
 init_utils();
 
 // src/version.ts
 var PACKAGE_VERSION = Object.keys({
-  "0.19.11-dev-dryrun": "xxxx"
+  "0.19.11-dev-dryrun2": "xxxx"
 })[0];
 //# sourceMappingURL=index.cjs.map