@xyo-network/archivist-indexeddb 3.5.2 → 3.6.0-rc.10

package/dist/browser/Archivist.d.ts

@@ -1,14 +1,11 @@
-import { Hash } from '@xylabs/hex';
+import { Hash, Hex } from '@xylabs/hex';
 import { AbstractArchivist } from '@xyo-network/archivist-abstract';
 import { ArchivistModuleEventData, ArchivistNextOptions } from '@xyo-network/archivist-model';
-import { Payload, PayloadWithMeta, Schema } from '@xyo-network/payload-model';
+import { Payload, Schema, WithStorageMeta } from '@xyo-network/payload-model';
 import { IDBPDatabase } from 'idb';
 import { IndexedDbArchivistParams } from './Params.ts';
-type StoredPayload = PayloadWithMeta & {
-    _hash: string;
-};
 export interface PayloadStore {
-    [s: string]: StoredPayload;
+    [s: string]: WithStorageMeta;
 }
 export declare class IndexedDbArchivist<TParams extends IndexedDbArchivistParams = IndexedDbArchivistParams, TEventData extends ArchivistModuleEventData = ArchivistModuleEventData> extends AbstractArchivist<TParams, TEventData> {
     static readonly configSchemas: Schema[];
@@ -19,9 +16,11 @@ export declare class IndexedDbArchivist<TParams extends IndexedDbArchivistParams
     private static readonly dataHashIndex;
     private static readonly hashIndex;
     private static readonly schemaIndex;
+    private static readonly sequenceIndex;
     static readonly hashIndexName: string;
     static readonly dataHashIndexName: string;
     static readonly schemaIndexName: string;
+    static readonly sequenceIndexName: string;
     private _dbName?;
     private _storeName?;
     /**
@@ -46,9 +45,10 @@ export declare class IndexedDbArchivist<TParams extends IndexedDbArchivistParams
      * The indexes to create on the store
      */
     private get indexes();
-    protected allHandler(): Promise<StoredPayload[]>;
+    protected allHandler(): Promise<WithStorageMeta<Payload>[]>;
     protected clearHandler(): Promise<void>;
     protected deleteHandler(hashes: Hash[]): Promise<Hash[]>;
+    protected getFromCursor(db: IDBPDatabase<PayloadStore>, storeName: string, order?: 'asc' | 'desc', limit?: number, cursor?: Hex): Promise<WithStorageMeta[]>;
     /**
      * Uses an index to get a payload by the index value, but returns the value with the primary key (from the root store)
      * @param db The db instance to use
@@ -57,11 +57,10 @@ export declare class IndexedDbArchivist<TParams extends IndexedDbArchivistParams
      * @param key The key to get from the index
      * @returns The primary key and the payload, or undefined if not found
      */
-    protected getFromIndexWithPrimaryKey(db: IDBPDatabase<PayloadStore>, storeName: string, indexName: string, key: IDBValidKey): Promise<[number, StoredPayload] | undefined>;
-    protected getFromOffset(db: IDBPDatabase<PayloadStore>, storeName: string, order?: 'asc' | 'desc', limit?: number, offset?: Hash): Promise<StoredPayload[]>;
-    protected getHandler(hashes: string[]): Promise<StoredPayload[]>;
-    protected insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]>;
-    protected nextHandler(options?: ArchivistNextOptions): Promise<StoredPayload[]>;
+    protected getFromIndexWithPrimaryKey(db: IDBPDatabase<PayloadStore>, storeName: string, indexName: string, key: IDBValidKey): Promise<[number, WithStorageMeta] | undefined>;
+    protected getHandler(hashes: string[]): Promise<WithStorageMeta[]>;
+    protected insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]>;
+    protected nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]>;
     protected startHandler(): Promise<boolean>;
     /**
      * Returns that the desired DB/Store initialized to the correct version
@@ -75,5 +74,4 @@ export declare class IndexedDbArchivist<TParams extends IndexedDbArchivistParams
      */
     private useDb;
 }
-export {};
 //# sourceMappingURL=Archivist.d.ts.map

package/dist/browser/Archivist.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,EAKL,wBAAwB,EACxB,oBAAoB,EAIrB,MAAM,8BAA8B,CAAA;AAGrC,OAAO,EACL,OAAO,EAAE,eAAe,EAAE,MAAM,EACjC,MAAM,4BAA4B,CAAA;AACnC,OAAO,EACgB,YAAY,EAClC,MAAM,KAAK,CAAA;AAGZ,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAA;AAEtD,KAAK,aAAa,GAAG,eAAe,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAExD,MAAM,WAAW,YAAY;IAC3B,CAAC,CAAC,EAAE,MAAM,GAAG,aAAa,CAAA;CAC3B;AAED,qBACa,kBAAkB,CAC7B,OAAO,SAAS,wBAAwB,GAAG,wBAAwB,EACnE,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CACtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAAC;IAC9C,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA2D;IAC3G,gBAAyB,mBAAmB,EAAE,MAAM,CAAiC;IACrF,MAAM,CAAC,QAAQ,CAAC,aAAa,eAAc;IAC3C,MAAM,CAAC,QAAQ,CAAC,gBAAgB,KAAI;IACpC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,cAAa;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAEpC;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAEhC;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAElC;IAGD,MAAM,CAAC,QAAQ,CAAC,aAAa,SAAuD;IAEpF,MAAM,CAAC,QAAQ,CAAC,iBAAiB,SAA2D;IAE5F,MAAM,CAAC,QAAQ,CAAC,eAAe,SAAyD;IAExF,OAAO,CAAC,OAAO,CAAC,CAAQ;IACxB,OAAO,CAAC,UAAU,CAAC,CAAQ;IAE3B;;;;;;OAMG;IACH,IAAI,MAAM,WAeT;IAED;;OAEG;IACH,IAAI,SAAS,WAEZ;IAED,IAAa,OAAO,aASnB;IAED;;;OAGG;IACH,IAAI,SAAS,WAUZ;IAED;;OAEG;IACH,OAAO,KAAK,OAAO,GAElB;cAEwB,UAAU,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;cAOtC,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;cAI7B,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IA4BvE;;;;;;;OAOG;cACa,0BAA0B,CACxC,EAAE,EAAE,YAAY,CAAC,YAAY,CAAC,EAC9B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,GACf,OAAO,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,SAAS,CAAC;cAiB/B,aAAa,CAC3B,EAAE,EAAE,YAAY,CAAC,YAAY,CAAC,EAC9B,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,KAAK,GAAG,MAAc,EAC7B,KAAK,GAAE,MAAW,EAClB,MAAM,CAAC,EAAE,IAAI,GACZ,OAAO,CAAC,aAAa,EAAE,CAAC;cA0CF,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;cAmCtD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;cAmC9D,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;cASrE,YAAY;IAQrC;;;OAGG;YACW,gBAAgB;IAoD9B;;;;OAIG;YACW,KAAK;CAWpB"}
+{"version":3,"file":"Archivist.d.ts","sourceRoot":"","sources":["../../src/Archivist.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA;AACnE,OAAO,EAKL,wBAAwB,EACxB,oBAAoB,EAIrB,MAAM,8BAA8B,CAAA;AAGrC,OAAO,EACL,OAAO,EAAE,MAAM,EAAqB,eAAe,EACpD,MAAM,4BAA4B,CAAA;AACnC,OAAO,EACgB,YAAY,EAClC,MAAM,KAAK,CAAA;AAGZ,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAA;AAEtD,MAAM,WAAW,YAAY;IAC3B,CAAC,CAAC,EAAE,MAAM,GAAG,eAAe,CAAA;CAC7B;AAED,qBACa,kBAAkB,CAC7B,OAAO,SAAS,wBAAwB,GAAG,wBAAwB,EACnE,UAAU,SAAS,wBAAwB,GAAG,wBAAwB,CACtE,SAAQ,iBAAiB,CAAC,OAAO,EAAE,UAAU,CAAC;IAC9C,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAA2D;IAC3G,gBAAyB,mBAAmB,EAAE,MAAM,CAAiC;IACrF,MAAM,CAAC,QAAQ,CAAC,aAAa,eAAc;IAC3C,MAAM,CAAC,QAAQ,CAAC,gBAAgB,KAAI;IACpC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,cAAa;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAEpC;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAEhC;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAElC;IAED,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAEpC;IAGD,MAAM,CAAC,QAAQ,CAAC,aAAa,SAAuD;IAEpF,MAAM,CAAC,QAAQ,CAAC,iBAAiB,SAA2D;IAE5F,MAAM,CAAC,QAAQ,CAAC,eAAe,SAAyD;IAExF,MAAM,CAAC,QAAQ,CAAC,iBAAiB,SAA2D;IAE5F,OAAO,CAAC,OAAO,CAAC,CAAQ;IACxB,OAAO,CAAC,UAAU,CAAC,CAAQ;IAE3B;;;;;;OAMG;IACH,IAAI,MAAM,WAeT;IAED;;OAEG;IACH,IAAI,SAAS,WAEZ;IAED,IAAa,OAAO,aASnB;IAED;;;OAGG;IACH,IAAI,SAAS,WAUZ;IAED;;OAEG;IACH,OAAO,KAAK,OAAO,GAQlB;cAEwB,UAAU,IAAI,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAOjD,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;cAI7B,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;cA+BvD,aAAa,CAC3B,EAAE,EAAE,YAAY,CAAC,YAAY,CAAC,EAC9B,SAAS,EAAE,MAAM,EACf,KAAK,GAAE,KAAK,GAAG,MAAc,EAC7B,KAAK,GAAE,MAAW,EAClB,MAAM,CAAC,EAAE,GAAG,GACb,OAAO,CAAC,eAAe,EAAE,CAAC;IAwC7B;;;;;;;OAOG;cACa,0BAA0B,CACxC,EAAE,EAAE,YAAY,CAAC,YAAY,CAAC,EAC9B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,WAAW,GACf,OAAO,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,GAAG,SAAS,CAAC;cAgBxB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;cAmCxD,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAiCvE,WAAW,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC;cAShF,YAAY;IAQrC;;;OAGG;YACW,gBAAgB;IAoD9B;;;;OAIG;YACW,KAAK;CAWpB"}

package/dist/browser/index.mjs

@@ -2,13 +2,14 @@ var __defProp = Object.defineProperty;
 var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
 
 // src/Archivist.ts
-import { uniq, uniqBy } from "@xylabs/array";
+import { uniq } from "@xylabs/array";
 import { assertEx } from "@xylabs/assert";
 import { exists } from "@xylabs/exists";
 import { AbstractArchivist } from "@xyo-network/archivist-abstract";
 import { ArchivistAllQuerySchema, ArchivistClearQuerySchema, ArchivistDeleteQuerySchema, ArchivistInsertQuerySchema, ArchivistNextQuerySchema, buildStandardIndexName } from "@xyo-network/archivist-model";
 import { creatableModule } from "@xyo-network/module-model";
 import { PayloadBuilder } from "@xyo-network/payload-builder";
+import { SequenceConstants } from "@xyo-network/payload-model";
 import { openDB } from "idb";
 
 // src/Schema.ts
@@ -39,7 +40,7 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
   static defaultStoreName = "payloads";
   static dataHashIndex = {
     key: {
-      $hash: 1
+      _dataHash: 1
     },
     multiEntry: false,
     unique: false
@@ -58,12 +59,21 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
     multiEntry: false,
     unique: false
   };
+  static sequenceIndex = {
+    key: {
+      _sequence: 1
+    },
+    multiEntry: false,
+    unique: true
+  };
   // eslint-disable-next-line @typescript-eslint/member-ordering
   static hashIndexName = buildStandardIndexName(_IndexedDbArchivist.hashIndex);
   // eslint-disable-next-line @typescript-eslint/member-ordering
   static dataHashIndexName = buildStandardIndexName(_IndexedDbArchivist.dataHashIndex);
   // eslint-disable-next-line @typescript-eslint/member-ordering
   static schemaIndexName = buildStandardIndexName(_IndexedDbArchivist.schemaIndex);
+  // eslint-disable-next-line @typescript-eslint/member-ordering
+  static sequenceIndexName = buildStandardIndexName(_IndexedDbArchivist.sequenceIndex);
   _dbName;
   _storeName;
   /**
@@ -128,12 +138,13 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
       _IndexedDbArchivist.dataHashIndex,
       _IndexedDbArchivist.hashIndex,
       _IndexedDbArchivist.schemaIndex,
+      _IndexedDbArchivist.sequenceIndex,
       ...this.config?.storage?.indexes ?? []
     ];
   }
   async allHandler() {
     const payloads = await this.useDb((db) => db.getAll(this.storeName));
-    return await Promise.all(payloads.map((payload) => PayloadBuilder.build(payload)));
+    return payloads;
   }
   async clearHandler() {
     await this.useDb((db) => db.clear(this.storeName));
@@ -143,10 +154,13 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
       ...new Set(hashes)
     ];
     const pairs = await PayloadBuilder.hashPairs(await this.getHandler(uniqueHashes));
-    const hashesToDelete = pairs.flatMap((pair) => [
-      pair[0].$hash,
-      pair[1]
-    ]);
+    const hashesToDelete = (await Promise.all(pairs.map(async (pair) => {
+      const dataHash0 = await PayloadBuilder.dataHash(pair[0]);
+      return [
+        dataHash0,
+        pair[1]
+      ];
+    }))).flat();
     const distinctHashes = [
       ...new Set(hashesToDelete)
     ];
@@ -161,6 +175,38 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
       return found.filter(exists).filter((hash) => uniqueHashes.includes(hash));
     });
   }
+  async getFromCursor(db, storeName, order = "asc", limit = 10, cursor) {
+    const transaction = db.transaction(storeName, "readonly");
+    const store = transaction.objectStore(storeName);
+    const sequenceIndex = assertEx(store.index(_IndexedDbArchivist.sequenceIndexName), () => "Failed to get sequence index");
+    let sequenceCursor = void 0;
+    const parsedCursor = cursor === SequenceConstants.minLocalSequence ? null : cursor;
+    sequenceCursor = assertEx(await sequenceIndex.openCursor(null, order === "desc" ? "prev" : "next"), () => `Failed to get cursor [${parsedCursor}, ${cursor}]`);
+    if (!sequenceCursor?.value) return [];
+    try {
+      sequenceCursor = parsedCursor ? sequenceCursor.value._sequence === parsedCursor ? await sequenceCursor?.advance(1) : await (await sequenceCursor?.continue(parsedCursor))?.advance(1) : sequenceCursor;
+    } catch {
+      return [];
+    }
+    let remaining = limit;
+    const result = [];
+    while (remaining) {
+      const value = sequenceCursor?.value;
+      if (value) {
+        result.push(value);
+        try {
+          sequenceCursor = await sequenceCursor?.advance(1);
+        } catch {
+          break;
+        }
+        if (sequenceCursor === null) {
+          break;
+        }
+      }
+      remaining--;
+    }
+    return result;
+  }
   /**
   * Uses an index to get a payload by the index value, but returns the value with the primary key (from the root store)
   * @param db The db instance to use
@@ -185,45 +231,6 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
       ];
     }
   }
-  // eslint-disable-next-line complexity
-  async getFromOffset(db, storeName, order = "asc", limit = 10, offset) {
-    const transaction = db.transaction(storeName, "readonly");
-    const store = transaction.objectStore(storeName);
-    const hashIndex = store.index(_IndexedDbArchivist.hashIndexName);
-    let primaryCursor = void 0;
-    if (offset) {
-      const hashCursor = assertEx(await hashIndex.openCursor(offset), () => "Failed to get cursor");
-      const startPrimaryKey = hashCursor?.primaryKey ?? 0;
-      primaryCursor = await (order === "desc" ? store.openCursor(IDBKeyRange.upperBound(startPrimaryKey), "prev") : store.openCursor(IDBKeyRange.lowerBound(startPrimaryKey), "next"));
-      if (!primaryCursor?.value) return [];
-      try {
-        primaryCursor = await primaryCursor?.advance(1);
-      } catch {
-        return [];
-      }
-    } else {
-      primaryCursor = await store.openCursor(null, order === "desc" ? "prev" : "next");
-      if (!primaryCursor?.value) return [];
-    }
-    let remaining = limit;
-    const result = [];
-    while (remaining) {
-      const value = primaryCursor?.value;
-      if (value) {
-        result.push(value);
-        try {
-          primaryCursor = await primaryCursor?.advance(1);
-        } catch {
-          break;
-        }
-        if (primaryCursor === null) {
-          break;
-        }
-      }
-      remaining--;
-    }
-    return result;
-  }
   async getHandler(hashes) {
     const payloads = await this.useDb((db) => Promise.all(
       // Filter duplicates to prevent unnecessary DB queries
@@ -244,19 +251,15 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
     }).map(([_key, payload]) => payload);
   }
   async insertHandler(payloads) {
-    const uniquePayloadHashPairs = uniqBy(await PayloadBuilder.hashPairs(payloads), ([, _hash]) => _hash);
+    const payloadWithStorageMeta = (await PayloadBuilder.addStorageMeta(payloads)).sort(PayloadBuilder.compareStorageMeta);
     return await this.useDb(async (db) => {
       const tx = db.transaction(this.storeName, "readwrite");
       const store = tx.objectStore(this.storeName);
       const inserted = [];
       try {
-        await Promise.all(uniquePayloadHashPairs.map(async ([payload, _hash]) => {
-          const existingRootHash = await store.index(_IndexedDbArchivist.hashIndexName).get(_hash);
-          if (!existingRootHash) {
-            await store.put({
-              ...payload,
-              _hash
-            });
+        await Promise.all(payloadWithStorageMeta.map(async (payload) => {
+          if (!await store.index(_IndexedDbArchivist.hashIndexName).get(payload._hash)) {
+            await store.put(payload);
             inserted.push(payload);
           }
         }));
@@ -267,9 +270,9 @@ var IndexedDbArchivist = class _IndexedDbArchivist extends AbstractArchivist {
     });
   }
   async nextHandler(options) {
-    const { limit, offset, order } = options ?? {};
+    const { limit, cursor, order } = options ?? {};
     return await this.useDb(async (db) => {
-      return await this.getFromOffset(db, this.storeName, order, limit ?? 10, offset);
+      return await this.getFromCursor(db, this.storeName, order, limit ?? 10, cursor);
     });
   }
   async startHandler() {

package/dist/browser/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Archivist.ts","../../src/Schema.ts","../../src/Config.ts"],"sourcesContent":["import { uniq, uniqBy } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuerySchema,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistNextQuerySchema,\n  buildStandardIndexName,\n  IndexDescription,\n} from '@xyo-network/archivist-model'\nimport { creatableModule } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n  Payload, PayloadWithMeta, Schema,\n} from '@xyo-network/payload-model'\nimport {\n  IDBPCursorWithValue, IDBPDatabase, openDB,\n} from 'idb'\n\nimport { IndexedDbArchivistConfigSchema } from './Config.ts'\nimport { IndexedDbArchivistParams } from './Params.ts'\n\ntype StoredPayload = PayloadWithMeta & { _hash: string }\n\nexport interface PayloadStore {\n  [s: string]: StoredPayload\n}\n\n@creatableModule()\nexport class IndexedDbArchivist<\n  TParams extends IndexedDbArchivistParams = IndexedDbArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n> extends AbstractArchivist<TParams, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexedDbArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = IndexedDbArchivistConfigSchema\n  static readonly defaultDbName = 'archivist'\n  static readonly defaultDbVersion = 1\n  static readonly defaultStoreName = 'payloads'\n  private static readonly dataHashIndex: IndexDescription = {\n    key: { $hash: 1 }, multiEntry: false, unique: false,\n  }\n\n  private static readonly hashIndex: IndexDescription = {\n    key: { _hash: 1 }, multiEntry: false, unique: true,\n  }\n\n  private static readonly schemaIndex: IndexDescription = {\n    key: { schema: 1 }, multiEntry: false, unique: false,\n  }\n\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly hashIndexName = buildStandardIndexName(IndexedDbArchivist.hashIndex)\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly dataHashIndexName = buildStandardIndexName(IndexedDbArchivist.dataHashIndex)\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly schemaIndexName = buildStandardIndexName(IndexedDbArchivist.schemaIndex)\n\n  private _dbName?: string\n  private _storeName?: string\n\n  /**\n   * The database name. If not supplied via config, it defaults\n   * to the module name (not guaranteed to be unique) and if module\n   * name is not supplied, it defaults to `archivist`. This behavior\n   * biases towards a single, isolated DB per archivist which seems to\n   * make the most sense for 99% of use cases.\n   */\n  get dbName() {\n    if (!this._dbName) {\n      if (this.config?.dbName) {\n        this._dbName = this.config?.dbName\n      } else {\n        if (this.config?.name) {\n          this.logger.warn('No dbName provided, using module name: ', this.config?.name)\n          this._dbName = this.config?.name\n        } else {\n          this.logger.warn('No dbName provided, using default name: ', IndexedDbArchivist.defaultDbName)\n          this._dbName = IndexedDbArchivist.defaultDbName\n        }\n      }\n    }\n    return assertEx(this._dbName)\n  }\n\n  /**\n   * The database version. If not supplied via config, it defaults to 1.\n   */\n  get dbVersion() {\n    return this.config?.dbVersion ?? IndexedDbArchivist.defaultDbVersion\n  }\n\n  override get queries() {\n    return [\n      ArchivistNextQuerySchema,\n      ArchivistAllQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistInsertQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  /**\n   * The name of the object store. If not supplied via config, it defaults\n   * to `payloads`.\n   */\n  get storeName() {\n    if (!this._storeName) {\n      if (this.config?.storeName) {\n        this._storeName = this.config?.storeName\n      } else {\n        this.logger.warn('No storeName provided, using default name: ', IndexedDbArchivist.defaultStoreName)\n        this._storeName = IndexedDbArchivist.defaultStoreName\n      }\n    }\n    return assertEx(this._storeName)\n  }\n\n  /**\n   * The indexes to create on the store\n   */\n  private get indexes() {\n    return [IndexedDbArchivist.dataHashIndex, IndexedDbArchivist.hashIndex, IndexedDbArchivist.schemaIndex, ...(this.config?.storage?.indexes ?? [])]\n  }\n\n  protected override async allHandler(): Promise<StoredPayload[]> {\n    // Get all payloads from the store\n    const payloads = await this.useDb(db => db.getAll(this.storeName))\n    // Remove any metadata before returning to the client\n    return await Promise.all(payloads.map(payload => PayloadBuilder.build(payload)))\n  }\n\n  protected override async clearHandler(): Promise<void> {\n    await this.useDb(db => db.clear(this.storeName))\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    // Filter duplicates to prevent unnecessary DB queries\n    const uniqueHashes = [...new Set(hashes)]\n    const pairs = await PayloadBuilder.hashPairs(await this.getHandler(uniqueHashes))\n    const hashesToDelete = pairs.flatMap<Hash>(pair => [pair[0].$hash, pair[1]])\n    // Remove any duplicates\n    const distinctHashes = [...new Set(hashesToDelete)]\n    return await this.useDb(async (db) => {\n      // Only return hashes that were successfully deleted\n      const found = await Promise.all(\n        distinctHashes.map(async (hash) => {\n          // Check if the hash exists\n          const existing\n            = (await db.getKeyFromIndex(this.storeName, IndexedDbArchivist.hashIndexName, hash))\n            ?? (await db.getKeyFromIndex(this.storeName, IndexedDbArchivist.dataHashIndexName, hash))\n          // If it does exist\n          if (existing) {\n            // Delete it\n            await db.delete(this.storeName, existing)\n            // Return the hash so it gets added to the list of deleted hashes\n            return hash\n          }\n        }),\n      )\n      return found.filter(exists).filter(hash => uniqueHashes.includes(hash))\n    })\n  }\n\n  /**\n   * Uses an index to get a payload by the index value, but returns the value with the primary key (from the root store)\n   * @param db The db instance to use\n   * @param storeName The name of the store to use\n   * @param indexName The index to use\n   * @param key The key to get from the index\n   * @returns The primary key and the payload, or undefined if not found\n   */\n  protected async getFromIndexWithPrimaryKey(\n    db: IDBPDatabase<PayloadStore>,\n    storeName: string,\n    indexName: string,\n    key: IDBValidKey,\n  ): Promise<[number, StoredPayload] | undefined> {\n    const transaction = db.transaction(storeName, 'readonly')\n    const store = transaction.objectStore(storeName)\n    const index = store.index(indexName)\n    const cursor = await index.openCursor(key)\n    if (cursor) {\n      const singleValue = cursor.value\n      // NOTE: It's known to be a number because we are using IndexedDB supplied auto-incrementing keys\n      if (typeof cursor.primaryKey !== 'number') {\n        throw new TypeError('primaryKey must be a number')\n      }\n\n      return [cursor.primaryKey, singleValue]\n    }\n  }\n\n  // eslint-disable-next-line complexity\n  protected async getFromOffset(\n    db: IDBPDatabase<PayloadStore>,\n    storeName: string,\n    order: 'asc' | 'desc' = 'asc',\n    limit: number = 10,\n    offset?: Hash,\n  ): Promise<StoredPayload[]> {\n    const transaction = db.transaction(storeName, 'readonly')\n    const store = transaction.objectStore(storeName)\n    const hashIndex = store.index(IndexedDbArchivist.hashIndexName)\n    let primaryCursor: IDBPCursorWithValue<PayloadStore, [string]> | null | undefined = undefined\n    if (offset) {\n      const hashCursor = assertEx(await hashIndex.openCursor(offset), () => 'Failed to get cursor')\n      const startPrimaryKey = (hashCursor?.primaryKey ?? 0) as number // we know the primary key is a number and starts at 1\n      primaryCursor = await (order === 'desc'\n        ? store.openCursor(IDBKeyRange.upperBound(startPrimaryKey), 'prev')\n        : store.openCursor(IDBKeyRange.lowerBound(startPrimaryKey), 'next'))\n      if (!primaryCursor?.value) return []\n      try {\n        primaryCursor = await primaryCursor?.advance(1) // advance to skip the offset value\n      } catch {\n        return []\n      }\n    } else {\n      primaryCursor = await store.openCursor(null, order === 'desc' ? 'prev' : 'next')\n      if (!primaryCursor?.value) return []\n    }\n\n    let remaining = limit\n    const result: StoredPayload[] = []\n    while (remaining) {\n      const value = primaryCursor?.value\n      if (value) {\n        result.push(value)\n        try {\n          primaryCursor = await primaryCursor?.advance(1)\n        } catch {\n          break\n        }\n        if (primaryCursor === null) {\n          break\n        }\n      }\n      remaining--\n    }\n    return result\n  }\n\n  protected override async getHandler(hashes: string[]): Promise<StoredPayload[]> {\n    const payloads = await this.useDb(db =>\n      Promise.all(\n        // Filter duplicates to prevent unnecessary DB queries\n        uniq(hashes).map(async (hash) => {\n          // Find by hash\n          const payload = await this.getFromIndexWithPrimaryKey(db, this.storeName, IndexedDbArchivist.hashIndexName, hash)\n          // If found, return\n          if (payload) return payload\n          // Otherwise, find by data hash\n          return this.getFromIndexWithPrimaryKey(db, this.storeName, IndexedDbArchivist.dataHashIndexName, hash)\n        }),\n      ))\n\n    const found = new Set<string>()\n    return (\n      payloads\n        // Filter out not found\n        .filter(exists)\n        // Sort by primary key\n        .sort((a, b) => a![0] - b![0])\n        // Filter out duplicates by hash\n        .filter(([_key, payload]) => {\n          if (found.has(payload._hash)) {\n            return false\n          } else {\n            found.add(payload._hash)\n            return true\n          }\n        })\n        // Return just the payloads\n        .map(([_key, payload]) => payload)\n    )\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]> {\n    // Get the unique pairs of payloads and their hashes\n    const uniquePayloadHashPairs = uniqBy(await PayloadBuilder.hashPairs(payloads), ([, _hash]) => _hash)\n    return await this.useDb(async (db) => {\n      // Perform all inserts via a single transaction to ensure atomicity\n      // with respect to checking for the pre-existence of the hash.\n      // This is done to prevent duplicate root hashes due to race\n      // conditions between checking vs insertion.\n      const tx = db.transaction(this.storeName, 'readwrite')\n      // Get the object store\n      const store = tx.objectStore(this.storeName)\n      // Return only the payloads that were successfully inserted\n      const inserted: PayloadWithMeta[] = []\n      try {\n        await Promise.all(\n          uniquePayloadHashPairs.map(async ([payload, _hash]) => {\n            // Check if the root hash already exists\n            const existingRootHash = await store.index(IndexedDbArchivist.hashIndexName).get(_hash)\n            // If it does not already exist\n            if (!existingRootHash) {\n              // Insert the payload\n              await store.put({ ...payload, _hash })\n              // Add it to the inserted list\n              inserted.push(payload)\n            }\n          }),\n        )\n      } finally {\n        // Ensure the transaction is closed\n        await tx.done\n      }\n      return inserted\n    })\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<StoredPayload[]> {\n    const {\n      limit, offset, order,\n    } = options ?? {}\n    return await this.useDb(async (db) => {\n      return await this.getFromOffset(db, this.storeName, order, limit ?? 10, offset)\n    })\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    // NOTE: We could defer this creation to first access but we\n    // want to fail fast here in case something is wrong\n    await this.useDb(() => {})\n    return true\n  }\n\n  /**\n   * Returns that the desired DB/Store initialized to the correct version\n   * @returns The initialized DB\n   */\n  private async getInitializedDb(): Promise<IDBPDatabase<PayloadStore>> {\n    const {\n      dbName, dbVersion, indexes, storeName, logger,\n    } = this\n    return await openDB<PayloadStore>(dbName, dbVersion, {\n      blocked(currentVersion, blockedVersion, event) {\n        logger.warn(`IndexedDbArchivist: Blocked from upgrading from ${currentVersion} to ${blockedVersion}`, event)\n      },\n      blocking(currentVersion, blockedVersion, event) {\n        logger.warn(`IndexedDbArchivist: Blocking upgrade from ${currentVersion} to ${blockedVersion}`, event)\n      },\n      terminated() {\n        logger.log('IndexedDbArchivist: Terminated')\n      },\n      upgrade(database, oldVersion, newVersion, transaction) {\n        // NOTE: This is called whenever the DB is created/updated. We could simply ensure the desired end\n        // state but, out of an abundance of caution, we will just delete (so we know where we are starting\n        // from a known good point) and recreate the desired state. This prioritizes resilience over data\n        // retention but we can revisit that tradeoff when it becomes limiting. Because distributed browser\n        // state is extremely hard to debug, this seems like fair tradeoff for now.\n        if (oldVersion !== newVersion) {\n          logger.log(`IndexedDbArchivist: Upgrading from ${oldVersion} to ${newVersion}`)\n          // Delete any existing databases that are not the current version\n          const objectStores = transaction.objectStoreNames\n          for (const name of objectStores) {\n            try {\n              database.deleteObjectStore(name)\n            } catch {\n              logger.log(`IndexedDbArchivist: Failed to delete existing object store ${name}`)\n            }\n          }\n        }\n        // Create the store\n        const store = database.createObjectStore(storeName, {\n          // If it isn't explicitly set, create a value by auto incrementing.\n          autoIncrement: true,\n        })\n        // Name the store\n        store.name = storeName\n        // Create an index on the hash\n        for (const {\n          key, multiEntry, unique,\n        } of indexes) {\n          const indexKeys = Object.keys(key)\n          const keys = indexKeys.length === 1 ? indexKeys[0] : indexKeys\n          const indexName = buildStandardIndexName({ key, unique })\n          store.createIndex(indexName, keys, { multiEntry, unique })\n        }\n      },\n    })\n  }\n\n  /**\n   * Executes a callback with the initialized DB and then closes the db\n   * @param callback The method to execute with the initialized DB\n   * @returns\n   */\n  private async useDb<T>(callback: (db: IDBPDatabase<PayloadStore>) => Promise<T> | T): Promise<T> {\n    // Get the initialized DB\n    const db = await this.getInitializedDb()\n    try {\n      // Perform the callback\n      return await callback(db)\n    } finally {\n      // Close the DB\n      db.close()\n    }\n  }\n}\n","export type IndexedDbArchivistSchema = 'network.xyo.archivist.indexeddb'\nexport const IndexedDbArchivistSchema: IndexedDbArchivistSchema = 'network.xyo.archivist.indexeddb'\n","import type { ArchivistConfig } from '@xyo-network/archivist-model'\n\nimport { IndexedDbArchivistSchema } from './Schema.ts'\n\nexport type IndexedDbArchivistConfigSchema = `${IndexedDbArchivistSchema}.config`\nexport const IndexedDbArchivistConfigSchema: IndexedDbArchivistConfigSchema = `${IndexedDbArchivistSchema}.config`\n\nexport type IndexedDbArchivistConfig = ArchivistConfig<{\n  /**\n   * The database name\n   */\n  dbName?: string\n  /**\n   * The version of the DB, defaults to 1\n   */\n  dbVersion?: number\n  schema: IndexedDbArchivistConfigSchema\n  /**\n   * The name of the object store\n   */\n  storeName?: string\n}>\n"],"mappings":";;;;AAAA,SAASA,MAAMC,cAAc;AAC7B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAEvB,SAASC,yBAAyB;AAClC,SACEC,yBACAC,2BACAC,4BACAC,4BAGAC,0BACAC,8BAEK;AACP,SAASC,uBAAuB;AAChC,SAASC,sBAAsB;AAI/B,SACqCC,cAC9B;;;ACtBA,IAAMC,2BAAqD;;;ACI3D,IAAMC,iCAAiE,GAAGC,wBAAAA;;;;;;;;;;AF8B1E,IAAMC,qBAAN,MAAMA,4BAGHC,kBAAAA;SAAAA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,gBAAgB;EAChC,OAAgBC,mBAAmB;EACnC,OAAgBC,mBAAmB;EACnC,OAAwBC,gBAAkC;IACxDC,KAAK;MAAEC,OAAO;IAAE;IAAGC,YAAY;IAAOC,QAAQ;EAChD;EAEA,OAAwBC,YAA8B;IACpDJ,KAAK;MAAEK,OAAO;IAAE;IAAGH,YAAY;IAAOC,QAAQ;EAChD;EAEA,OAAwBG,cAAgC;IACtDN,KAAK;MAAEO,QAAQ;IAAE;IAAGL,YAAY;IAAOC,QAAQ;EACjD;;EAGA,OAAgBK,gBAAgBC,uBAAuBlB,oBAAmBa,SAAS;;EAEnF,OAAgBM,oBAAoBD,uBAAuBlB,oBAAmBQ,aAAa;;EAE3F,OAAgBY,kBAAkBF,uBAAuBlB,oBAAmBe,WAAW;EAE/EM;EACAC;;;;;;;;EASR,IAAIC,SAAS;AACX,QAAI,CAAC,KAAKF,SAAS;AACjB,UAAI,KAAKG,QAAQD,QAAQ;AACvB,aAAKF,UAAU,KAAKG,QAAQD;MAC9B,OAAO;AACL,YAAI,KAAKC,QAAQC,MAAM;AACrB,eAAKC,OAAOC,KAAK,2CAA2C,KAAKH,QAAQC,IAAAA;AACzE,eAAKJ,UAAU,KAAKG,QAAQC;QAC9B,OAAO;AACL,eAAKC,OAAOC,KAAK,4CAA4C3B,oBAAmBK,aAAa;AAC7F,eAAKgB,UAAUrB,oBAAmBK;QACpC;MACF;IACF;AACA,WAAOuB,SAAS,KAAKP,OAAO;EAC9B;;;;EAKA,IAAIQ,YAAY;AACd,WAAO,KAAKL,QAAQK,aAAa7B,oBAAmBM;EACtD;EAEA,IAAawB,UAAU;AACrB,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;SACG,MAAML;;EAEb;;;;;EAMA,IAAIM,YAAY;AACd,QAAI,CAAC,KAAKd,YAAY;AACpB,UAAI,KAAKE,QAAQY,WAAW;AAC1B,aAAKd,aAAa,KAAKE,QAAQY;MACjC,OAAO;AACL,aAAKV,OAAOC,KAAK,+CAA+C3B,oBAAmBO,gBAAgB;AACnG,aAAKe,aAAatB,oBAAmBO;MACvC;IACF;AACA,WAAOqB,SAAS,KAAKN,UAAU;EACjC;;;;EAKA,IAAYe,UAAU;AACpB,WAAO;MAACrC,oBAAmBQ;MAAeR,oBAAmBa;MAAWb,oBAAmBe;SAAiB,KAAKS,QAAQc,SAASD,WAAW,CAAA;;EAC/I;EAEA,MAAyBE,aAAuC;AAE9D,UAAMC,WAAW,MAAM,KAAKC,MAAMC,CAAAA,OAAMA,GAAGC,OAAO,KAAKP,SAAS,CAAA;AAEhE,WAAO,MAAMQ,QAAQC,IAAIL,SAASM,IAAIC,CAAAA,YAAWC,eAAeC,MAAMF,OAAAA,CAAAA,CAAAA;EACxE;EAEA,MAAyBG,eAA8B;AACrD,UAAM,KAAKT,MAAMC,CAAAA,OAAMA,GAAGS,MAAM,KAAKf,SAAS,CAAA;EAChD;EAEA,MAAyBgB,cAAcC,QAAiC;AAEtE,UAAMC,eAAe;SAAI,IAAIC,IAAIF,MAAAA;;AACjC,UAAMG,QAAQ,MAAMR,eAAeS,UAAU,MAAM,KAAKC,WAAWJ,YAAAA,CAAAA;AACnE,UAAMK,iBAAiBH,MAAMI,QAAcC,CAAAA,SAAQ;MAACA,KAAK,CAAA,EAAGnD;MAAOmD,KAAK,CAAA;KAAG;AAE3E,UAAMC,iBAAiB;SAAI,IAAIP,IAAII,cAAAA;;AACnC,WAAO,MAAM,KAAKlB,MAAM,OAAOC,OAAAA;AAE7B,YAAMqB,QAAQ,MAAMnB,QAAQC,IAC1BiB,eAAehB,IAAI,OAAOkB,SAAAA;AAExB,cAAMC,WACD,MAAMvB,GAAGwB,gBAAgB,KAAK9B,WAAWpC,oBAAmBiB,eAAe+C,IAAAA,KAC1E,MAAMtB,GAAGwB,gBAAgB,KAAK9B,WAAWpC,oBAAmBmB,mBAAmB6C,IAAAA;AAErF,YAAIC,UAAU;AAEZ,gBAAMvB,GAAGyB,OAAO,KAAK/B,WAAW6B,QAAAA;AAEhC,iBAAOD;QACT;MACF,CAAA,CAAA;AAEF,aAAOD,MAAMK,OAAOC,MAAAA,EAAQD,OAAOJ,CAAAA,SAAQV,aAAagB,SAASN,IAAAA,CAAAA;IACnE,CAAA;EACF;;;;;;;;;EAUA,MAAgBO,2BACd7B,IACAN,WACAoC,WACA/D,KAC8C;AAC9C,UAAMgE,cAAc/B,GAAG+B,YAAYrC,WAAW,UAAA;AAC9C,UAAMsC,QAAQD,YAAYE,YAAYvC,SAAAA;AACtC,UAAMwC,QAAQF,MAAME,MAAMJ,SAAAA;AAC1B,UAAMK,SAAS,MAAMD,MAAME,WAAWrE,GAAAA;AACtC,QAAIoE,QAAQ;AACV,YAAME,cAAcF,OAAOG;AAE3B,UAAI,OAAOH,OAAOI,eAAe,UAAU;AACzC,cAAM,IAAIC,UAAU,6BAAA;MACtB;AAEA,aAAO;QAACL,OAAOI;QAAYF;;IAC7B;EACF;;EAGA,MAAgBI,cACdzC,IACAN,WACAgD,QAAwB,OACxBC,QAAgB,IAChBC,QAC0B;AAC1B,UAAMb,cAAc/B,GAAG+B,YAAYrC,WAAW,UAAA;AAC9C,UAAMsC,QAAQD,YAAYE,YAAYvC,SAAAA;AACtC,UAAMvB,YAAY6D,MAAME,MAAM5E,oBAAmBiB,aAAa;AAC9D,QAAIsE,gBAAgFC;AACpF,QAAIF,QAAQ;AACV,YAAMG,aAAa7D,SAAS,MAAMf,UAAUiE,WAAWQ,MAAAA,GAAS,MAAM,sBAAA;AACtE,YAAMI,kBAAmBD,YAAYR,cAAc;AACnDM,sBAAgB,OAAOH,UAAU,SAC7BV,MAAMI,WAAWa,YAAYC,WAAWF,eAAAA,GAAkB,MAAA,IAC1DhB,MAAMI,WAAWa,YAAYE,WAAWH,eAAAA,GAAkB,MAAA;AAC9D,UAAI,CAACH,eAAeP,MAAO,QAAO,CAAA;AAClC,UAAI;AACFO,wBAAgB,MAAMA,eAAeO,QAAQ,CAAA;MAC/C,QAAQ;AACN,eAAO,CAAA;MACT;IACF,OAAO;AACLP,sBAAgB,MAAMb,MAAMI,WAAW,MAAMM,UAAU,SAAS,SAAS,MAAA;AACzE,UAAI,CAACG,eAAeP,MAAO,QAAO,CAAA;IACpC;AAEA,QAAIe,YAAYV;AAChB,UAAMW,SAA0B,CAAA;AAChC,WAAOD,WAAW;AAChB,YAAMf,QAAQO,eAAeP;AAC7B,UAAIA,OAAO;AACTgB,eAAOC,KAAKjB,KAAAA;AACZ,YAAI;AACFO,0BAAgB,MAAMA,eAAeO,QAAQ,CAAA;QAC/C,QAAQ;AACN;QACF;AACA,YAAIP,kBAAkB,MAAM;AAC1B;QACF;MACF;AACAQ;IACF;AACA,WAAOC;EACT;EAEA,MAAyBtC,WAAWL,QAA4C;AAC9E,UAAMb,WAAW,MAAM,KAAKC,MAAMC,CAAAA,OAChCE,QAAQC;;MAENqD,KAAK7C,MAAAA,EAAQP,IAAI,OAAOkB,SAAAA;AAEtB,cAAMjB,UAAU,MAAM,KAAKwB,2BAA2B7B,IAAI,KAAKN,WAAWpC,oBAAmBiB,eAAe+C,IAAAA;AAE5G,YAAIjB,QAAS,QAAOA;AAEpB,eAAO,KAAKwB,2BAA2B7B,IAAI,KAAKN,WAAWpC,oBAAmBmB,mBAAmB6C,IAAAA;MACnG,CAAA;IAAA,CAAA;AAGJ,UAAMD,QAAQ,oBAAIR,IAAAA;AAClB,WACEf,SAEG4B,OAAOC,MAAAA,EAEP8B,KAAK,CAACC,GAAGC,MAAMD,EAAG,CAAA,IAAKC,EAAG,CAAA,CAAE,EAE5BjC,OAAO,CAAC,CAACkC,MAAMvD,OAAAA,MAAQ;AACtB,UAAIgB,MAAMwC,IAAIxD,QAAQjC,KAAK,GAAG;AAC5B,eAAO;MACT,OAAO;AACLiD,cAAMyC,IAAIzD,QAAQjC,KAAK;AACvB,eAAO;MACT;IACF,CAAA,EAECgC,IAAI,CAAC,CAACwD,MAAMvD,OAAAA,MAAaA,OAAAA;EAEhC;EAEA,MAAyB0D,cAAcjE,UAAiD;AAEtF,UAAMkE,yBAAyBC,OAAO,MAAM3D,eAAeS,UAAUjB,QAAAA,GAAW,CAAC,CAAA,EAAG1B,KAAAA,MAAWA,KAAAA;AAC/F,WAAO,MAAM,KAAK2B,MAAM,OAAOC,OAAAA;AAK7B,YAAMkE,KAAKlE,GAAG+B,YAAY,KAAKrC,WAAW,WAAA;AAE1C,YAAMsC,QAAQkC,GAAGjC,YAAY,KAAKvC,SAAS;AAE3C,YAAMyE,WAA8B,CAAA;AACpC,UAAI;AACF,cAAMjE,QAAQC,IACZ6D,uBAAuB5D,IAAI,OAAO,CAACC,SAASjC,KAAAA,MAAM;AAEhD,gBAAMgG,mBAAmB,MAAMpC,MAAME,MAAM5E,oBAAmBiB,aAAa,EAAE8F,IAAIjG,KAAAA;AAEjF,cAAI,CAACgG,kBAAkB;AAErB,kBAAMpC,MAAMsC,IAAI;cAAE,GAAGjE;cAASjC;YAAM,CAAA;AAEpC+F,qBAASZ,KAAKlD,OAAAA;UAChB;QACF,CAAA,CAAA;MAEJ,UAAA;AAEE,cAAM6D,GAAGK;MACX;AACA,aAAOJ;IACT,CAAA;EACF;EAEA,MAAyBK,YAAYC,SAA0D;AAC7F,UAAM,EACJ9B,OAAOC,QAAQF,MAAK,IAClB+B,WAAW,CAAC;AAChB,WAAO,MAAM,KAAK1E,MAAM,OAAOC,OAAAA;AAC7B,aAAO,MAAM,KAAKyC,cAAczC,IAAI,KAAKN,WAAWgD,OAAOC,SAAS,IAAIC,MAAAA;IAC1E,CAAA;EACF;EAEA,MAAyB8B,eAAe;AACtC,UAAM,MAAMA,aAAAA;AAGZ,UAAM,KAAK3E,MAAM,MAAA;IAAO,CAAA;AACxB,WAAO;EACT;;;;;EAMA,MAAc4E,mBAAwD;AACpE,UAAM,EACJ9F,QAAQM,WAAWQ,SAASD,WAAWV,OAAM,IAC3C;AACJ,WAAO,MAAM4F,OAAqB/F,QAAQM,WAAW;MACnD0F,QAAQC,gBAAgBC,gBAAgBC,OAAK;AAC3ChG,eAAOC,KAAK,mDAAmD6F,cAAAA,OAAqBC,cAAAA,IAAkBC,KAAAA;MACxG;MACAC,SAASH,gBAAgBC,gBAAgBC,OAAK;AAC5ChG,eAAOC,KAAK,6CAA6C6F,cAAAA,OAAqBC,cAAAA,IAAkBC,KAAAA;MAClG;MACAE,aAAAA;AACElG,eAAOmG,IAAI,gCAAA;MACb;MACAC,QAAQC,UAAUC,YAAYC,YAAYxD,aAAW;AAMnD,YAAIuD,eAAeC,YAAY;AAC7BvG,iBAAOmG,IAAI,sCAAsCG,UAAAA,OAAiBC,UAAAA,EAAY;AAE9E,gBAAMC,eAAezD,YAAY0D;AACjC,qBAAW1G,QAAQyG,cAAc;AAC/B,gBAAI;AACFH,uBAASK,kBAAkB3G,IAAAA;YAC7B,QAAQ;AACNC,qBAAOmG,IAAI,8DAA8DpG,IAAAA,EAAM;YACjF;UACF;QACF;AAEA,cAAMiD,QAAQqD,SAASM,kBAAkBjG,WAAW;;UAElDkG,eAAe;QACjB,CAAA;AAEA5D,cAAMjD,OAAOW;AAEb,mBAAW,EACT3B,KAAKE,YAAYC,OAAM,KACpByB,SAAS;AACZ,gBAAMkG,YAAYC,OAAOC,KAAKhI,GAAAA;AAC9B,gBAAMgI,OAAOF,UAAUG,WAAW,IAAIH,UAAU,CAAA,IAAKA;AACrD,gBAAM/D,YAAYtD,uBAAuB;YAAET;YAAKG;UAAO,CAAA;AACvD8D,gBAAMiE,YAAYnE,WAAWiE,MAAM;YAAE9H;YAAYC;UAAO,CAAA;QAC1D;MACF;IACF,CAAA;EACF;;;;;;EAOA,MAAc6B,MAASmG,UAA0E;AAE/F,UAAMlG,KAAK,MAAM,KAAK2E,iBAAgB;AACtC,QAAI;AAEF,aAAO,MAAMuB,SAASlG,EAAAA;IACxB,UAAA;AAEEA,SAAGmG,MAAK;IACV;EACF;AACF;;;;","names":["uniq","uniqBy","assertEx","exists","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","buildStandardIndexName","creatableModule","PayloadBuilder","openDB","IndexedDbArchivistSchema","IndexedDbArchivistConfigSchema","IndexedDbArchivistSchema","IndexedDbArchivist","AbstractArchivist","configSchemas","IndexedDbArchivistConfigSchema","defaultConfigSchema","defaultDbName","defaultDbVersion","defaultStoreName","dataHashIndex","key","$hash","multiEntry","unique","hashIndex","_hash","schemaIndex","schema","hashIndexName","buildStandardIndexName","dataHashIndexName","schemaIndexName","_dbName","_storeName","dbName","config","name","logger","warn","assertEx","dbVersion","queries","ArchivistNextQuerySchema","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","storeName","indexes","storage","allHandler","payloads","useDb","db","getAll","Promise","all","map","payload","PayloadBuilder","build","clearHandler","clear","deleteHandler","hashes","uniqueHashes","Set","pairs","hashPairs","getHandler","hashesToDelete","flatMap","pair","distinctHashes","found","hash","existing","getKeyFromIndex","delete","filter","exists","includes","getFromIndexWithPrimaryKey","indexName","transaction","store","objectStore","index","cursor","openCursor","singleValue","value","primaryKey","TypeError","getFromOffset","order","limit","offset","primaryCursor","undefined","hashCursor","startPrimaryKey","IDBKeyRange","upperBound","lowerBound","advance","remaining","result","push","uniq","sort","a","b","_key","has","add","insertHandler","uniquePayloadHashPairs","uniqBy","tx","inserted","existingRootHash","get","put","done","nextHandler","options","startHandler","getInitializedDb","openDB","blocked","currentVersion","blockedVersion","event","blocking","terminated","log","upgrade","database","oldVersion","newVersion","objectStores","objectStoreNames","deleteObjectStore","createObjectStore","autoIncrement","indexKeys","Object","keys","length","createIndex","callback","close"]}
+{"version":3,"sources":["../../src/Archivist.ts","../../src/Schema.ts","../../src/Config.ts"],"sourcesContent":["/* eslint-disable complexity */\nimport { uniq } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport { Hash, Hex } from '@xylabs/hex'\nimport { AbstractArchivist } from '@xyo-network/archivist-abstract'\nimport {\n  ArchivistAllQuerySchema,\n  ArchivistClearQuerySchema,\n  ArchivistDeleteQuerySchema,\n  ArchivistInsertQuerySchema,\n  ArchivistModuleEventData,\n  ArchivistNextOptions,\n  ArchivistNextQuerySchema,\n  buildStandardIndexName,\n  IndexDescription,\n} from '@xyo-network/archivist-model'\nimport { creatableModule } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport {\n  Payload, Schema, SequenceConstants, WithStorageMeta,\n} from '@xyo-network/payload-model'\nimport {\n  IDBPCursorWithValue, IDBPDatabase, openDB,\n} from 'idb'\n\nimport { IndexedDbArchivistConfigSchema } from './Config.ts'\nimport { IndexedDbArchivistParams } from './Params.ts'\n\nexport interface PayloadStore {\n  [s: string]: WithStorageMeta\n}\n\n@creatableModule()\nexport class IndexedDbArchivist<\n  TParams extends IndexedDbArchivistParams = IndexedDbArchivistParams,\n  TEventData extends ArchivistModuleEventData = ArchivistModuleEventData,\n> extends AbstractArchivist<TParams, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexedDbArchivistConfigSchema]\n  static override readonly defaultConfigSchema: Schema = IndexedDbArchivistConfigSchema\n  static readonly defaultDbName = 'archivist'\n  static readonly defaultDbVersion = 1\n  static readonly defaultStoreName = 'payloads'\n  private static readonly dataHashIndex: IndexDescription = {\n    key: { _dataHash: 1 }, multiEntry: false, unique: false,\n  }\n\n  private static readonly hashIndex: IndexDescription = {\n    key: { _hash: 1 }, multiEntry: false, unique: true,\n  }\n\n  private static readonly schemaIndex: IndexDescription = {\n    key: { schema: 1 }, multiEntry: false, unique: false,\n  }\n\n  private static readonly sequenceIndex: IndexDescription = {\n    key: { _sequence: 1 }, multiEntry: false, unique: true,\n  }\n\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly hashIndexName = buildStandardIndexName(IndexedDbArchivist.hashIndex)\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly dataHashIndexName = buildStandardIndexName(IndexedDbArchivist.dataHashIndex)\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly schemaIndexName = buildStandardIndexName(IndexedDbArchivist.schemaIndex)\n  // eslint-disable-next-line @typescript-eslint/member-ordering\n  static readonly sequenceIndexName = buildStandardIndexName(IndexedDbArchivist.sequenceIndex)\n\n  private _dbName?: string\n  private _storeName?: string\n\n  /**\n   * The database name. If not supplied via config, it defaults\n   * to the module name (not guaranteed to be unique) and if module\n   * name is not supplied, it defaults to `archivist`. This behavior\n   * biases towards a single, isolated DB per archivist which seems to\n   * make the most sense for 99% of use cases.\n   */\n  get dbName() {\n    if (!this._dbName) {\n      if (this.config?.dbName) {\n        this._dbName = this.config?.dbName\n      } else {\n        if (this.config?.name) {\n          this.logger.warn('No dbName provided, using module name: ', this.config?.name)\n          this._dbName = this.config?.name\n        } else {\n          this.logger.warn('No dbName provided, using default name: ', IndexedDbArchivist.defaultDbName)\n          this._dbName = IndexedDbArchivist.defaultDbName\n        }\n      }\n    }\n    return assertEx(this._dbName)\n  }\n\n  /**\n   * The database version. If not supplied via config, it defaults to 1.\n   */\n  get dbVersion() {\n    return this.config?.dbVersion ?? IndexedDbArchivist.defaultDbVersion\n  }\n\n  override get queries() {\n    return [\n      ArchivistNextQuerySchema,\n      ArchivistAllQuerySchema,\n      ArchivistClearQuerySchema,\n      ArchivistDeleteQuerySchema,\n      ArchivistInsertQuerySchema,\n      ...super.queries,\n    ]\n  }\n\n  /**\n   * The name of the object store. If not supplied via config, it defaults\n   * to `payloads`.\n   */\n  get storeName() {\n    if (!this._storeName) {\n      if (this.config?.storeName) {\n        this._storeName = this.config?.storeName\n      } else {\n        this.logger.warn('No storeName provided, using default name: ', IndexedDbArchivist.defaultStoreName)\n        this._storeName = IndexedDbArchivist.defaultStoreName\n      }\n    }\n    return assertEx(this._storeName)\n  }\n\n  /**\n   * The indexes to create on the store\n   */\n  private get indexes() {\n    return [\n      IndexedDbArchivist.dataHashIndex,\n      IndexedDbArchivist.hashIndex,\n      IndexedDbArchivist.schemaIndex,\n      IndexedDbArchivist.sequenceIndex,\n      ...(this.config?.storage?.indexes ?? []),\n    ]\n  }\n\n  protected override async allHandler(): Promise<WithStorageMeta<Payload>[]> {\n    // Get all payloads from the store\n    const payloads = await this.useDb(db => db.getAll(this.storeName))\n    // Remove any metadata before returning to the client\n    return payloads\n  }\n\n  protected override async clearHandler(): Promise<void> {\n    await this.useDb(db => db.clear(this.storeName))\n  }\n\n  protected override async deleteHandler(hashes: Hash[]): Promise<Hash[]> {\n    // Filter duplicates to prevent unnecessary DB queries\n    const uniqueHashes = [...new Set(hashes)]\n    const pairs = await PayloadBuilder.hashPairs(await this.getHandler(uniqueHashes))\n    const hashesToDelete = (await Promise.all(pairs.map(async (pair) => {\n      const dataHash0 = await PayloadBuilder.dataHash(pair[0])\n      return [dataHash0, pair[1]]\n    }))).flat()\n    // Remove any duplicates\n    const distinctHashes = [...new Set(hashesToDelete)]\n    return await this.useDb(async (db) => {\n      // Only return hashes that were successfully deleted\n      const found = await Promise.all(\n        distinctHashes.map(async (hash) => {\n          // Check if the hash exists\n          const existing\n            = (await db.getKeyFromIndex(this.storeName, IndexedDbArchivist.hashIndexName, hash))\n            ?? (await db.getKeyFromIndex(this.storeName, IndexedDbArchivist.dataHashIndexName, hash))\n          // If it does exist\n          if (existing) {\n            // Delete it\n            await db.delete(this.storeName, existing)\n            // Return the hash so it gets added to the list of deleted hashes\n            return hash\n          }\n        }),\n      )\n      return found.filter(exists).filter(hash => uniqueHashes.includes(hash))\n    })\n  }\n\n  protected async getFromCursor(\n    db: IDBPDatabase<PayloadStore>,\n    storeName: string,\n      order: 'asc' | 'desc' = 'asc',\n      limit: number = 10,\n      cursor?: Hex,\n  ): Promise<WithStorageMeta[]> {\n    // TODO: We have to handle the case where the cursor is not found, and then find the correct cursor to start with (thunked cursor)\n    const transaction = db.transaction(storeName, 'readonly')\n    const store = transaction.objectStore(storeName)\n    const sequenceIndex = assertEx(store.index(IndexedDbArchivist.sequenceIndexName), () => 'Failed to get sequence index')\n    let sequenceCursor: IDBPCursorWithValue<PayloadStore, [string]> | null | undefined = undefined\n    const parsedCursor = cursor === SequenceConstants.minLocalSequence ? null : cursor\n    sequenceCursor = assertEx(await sequenceIndex.openCursor(\n      null,\n      order === 'desc' ? 'prev' : 'next',\n    ), () => `Failed to get cursor [${parsedCursor}, ${cursor}]`)\n    if (!sequenceCursor?.value) return []\n    try {\n      sequenceCursor = parsedCursor\n        ? sequenceCursor.value._sequence === parsedCursor ? await sequenceCursor?.advance(1) : await (await sequenceCursor?.continue(parsedCursor))?.advance(1)\n        : sequenceCursor // advance to skip the initial value\n    } catch {\n      return []\n    }\n\n    let remaining = limit\n    const result: WithStorageMeta[] = []\n    while (remaining) {\n      const value = sequenceCursor?.value\n      if (value) {\n        result.push(value)\n        try {\n          sequenceCursor = await sequenceCursor?.advance(1)\n        } catch {\n          break\n        }\n        if (sequenceCursor === null) {\n          break\n        }\n      }\n      remaining--\n    }\n    return result\n  }\n\n  /**\n   * Uses an index to get a payload by the index value, but returns the value with the primary key (from the root store)\n   * @param db The db instance to use\n   * @param storeName The name of the store to use\n   * @param indexName The index to use\n   * @param key The key to get from the index\n   * @returns The primary key and the payload, or undefined if not found\n   */\n  protected async getFromIndexWithPrimaryKey(\n    db: IDBPDatabase<PayloadStore>,\n    storeName: string,\n    indexName: string,\n    key: IDBValidKey,\n  ): Promise<[number, WithStorageMeta] | undefined> {\n    const transaction = db.transaction(storeName, 'readonly')\n    const store = transaction.objectStore(storeName)\n    const index = store.index(indexName)\n    const cursor = await index.openCursor(key)\n    if (cursor) {\n      const singleValue = cursor.value\n      // NOTE: It's known to be a number because we are using IndexedDB supplied auto-incrementing keys\n      if (typeof cursor.primaryKey !== 'number') {\n        throw new TypeError('primaryKey must be a number')\n      }\n\n      return [cursor.primaryKey, singleValue]\n    }\n  }\n\n  protected override async getHandler(hashes: string[]): Promise<WithStorageMeta[]> {\n    const payloads = await this.useDb(db =>\n      Promise.all(\n        // Filter duplicates to prevent unnecessary DB queries\n        uniq(hashes).map(async (hash) => {\n          // Find by hash\n          const payload = await this.getFromIndexWithPrimaryKey(db, this.storeName, IndexedDbArchivist.hashIndexName, hash)\n          // If found, return\n          if (payload) return payload\n          // Otherwise, find by data hash\n          return this.getFromIndexWithPrimaryKey(db, this.storeName, IndexedDbArchivist.dataHashIndexName, hash)\n        }),\n      ))\n\n    const found = new Set<string>()\n    return (\n      payloads\n        // Filter out not found\n        .filter(exists)\n        // Sort by primary key\n        .sort((a, b) => a![0] - b![0])\n        // Filter out duplicates by hash\n        .filter(([_key, payload]) => {\n          if (found.has(payload._hash)) {\n            return false\n          } else {\n            found.add(payload._hash)\n            return true\n          }\n        })\n        // Return just the payloads\n        .map(([_key, payload]) => payload)\n    )\n  }\n\n  protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {\n    // Get the unique pairs of payloads and their hashes\n    const payloadWithStorageMeta = (await PayloadBuilder.addStorageMeta(payloads)).sort(PayloadBuilder.compareStorageMeta)\n    return await this.useDb(async (db) => {\n      // Perform all inserts via a single transaction to ensure atomicity\n      // with respect to checking for the pre-existence of the hash.\n      // This is done to prevent duplicate root hashes due to race\n      // conditions between checking vs insertion.\n      const tx = db.transaction(this.storeName, 'readwrite')\n      // Get the object store\n      const store = tx.objectStore(this.storeName)\n      // Return only the payloads that were successfully inserted\n      const inserted: WithStorageMeta<Payload>[] = []\n      try {\n        await Promise.all(\n          payloadWithStorageMeta.map(async (payload) => {\n            // only insert if hash does not already exist\n            if (!await store.index(IndexedDbArchivist.hashIndexName).get(payload._hash)) {\n            // Insert the payload\n              await store.put(payload)\n              // Add it to the inserted list\n              inserted.push(payload)\n            }\n          }),\n        )\n      } finally {\n        // Ensure the transaction is closed\n        await tx.done\n      }\n      return inserted\n    })\n  }\n\n  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {\n    const {\n      limit, cursor, order,\n    } = options ?? {}\n    return await this.useDb(async (db) => {\n      return await this.getFromCursor(db, this.storeName, order, limit ?? 10, cursor)\n    })\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    // NOTE: We could defer this creation to first access but we\n    // want to fail fast here in case something is wrong\n    await this.useDb(() => {})\n    return true\n  }\n\n  /**\n   * Returns that the desired DB/Store initialized to the correct version\n   * @returns The initialized DB\n   */\n  private async getInitializedDb(): Promise<IDBPDatabase<PayloadStore>> {\n    const {\n      dbName, dbVersion, indexes, storeName, logger,\n    } = this\n    return await openDB<PayloadStore>(dbName, dbVersion, {\n      blocked(currentVersion, blockedVersion, event) {\n        logger.warn(`IndexedDbArchivist: Blocked from upgrading from ${currentVersion} to ${blockedVersion}`, event)\n      },\n      blocking(currentVersion, blockedVersion, event) {\n        logger.warn(`IndexedDbArchivist: Blocking upgrade from ${currentVersion} to ${blockedVersion}`, event)\n      },\n      terminated() {\n        logger.log('IndexedDbArchivist: Terminated')\n      },\n      upgrade(database, oldVersion, newVersion, transaction) {\n        // NOTE: This is called whenever the DB is created/updated. We could simply ensure the desired end\n        // state but, out of an abundance of caution, we will just delete (so we know where we are starting\n        // from a known good point) and recreate the desired state. This prioritizes resilience over data\n        // retention but we can revisit that tradeoff when it becomes limiting. Because distributed browser\n        // state is extremely hard to debug, this seems like fair tradeoff for now.\n        if (oldVersion !== newVersion) {\n          logger.log(`IndexedDbArchivist: Upgrading from ${oldVersion} to ${newVersion}`)\n          // Delete any existing databases that are not the current version\n          const objectStores = transaction.objectStoreNames\n          for (const name of objectStores) {\n            try {\n              database.deleteObjectStore(name)\n            } catch {\n              logger.log(`IndexedDbArchivist: Failed to delete existing object store ${name}`)\n            }\n          }\n        }\n        // Create the store\n        const store = database.createObjectStore(storeName, {\n          // If it isn't explicitly set, create a value by auto incrementing.\n          autoIncrement: true,\n        })\n        // Name the store\n        store.name = storeName\n        // Create an index on the hash\n        for (const {\n          key, multiEntry, unique,\n        } of indexes) {\n          const indexKeys = Object.keys(key)\n          const keys = indexKeys.length === 1 ? indexKeys[0] : indexKeys\n          const indexName = buildStandardIndexName({ key, unique })\n          store.createIndex(indexName, keys, { multiEntry, unique })\n        }\n      },\n    })\n  }\n\n  /**\n   * Executes a callback with the initialized DB and then closes the db\n   * @param callback The method to execute with the initialized DB\n   * @returns\n   */\n  private async useDb<T>(callback: (db: IDBPDatabase<PayloadStore>) => Promise<T> | T): Promise<T> {\n    // Get the initialized DB\n    const db = await this.getInitializedDb()\n    try {\n      // Perform the callback\n      return await callback(db)\n    } finally {\n      // Close the DB\n      db.close()\n    }\n  }\n}\n","export type IndexedDbArchivistSchema = 'network.xyo.archivist.indexeddb'\nexport const IndexedDbArchivistSchema: IndexedDbArchivistSchema = 'network.xyo.archivist.indexeddb'\n","import type { ArchivistConfig } from '@xyo-network/archivist-model'\n\nimport { IndexedDbArchivistSchema } from './Schema.ts'\n\nexport type IndexedDbArchivistConfigSchema = `${IndexedDbArchivistSchema}.config`\nexport const IndexedDbArchivistConfigSchema: IndexedDbArchivistConfigSchema = `${IndexedDbArchivistSchema}.config`\n\nexport type IndexedDbArchivistConfig = ArchivistConfig<{\n  /**\n   * The database name\n   */\n  dbName?: string\n  /**\n   * The version of the DB, defaults to 1\n   */\n  dbVersion?: number\n  schema: IndexedDbArchivistConfigSchema\n  /**\n   * The name of the object store\n   */\n  storeName?: string\n}>\n"],"mappings":";;;;AACA,SAASA,YAAY;AACrB,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAEvB,SAASC,yBAAyB;AAClC,SACEC,yBACAC,2BACAC,4BACAC,4BAGAC,0BACAC,8BAEK;AACP,SAASC,uBAAuB;AAChC,SAASC,sBAAsB;AAC/B,SACmBC,yBACZ;AACP,SACqCC,cAC9B;;;ACvBA,IAAMC,2BAAqD;;;ACI3D,IAAMC,iCAAiE,GAAGC,wBAAAA;;;AFLpD,SAAA,aAAA,YAAA,QAAA,KAAA,MAAA;;;;;;AAAA;AAkCtB,IAAMC,qBAAN,MAAMA,4BAGHC,kBAAAA;EArCV,OAqCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EACvD,OAAgBE,gBAAgB;EAChC,OAAgBC,mBAAmB;EACnC,OAAgBC,mBAAmB;EACnC,OAAwBC,gBAAkC;IACxDC,KAAK;MAAEC,WAAW;IAAE;IAAGC,YAAY;IAAOC,QAAQ;EACpD;EAEA,OAAwBC,YAA8B;IACpDJ,KAAK;MAAEK,OAAO;IAAE;IAAGH,YAAY;IAAOC,QAAQ;EAChD;EAEA,OAAwBG,cAAgC;IACtDN,KAAK;MAAEO,QAAQ;IAAE;IAAGL,YAAY;IAAOC,QAAQ;EACjD;EAEA,OAAwBK,gBAAkC;IACxDR,KAAK;MAAES,WAAW;IAAE;IAAGP,YAAY;IAAOC,QAAQ;EACpD;;EAGA,OAAgBO,gBAAgBC,uBAAuBpB,oBAAmBa,SAAS;;EAEnF,OAAgBQ,oBAAoBD,uBAAuBpB,oBAAmBQ,aAAa;;EAE3F,OAAgBc,kBAAkBF,uBAAuBpB,oBAAmBe,WAAW;;EAEvF,OAAgBQ,oBAAoBH,uBAAuBpB,oBAAmBiB,aAAa;EAEnFO;EACAC;;;;;;;;EASR,IAAIC,SAAS;AACX,QAAI,CAAC,KAAKF,SAAS;AACjB,UAAI,KAAKG,QAAQD,QAAQ;AACvB,aAAKF,UAAU,KAAKG,QAAQD;MAC9B,OAAO;AACL,YAAI,KAAKC,QAAQC,MAAM;AACrB,eAAKC,OAAOC,KAAK,2CAA2C,KAAKH,QAAQC,IAAAA;AACzE,eAAKJ,UAAU,KAAKG,QAAQC;QAC9B,OAAO;AACL,eAAKC,OAAOC,KAAK,4CAA4C9B,oBAAmBK,aAAa;AAC7F,eAAKmB,UAAUxB,oBAAmBK;QACpC;MACF;IACF;AACA,WAAO0B,SAAS,KAAKP,OAAO;EAC9B;;;;EAKA,IAAIQ,YAAY;AACd,WAAO,KAAKL,QAAQK,aAAahC,oBAAmBM;EACtD;EAEA,IAAa2B,UAAU;AACrB,WAAO;MACLC;MACAC;MACAC;MACAC;MACAC;SACG,MAAML;;EAEb;;;;;EAMA,IAAIM,YAAY;AACd,QAAI,CAAC,KAAKd,YAAY;AACpB,UAAI,KAAKE,QAAQY,WAAW;AAC1B,aAAKd,aAAa,KAAKE,QAAQY;MACjC,OAAO;AACL,aAAKV,OAAOC,KAAK,+CAA+C9B,oBAAmBO,gBAAgB;AACnG,aAAKkB,aAAazB,oBAAmBO;MACvC;IACF;AACA,WAAOwB,SAAS,KAAKN,UAAU;EACjC;;;;EAKA,IAAYe,UAAU;AACpB,WAAO;MACLxC,oBAAmBQ;MACnBR,oBAAmBa;MACnBb,oBAAmBe;MACnBf,oBAAmBiB;SACf,KAAKU,QAAQc,SAASD,WAAW,CAAA;;EAEzC;EAEA,MAAyBE,aAAkD;AAEzE,UAAMC,WAAW,MAAM,KAAKC,MAAMC,CAAAA,OAAMA,GAAGC,OAAO,KAAKP,SAAS,CAAA;AAEhE,WAAOI;EACT;EAEA,MAAyBI,eAA8B;AACrD,UAAM,KAAKH,MAAMC,CAAAA,OAAMA,GAAGG,MAAM,KAAKT,SAAS,CAAA;EAChD;EAEA,MAAyBU,cAAcC,QAAiC;AAEtE,UAAMC,eAAe;SAAI,IAAIC,IAAIF,MAAAA;;AACjC,UAAMG,QAAQ,MAAMC,eAAeC,UAAU,MAAM,KAAKC,WAAWL,YAAAA,CAAAA;AACnE,UAAMM,kBAAkB,MAAMC,QAAQC,IAAIN,MAAMO,IAAI,OAAOC,SAAAA;AACzD,YAAMC,YAAY,MAAMR,eAAeS,SAASF,KAAK,CAAA,CAAE;AACvD,aAAO;QAACC;QAAWD,KAAK,CAAA;;IAC1B,CAAA,CAAA,GAAKG,KAAI;AAET,UAAMC,iBAAiB;SAAI,IAAIb,IAAIK,cAAAA;;AACnC,WAAO,MAAM,KAAKb,MAAM,OAAOC,OAAAA;AAE7B,YAAMqB,QAAQ,MAAMR,QAAQC,IAC1BM,eAAeL,IAAI,OAAOO,SAAAA;AAExB,cAAMC,WACD,MAAMvB,GAAGwB,gBAAgB,KAAK9B,WAAWvC,oBAAmBmB,eAAegD,IAAAA,KAC1E,MAAMtB,GAAGwB,gBAAgB,KAAK9B,WAAWvC,oBAAmBqB,mBAAmB8C,IAAAA;AAErF,YAAIC,UAAU;AAEZ,gBAAMvB,GAAGyB,OAAO,KAAK/B,WAAW6B,QAAAA;AAEhC,iBAAOD;QACT;MACF,CAAA,CAAA;AAEF,aAAOD,MAAMK,OAAOC,MAAAA,EAAQD,OAAOJ,CAAAA,SAAQhB,aAAasB,SAASN,IAAAA,CAAAA;IACnE,CAAA;EACF;EAEA,MAAgBO,cACd7B,IACAN,WACEoC,QAAwB,OACxBC,QAAgB,IAChBC,QAC0B;AAE5B,UAAMC,cAAcjC,GAAGiC,YAAYvC,WAAW,UAAA;AAC9C,UAAMwC,QAAQD,YAAYE,YAAYzC,SAAAA;AACtC,UAAMtB,gBAAgBc,SAASgD,MAAME,MAAMjF,oBAAmBuB,iBAAiB,GAAG,MAAM,8BAAA;AACxF,QAAI2D,iBAAiFC;AACrF,UAAMC,eAAeP,WAAWQ,kBAAkBC,mBAAmB,OAAOT;AAC5EK,qBAAiBnD,SAAS,MAAMd,cAAcsE,WAC5C,MACAZ,UAAU,SAAS,SAAS,MAAA,GAC3B,MAAM,yBAAyBS,YAAAA,KAAiBP,MAAAA,GAAS;AAC5D,QAAI,CAACK,gBAAgBM,MAAO,QAAO,CAAA;AACnC,QAAI;AACFN,uBAAiBE,eACbF,eAAeM,MAAMtE,cAAckE,eAAe,MAAMF,gBAAgBO,QAAQ,CAAA,IAAK,OAAO,MAAMP,gBAAgBQ,SAASN,YAAAA,IAAgBK,QAAQ,CAAA,IACnJP;IACN,QAAQ;AACN,aAAO,CAAA;IACT;AAEA,QAAIS,YAAYf;AAChB,UAAMgB,SAA4B,CAAA;AAClC,WAAOD,WAAW;AAChB,YAAMH,QAAQN,gBAAgBM;AAC9B,UAAIA,OAAO;AACTI,eAAOC,KAAKL,KAAAA;AACZ,YAAI;AACFN,2BAAiB,MAAMA,gBAAgBO,QAAQ,CAAA;QACjD,QAAQ;AACN;QACF;AACA,YAAIP,mBAAmB,MAAM;AAC3B;QACF;MACF;AACAS;IACF;AACA,WAAOC;EACT;;;;;;;;;EAUA,MAAgBE,2BACdjD,IACAN,WACAwD,WACAtF,KACgD;AAChD,UAAMqE,cAAcjC,GAAGiC,YAAYvC,WAAW,UAAA;AAC9C,UAAMwC,QAAQD,YAAYE,YAAYzC,SAAAA;AACtC,UAAM0C,QAAQF,MAAME,MAAMc,SAAAA;AAC1B,UAAMlB,SAAS,MAAMI,MAAMM,WAAW9E,GAAAA;AACtC,QAAIoE,QAAQ;AACV,YAAMmB,cAAcnB,OAAOW;AAE3B,UAAI,OAAOX,OAAOoB,eAAe,UAAU;AACzC,cAAM,IAAIC,UAAU,6BAAA;MACtB;AAEA,aAAO;QAACrB,OAAOoB;QAAYD;;IAC7B;EACF;EAEA,MAAyBxC,WAAWN,QAA8C;AAChF,UAAMP,WAAW,MAAM,KAAKC,MAAMC,CAAAA,OAChCa,QAAQC;;MAENwC,KAAKjD,MAAAA,EAAQU,IAAI,OAAOO,SAAAA;AAEtB,cAAMiC,UAAU,MAAM,KAAKN,2BAA2BjD,IAAI,KAAKN,WAAWvC,oBAAmBmB,eAAegD,IAAAA;AAE5G,YAAIiC,QAAS,QAAOA;AAEpB,eAAO,KAAKN,2BAA2BjD,IAAI,KAAKN,WAAWvC,oBAAmBqB,mBAAmB8C,IAAAA;MACnG,CAAA;IAAA,CAAA;AAGJ,UAAMD,QAAQ,oBAAId,IAAAA;AAClB,WACET,SAEG4B,OAAOC,MAAAA,EAEP6B,KAAK,CAACC,GAAGC,MAAMD,EAAG,CAAA,IAAKC,EAAG,CAAA,CAAE,EAE5BhC,OAAO,CAAC,CAACiC,MAAMJ,OAAAA,MAAQ;AACtB,UAAIlC,MAAMuC,IAAIL,QAAQtF,KAAK,GAAG;AAC5B,eAAO;MACT,OAAO;AACLoD,cAAMwC,IAAIN,QAAQtF,KAAK;AACvB,eAAO;MACT;IACF,CAAA,EAEC8C,IAAI,CAAC,CAAC4C,MAAMJ,OAAAA,MAAaA,OAAAA;EAEhC;EAEA,MAAyBO,cAAchE,UAA0D;AAE/F,UAAMiE,0BAA0B,MAAMtD,eAAeuD,eAAelE,QAAAA,GAAW0D,KAAK/C,eAAewD,kBAAkB;AACrH,WAAO,MAAM,KAAKlE,MAAM,OAAOC,OAAAA;AAK7B,YAAMkE,KAAKlE,GAAGiC,YAAY,KAAKvC,WAAW,WAAA;AAE1C,YAAMwC,QAAQgC,GAAG/B,YAAY,KAAKzC,SAAS;AAE3C,YAAMyE,WAAuC,CAAA;AAC7C,UAAI;AACF,cAAMtD,QAAQC,IACZiD,uBAAuBhD,IAAI,OAAOwC,YAAAA;AAEhC,cAAI,CAAC,MAAMrB,MAAME,MAAMjF,oBAAmBmB,aAAa,EAAE8F,IAAIb,QAAQtF,KAAK,GAAG;AAE3E,kBAAMiE,MAAMmC,IAAId,OAAAA;AAEhBY,qBAASnB,KAAKO,OAAAA;UAChB;QACF,CAAA,CAAA;MAEJ,UAAA;AAEE,cAAMW,GAAGI;MACX;AACA,aAAOH;IACT,CAAA;EACF;EAEA,MAAyBI,YAAYC,SAAqE;AACxG,UAAM,EACJzC,OAAOC,QAAQF,MAAK,IAClB0C,WAAW,CAAC;AAChB,WAAO,MAAM,KAAKzE,MAAM,OAAOC,OAAAA;AAC7B,aAAO,MAAM,KAAK6B,cAAc7B,IAAI,KAAKN,WAAWoC,OAAOC,SAAS,IAAIC,MAAAA;IAC1E,CAAA;EACF;EAEA,MAAyByC,eAAe;AACtC,UAAM,MAAMA,aAAAA;AAGZ,UAAM,KAAK1E,MAAM,MAAA;IAAO,CAAA;AACxB,WAAO;EACT;;;;;EAMA,MAAc2E,mBAAwD;AACpE,UAAM,EACJ7F,QAAQM,WAAWQ,SAASD,WAAWV,OAAM,IAC3C;AACJ,WAAO,MAAM2F,OAAqB9F,QAAQM,WAAW;MACnDyF,QAAQC,gBAAgBC,gBAAgBC,OAAK;AAC3C/F,eAAOC,KAAK,mDAAmD4F,cAAAA,OAAqBC,cAAAA,IAAkBC,KAAAA;MACxG;MACAC,SAASH,gBAAgBC,gBAAgBC,OAAK;AAC5C/F,eAAOC,KAAK,6CAA6C4F,cAAAA,OAAqBC,cAAAA,IAAkBC,KAAAA;MAClG;MACAE,aAAAA;AACEjG,eAAOkG,IAAI,gCAAA;MACb;MACAC,QAAQC,UAAUC,YAAYC,YAAYrD,aAAW;AAMnD,YAAIoD,eAAeC,YAAY;AAC7BtG,iBAAOkG,IAAI,sCAAsCG,UAAAA,OAAiBC,UAAAA,EAAY;AAE9E,gBAAMC,eAAetD,YAAYuD;AACjC,qBAAWzG,QAAQwG,cAAc;AAC/B,gBAAI;AACFH,uBAASK,kBAAkB1G,IAAAA;YAC7B,QAAQ;AACNC,qBAAOkG,IAAI,8DAA8DnG,IAAAA,EAAM;YACjF;UACF;QACF;AAEA,cAAMmD,QAAQkD,SAASM,kBAAkBhG,WAAW;;UAElDiG,eAAe;QACjB,CAAA;AAEAzD,cAAMnD,OAAOW;AAEb,mBAAW,EACT9B,KAAKE,YAAYC,OAAM,KACpB4B,SAAS;AACZ,gBAAMiG,YAAYC,OAAOC,KAAKlI,GAAAA;AAC9B,gBAAMkI,OAAOF,UAAUG,WAAW,IAAIH,UAAU,CAAA,IAAKA;AACrD,gBAAM1C,YAAY3E,uBAAuB;YAAEX;YAAKG;UAAO,CAAA;AACvDmE,gBAAM8D,YAAY9C,WAAW4C,MAAM;YAAEhI;YAAYC;UAAO,CAAA;QAC1D;MACF;IACF,CAAA;EACF;;;;;;EAOA,MAAcgC,MAASkG,UAA0E;AAE/F,UAAMjG,KAAK,MAAM,KAAK0E,iBAAgB;AACtC,QAAI;AAEF,aAAO,MAAMuB,SAASjG,EAAAA;IACxB,UAAA;AAEEA,SAAGkG,MAAK;IACV;EACF;AACF;;;;","names":["uniq","assertEx","exists","AbstractArchivist","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","ArchivistNextQuerySchema","buildStandardIndexName","creatableModule","PayloadBuilder","SequenceConstants","openDB","IndexedDbArchivistSchema","IndexedDbArchivistConfigSchema","IndexedDbArchivistSchema","IndexedDbArchivist","AbstractArchivist","configSchemas","IndexedDbArchivistConfigSchema","defaultConfigSchema","defaultDbName","defaultDbVersion","defaultStoreName","dataHashIndex","key","_dataHash","multiEntry","unique","hashIndex","_hash","schemaIndex","schema","sequenceIndex","_sequence","hashIndexName","buildStandardIndexName","dataHashIndexName","schemaIndexName","sequenceIndexName","_dbName","_storeName","dbName","config","name","logger","warn","assertEx","dbVersion","queries","ArchivistNextQuerySchema","ArchivistAllQuerySchema","ArchivistClearQuerySchema","ArchivistDeleteQuerySchema","ArchivistInsertQuerySchema","storeName","indexes","storage","allHandler","payloads","useDb","db","getAll","clearHandler","clear","deleteHandler","hashes","uniqueHashes","Set","pairs","PayloadBuilder","hashPairs","getHandler","hashesToDelete","Promise","all","map","pair","dataHash0","dataHash","flat","distinctHashes","found","hash","existing","getKeyFromIndex","delete","filter","exists","includes","getFromCursor","order","limit","cursor","transaction","store","objectStore","index","sequenceCursor","undefined","parsedCursor","SequenceConstants","minLocalSequence","openCursor","value","advance","continue","remaining","result","push","getFromIndexWithPrimaryKey","indexName","singleValue","primaryKey","TypeError","uniq","payload","sort","a","b","_key","has","add","insertHandler","payloadWithStorageMeta","addStorageMeta","compareStorageMeta","tx","inserted","get","put","done","nextHandler","options","startHandler","getInitializedDb","openDB","blocked","currentVersion","blockedVersion","event","blocking","terminated","log","upgrade","database","oldVersion","newVersion","objectStores","objectStoreNames","deleteObjectStore","createObjectStore","autoIncrement","indexKeys","Object","keys","length","createIndex","callback","close"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/archivist-indexeddb",
-  "version": "3.5.2",
+  "version": "3.6.0-rc.10",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,29 +29,31 @@
   "module": "dist/browser/index.mjs",
   "types": "dist/browser/index.d.ts",
   "dependencies": {
-    "@xylabs/array": "^4.4.9",
-    "@xylabs/assert": "^4.4.9",
-    "@xylabs/exists": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xyo-network/archivist-abstract": "^3.5.2",
-    "@xyo-network/archivist-model": "^3.5.2",
-    "@xyo-network/module-model": "^3.5.2",
-    "@xyo-network/payload-builder": "^3.5.2",
-    "@xyo-network/payload-model": "^3.5.2",
-    "idb": "^8.0.0"
+    "@xylabs/array": "^4.4.21",
+    "@xylabs/assert": "^4.4.21",
+    "@xylabs/exists": "^4.4.21",
+    "@xylabs/hex": "^4.4.21",
+    "@xyo-network/archivist-abstract": "^3.6.0-rc.10",
+    "@xyo-network/archivist-model": "^3.6.0-rc.10",
+    "@xyo-network/module-model": "^3.6.0-rc.10",
+    "@xyo-network/payload-builder": "^3.6.0-rc.10",
+    "@xyo-network/payload-model": "^3.6.0-rc.10",
+    "idb": "^8.0.1"
   },
   "devDependencies": {
-    "@xylabs/object": "^4.4.9",
-    "@xylabs/ts-scripts-yarn3": "^4.2.4",
-    "@xylabs/tsconfig": "^4.2.4",
-    "@xyo-network/account": "^3.5.2",
-    "@xyo-network/id-payload-plugin": "^3.5.2",
-    "@xyo-network/payload-wrapper": "^3.5.2",
+    "@xylabs/delay": "^4.4.21",
+    "@xylabs/object": "^4.4.21",
+    "@xylabs/ts-scripts-yarn3": "^4.2.6",
+    "@xylabs/tsconfig": "^4.2.6",
+    "@xyo-network/account": "^3.6.0-rc.10",
+    "@xyo-network/id-payload-plugin": "^3.6.0-rc.10",
+    "@xyo-network/payload-wrapper": "^3.6.0-rc.10",
     "fake-indexeddb": "^6.0.0",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.5.2"
 }

package/src/Archivist.ts

@@ -1,7 +1,8 @@
-import { uniq, uniqBy } from '@xylabs/array'
+/* eslint-disable complexity */
+import { uniq } from '@xylabs/array'
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
-import { Hash } from '@xylabs/hex'
+import { Hash, Hex } from '@xylabs/hex'
 import { AbstractArchivist } from '@xyo-network/archivist-abstract'
 import {
   ArchivistAllQuerySchema,
@@ -17,7 +18,7 @@ import {
 import { creatableModule } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import {
-  Payload, PayloadWithMeta, Schema,
+  Payload, Schema, SequenceConstants, WithStorageMeta,
 } from '@xyo-network/payload-model'
 import {
   IDBPCursorWithValue, IDBPDatabase, openDB,
@@ -26,10 +27,8 @@ import {
 import { IndexedDbArchivistConfigSchema } from './Config.ts'
 import { IndexedDbArchivistParams } from './Params.ts'
 
-type StoredPayload = PayloadWithMeta & { _hash: string }
-
 export interface PayloadStore {
-  [s: string]: StoredPayload
+  [s: string]: WithStorageMeta
 }
 
 @creatableModule()
@@ -43,7 +42,7 @@ export class IndexedDbArchivist<
   static readonly defaultDbVersion = 1
   static readonly defaultStoreName = 'payloads'
   private static readonly dataHashIndex: IndexDescription = {
-    key: { $hash: 1 }, multiEntry: false, unique: false,
+    key: { _dataHash: 1 }, multiEntry: false, unique: false,
   }
 
   private static readonly hashIndex: IndexDescription = {
@@ -54,12 +53,18 @@ export class IndexedDbArchivist<
     key: { schema: 1 }, multiEntry: false, unique: false,
   }
 
+  private static readonly sequenceIndex: IndexDescription = {
+    key: { _sequence: 1 }, multiEntry: false, unique: true,
+  }
+
   // eslint-disable-next-line @typescript-eslint/member-ordering
   static readonly hashIndexName = buildStandardIndexName(IndexedDbArchivist.hashIndex)
   // eslint-disable-next-line @typescript-eslint/member-ordering
   static readonly dataHashIndexName = buildStandardIndexName(IndexedDbArchivist.dataHashIndex)
   // eslint-disable-next-line @typescript-eslint/member-ordering
   static readonly schemaIndexName = buildStandardIndexName(IndexedDbArchivist.schemaIndex)
+  // eslint-disable-next-line @typescript-eslint/member-ordering
+  static readonly sequenceIndexName = buildStandardIndexName(IndexedDbArchivist.sequenceIndex)
 
   private _dbName?: string
   private _storeName?: string
@@ -126,14 +131,20 @@ export class IndexedDbArchivist<
    * The indexes to create on the store
    */
   private get indexes() {
-    return [IndexedDbArchivist.dataHashIndex, IndexedDbArchivist.hashIndex, IndexedDbArchivist.schemaIndex, ...(this.config?.storage?.indexes ?? [])]
+    return [
+      IndexedDbArchivist.dataHashIndex,
+      IndexedDbArchivist.hashIndex,
+      IndexedDbArchivist.schemaIndex,
+      IndexedDbArchivist.sequenceIndex,
+      ...(this.config?.storage?.indexes ?? []),
+    ]
   }
 
-  protected override async allHandler(): Promise<StoredPayload[]> {
+  protected override async allHandler(): Promise<WithStorageMeta<Payload>[]> {
     // Get all payloads from the store
     const payloads = await this.useDb(db => db.getAll(this.storeName))
     // Remove any metadata before returning to the client
-    return await Promise.all(payloads.map(payload => PayloadBuilder.build(payload)))
+    return payloads
   }
 
   protected override async clearHandler(): Promise<void> {
@@ -144,7 +155,10 @@ export class IndexedDbArchivist<
     // Filter duplicates to prevent unnecessary DB queries
     const uniqueHashes = [...new Set(hashes)]
     const pairs = await PayloadBuilder.hashPairs(await this.getHandler(uniqueHashes))
-    const hashesToDelete = pairs.flatMap<Hash>(pair => [pair[0].$hash, pair[1]])
+    const hashesToDelete = (await Promise.all(pairs.map(async (pair) => {
+      const dataHash0 = await PayloadBuilder.dataHash(pair[0])
+      return [dataHash0, pair[1]]
+    }))).flat()
     // Remove any duplicates
     const distinctHashes = [...new Set(hashesToDelete)]
     return await this.useDb(async (db) => {
@@ -168,6 +182,52 @@ export class IndexedDbArchivist<
     })
   }
 
+  protected async getFromCursor(
+    db: IDBPDatabase<PayloadStore>,
+    storeName: string,
+      order: 'asc' | 'desc' = 'asc',
+      limit: number = 10,
+      cursor?: Hex,
+  ): Promise<WithStorageMeta[]> {
+    // TODO: We have to handle the case where the cursor is not found, and then find the correct cursor to start with (thunked cursor)
+    const transaction = db.transaction(storeName, 'readonly')
+    const store = transaction.objectStore(storeName)
+    const sequenceIndex = assertEx(store.index(IndexedDbArchivist.sequenceIndexName), () => 'Failed to get sequence index')
+    let sequenceCursor: IDBPCursorWithValue<PayloadStore, [string]> | null | undefined = undefined
+    const parsedCursor = cursor === SequenceConstants.minLocalSequence ? null : cursor
+    sequenceCursor = assertEx(await sequenceIndex.openCursor(
+      null,
+      order === 'desc' ? 'prev' : 'next',
+    ), () => `Failed to get cursor [${parsedCursor}, ${cursor}]`)
+    if (!sequenceCursor?.value) return []
+    try {
+      sequenceCursor = parsedCursor
+        ? sequenceCursor.value._sequence === parsedCursor ? await sequenceCursor?.advance(1) : await (await sequenceCursor?.continue(parsedCursor))?.advance(1)
+        : sequenceCursor // advance to skip the initial value
+    } catch {
+      return []
+    }
+
+    let remaining = limit
+    const result: WithStorageMeta[] = []
+    while (remaining) {
+      const value = sequenceCursor?.value
+      if (value) {
+        result.push(value)
+        try {
+          sequenceCursor = await sequenceCursor?.advance(1)
+        } catch {
+          break
+        }
+        if (sequenceCursor === null) {
+          break
+        }
+      }
+      remaining--
+    }
+    return result
+  }
+
   /**
    * Uses an index to get a payload by the index value, but returns the value with the primary key (from the root store)
    * @param db The db instance to use
@@ -181,7 +241,7 @@ export class IndexedDbArchivist<
     storeName: string,
     indexName: string,
     key: IDBValidKey,
-  ): Promise<[number, StoredPayload] | undefined> {
+  ): Promise<[number, WithStorageMeta] | undefined> {
     const transaction = db.transaction(storeName, 'readonly')
     const store = transaction.objectStore(storeName)
     const index = store.index(indexName)
@@ -197,56 +257,7 @@ export class IndexedDbArchivist<
     }
   }
 
-  // eslint-disable-next-line complexity
-  protected async getFromOffset(
-    db: IDBPDatabase<PayloadStore>,
-    storeName: string,
-    order: 'asc' | 'desc' = 'asc',
-    limit: number = 10,
-    offset?: Hash,
-  ): Promise<StoredPayload[]> {
-    const transaction = db.transaction(storeName, 'readonly')
-    const store = transaction.objectStore(storeName)
-    const hashIndex = store.index(IndexedDbArchivist.hashIndexName)
-    let primaryCursor: IDBPCursorWithValue<PayloadStore, [string]> | null | undefined = undefined
-    if (offset) {
-      const hashCursor = assertEx(await hashIndex.openCursor(offset), () => 'Failed to get cursor')
-      const startPrimaryKey = (hashCursor?.primaryKey ?? 0) as number // we know the primary key is a number and starts at 1
-      primaryCursor = await (order === 'desc'
-        ? store.openCursor(IDBKeyRange.upperBound(startPrimaryKey), 'prev')
-        : store.openCursor(IDBKeyRange.lowerBound(startPrimaryKey), 'next'))
-      if (!primaryCursor?.value) return []
-      try {
-        primaryCursor = await primaryCursor?.advance(1) // advance to skip the offset value
-      } catch {
-        return []
-      }
-    } else {
-      primaryCursor = await store.openCursor(null, order === 'desc' ? 'prev' : 'next')
-      if (!primaryCursor?.value) return []
-    }
-
-    let remaining = limit
-    const result: StoredPayload[] = []
-    while (remaining) {
-      const value = primaryCursor?.value
-      if (value) {
-        result.push(value)
-        try {
-          primaryCursor = await primaryCursor?.advance(1)
-        } catch {
-          break
-        }
-        if (primaryCursor === null) {
-          break
-        }
-      }
-      remaining--
-    }
-    return result
-  }
-
-  protected override async getHandler(hashes: string[]): Promise<StoredPayload[]> {
+  protected override async getHandler(hashes: string[]): Promise<WithStorageMeta[]> {
     const payloads = await this.useDb(db =>
       Promise.all(
         // Filter duplicates to prevent unnecessary DB queries
@@ -281,9 +292,9 @@ export class IndexedDbArchivist<
     )
   }
 
-  protected override async insertHandler(payloads: Payload[]): Promise<PayloadWithMeta[]> {
+  protected override async insertHandler(payloads: Payload[]): Promise<WithStorageMeta<Payload>[]> {
     // Get the unique pairs of payloads and their hashes
-    const uniquePayloadHashPairs = uniqBy(await PayloadBuilder.hashPairs(payloads), ([, _hash]) => _hash)
+    const payloadWithStorageMeta = (await PayloadBuilder.addStorageMeta(payloads)).sort(PayloadBuilder.compareStorageMeta)
     return await this.useDb(async (db) => {
       // Perform all inserts via a single transaction to ensure atomicity
       // with respect to checking for the pre-existence of the hash.
@@ -293,16 +304,14 @@ export class IndexedDbArchivist<
       // Get the object store
       const store = tx.objectStore(this.storeName)
       // Return only the payloads that were successfully inserted
-      const inserted: PayloadWithMeta[] = []
+      const inserted: WithStorageMeta<Payload>[] = []
       try {
         await Promise.all(
-          uniquePayloadHashPairs.map(async ([payload, _hash]) => {
-            // Check if the root hash already exists
-            const existingRootHash = await store.index(IndexedDbArchivist.hashIndexName).get(_hash)
-            // If it does not already exist
-            if (!existingRootHash) {
-              // Insert the payload
-              await store.put({ ...payload, _hash })
+          payloadWithStorageMeta.map(async (payload) => {
+            // only insert if hash does not already exist
+            if (!await store.index(IndexedDbArchivist.hashIndexName).get(payload._hash)) {
+            // Insert the payload
+              await store.put(payload)
               // Add it to the inserted list
               inserted.push(payload)
             }
@@ -316,12 +325,12 @@ export class IndexedDbArchivist<
     })
   }
 
-  protected override async nextHandler(options?: ArchivistNextOptions): Promise<StoredPayload[]> {
+  protected override async nextHandler(options?: ArchivistNextOptions): Promise<WithStorageMeta<Payload>[]> {
     const {
-      limit, offset, order,
+      limit, cursor, order,
     } = options ?? {}
     return await this.useDb(async (db) => {
-      return await this.getFromOffset(db, this.storeName, order, limit ?? 10, offset)
+      return await this.getFromCursor(db, this.storeName, order, limit ?? 10, cursor)
     })
   }