@xyo-network/diviner-payload-indexeddb 3.6.0-rc.4 → 3.6.0-rc.6

package/dist/browser/index.mjs

@@ -17,7 +17,7 @@ import { IndexedDbArchivist } from "@xyo-network/archivist-indexeddb";
 import { IndexSeparator } from "@xyo-network/archivist-model";
 import { PayloadDiviner } from "@xyo-network/diviner-payload-abstract";
 import { isPayloadDivinerQueryPayload } from "@xyo-network/diviner-payload-model";
-import { StorageMetaConstants } from "@xyo-network/payload-model";
+import { SequenceConstants } from "@xyo-network/payload-model";
 import { openDB } from "idb";
 var payloadValueFilter = /* @__PURE__ */ __name((key, value) => {
   if (!value) return void 0;
@@ -75,7 +75,7 @@ var IndexedDbPayloadDiviner = class extends PayloadDiviner {
       const tx = db.transaction(this.storeName, "readonly");
       const store = tx.objectStore(this.storeName);
       const results = [];
-      let parsedCursor = cursor ?? StorageMetaConstants.minLocalSequence;
+      let parsedCursor = cursor ?? SequenceConstants.minLocalSequence;
       const parsedLimit = limit ?? 10;
       assertEx((schemas?.length ?? 1) === 1, () => "IndexedDbPayloadDiviner: Only one filter schema supported");
       const filterSchema = schemas?.[0];
@@ -92,7 +92,7 @@ var IndexedDbPayloadDiviner = class extends PayloadDiviner {
       let dbCursor = suggestedIndex ? await store.index(suggestedIndex).openCursor(IDBKeyRange.only(keyRangeValue), direction) : await store.openCursor(suggestedIndex, direction);
       while (dbCursor && parsedCursor) {
         dbCursor = await dbCursor.advance(1);
-        parsedCursor = StorageMetaConstants.minLocalSequence;
+        parsedCursor = SequenceConstants.minLocalSequence;
       }
       while (dbCursor && results.length < parsedLimit) {
         const value = dbCursor.value;

package/dist/browser/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Schema.ts","../../src/Config.ts","../../src/Diviner.ts"],"sourcesContent":["import { PayloadDivinerSchema } from '@xyo-network/diviner-payload-model'\n\nexport const IndexedDbPayloadDivinerSchema = `${PayloadDivinerSchema}.indexeddb` as const\nexport type IndexedDbPayloadDivinerSchema = typeof IndexedDbPayloadDivinerSchema\n","import type { IndexDescription } from '@xyo-network/archivist-model'\nimport type { DivinerConfig } from '@xyo-network/diviner-model'\n\nimport { IndexedDbPayloadDivinerSchema } from './Schema.ts'\n\nexport const IndexedDbPayloadDivinerConfigSchema = `${IndexedDbPayloadDivinerSchema}.config` as const\nexport type IndexedDbPayloadDivinerConfigSchema = typeof IndexedDbPayloadDivinerConfigSchema\n\nexport type IndexedDbPayloadDivinerConfig = DivinerConfig<{\n  /**\n   * The database name\n   */\n  dbName?: string\n  /**\n   * The version of the DB, defaults to 1\n   */\n  dbVersion?: number\n  schema: IndexedDbPayloadDivinerConfigSchema\n  /**\n   * The storage configuration\n   * // TODO: Hoist to main diviner config\n   */\n  storage?: {\n    /**\n     * The indexes to create on the object store\n     */\n    indexes?: IndexDescription[]\n  }\n  /**\n   * The name of the object store\n   */\n  storeName?: string\n}>\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { AnyObject } from '@xylabs/object'\nimport { removeFields } from '@xylabs/object'\nimport { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'\nimport { IndexSeparator } from '@xyo-network/archivist-model'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport {\n  type Payload, type Schema, StorageMetaConstants,\n} from '@xyo-network/payload-model'\nimport type { IDBPDatabase, IDBPObjectStore } from 'idb'\nimport { openDB } from 'idb'\n\nimport { IndexedDbPayloadDivinerConfigSchema } from './Config.ts'\nimport type { IndexedDbPayloadDivinerParams } from './Params.ts'\n\ninterface PayloadStore {\n  [s: string]: Payload\n}\n\ntype AnyPayload = Payload<Record<string, unknown>>\n\ntype ValueFilter = (payload?: AnyPayload | null) => boolean\n\nconst payloadValueFilter = (key: keyof AnyPayload, value?: unknown | unknown[]): ValueFilter | undefined => {\n  if (!value) return undefined\n  return (payload) => {\n    if (!payload) return false\n    const sourceValue = payload?.[key]\n    if (sourceValue === undefined) return false\n    return Array.isArray(sourceValue) && Array.isArray(value) ? containsAll(sourceValue, value) : sourceValue == value\n  }\n}\n\n// Function to extract fields from an index name\nconst extractFields = (indexName: string): string[] => {\n  return indexName\n    .slice(3)\n    .split(IndexSeparator)\n    .map(field => field.toLowerCase())\n}\n\nexport class IndexedDbPayloadDiviner<\n  TParams extends IndexedDbPayloadDivinerParams = IndexedDbPayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexedDbPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = IndexedDbPayloadDivinerConfigSchema\n\n  private _db: IDBPDatabase<PayloadStore> | undefined\n\n  /**\n   * The database name. If not supplied via config, it defaults\n   * to the archivist's name and if archivist's name is not supplied,\n   * it defaults to `archivist`. This behavior\n   * biases towards a single, isolated DB per archivist which seems to\n   * make the most sense for 99% of use cases.\n   */\n  get dbName() {\n    return this.config?.dbName ?? this.config?.archivist ?? IndexedDbArchivist.defaultDbName\n  }\n\n  /**\n   * The database version. If not supplied via config, it defaults to the archivist default version.\n   */\n  get dbVersion() {\n    return this.config?.dbVersion ?? IndexedDbArchivist.defaultDbVersion\n  }\n\n  /**\n   * The name of the object store. If not supplied via config, it defaults\n   * to `payloads`.\n   */\n  get storeName() {\n    return this.config?.storeName ?? IndexedDbArchivist.defaultStoreName\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {\n    const query = payloads?.find(isPayloadDivinerQueryPayload) as TIn\n    if (!query) return []\n    const result = await this.tryUseDb(async (db) => {\n      const {\n        schemas, limit, cursor, order, ...props\n      } = removeFields(query as unknown as TIn & { sources?: Hash[] }, [\n        'hash',\n        'schema',\n      ])\n      const tx = db.transaction(this.storeName, 'readonly')\n      const store = tx.objectStore(this.storeName)\n      const results: TOut[] = []\n      let parsedCursor = cursor ?? StorageMetaConstants.minLocalSequence\n      const parsedLimit = limit ?? 10\n      assertEx((schemas?.length ?? 1) === 1, () => 'IndexedDbPayloadDiviner: Only one filter schema supported')\n      const filterSchema = schemas?.[0]\n      const filter = filterSchema ? { schema: filterSchema, ...props } : { ...props }\n      const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next'\n      const suggestedIndex = this.selectBestIndex(filter, store)\n      const keyRangeValue = this.getKeyRangeValue(suggestedIndex, filter)\n      const valueFilters: ValueFilter[]\n        = props\n          ? Object.entries(props)\n              .map(([key, value]) => payloadValueFilter(key, value))\n              .filter(exists)\n          : []\n      let dbCursor\n        = suggestedIndex\n          // Conditionally filter on schemas\n          ? await store.index(suggestedIndex).openCursor(IDBKeyRange.only(keyRangeValue), direction)\n          // Just iterate all records\n          : await store.openCursor(suggestedIndex, direction)\n\n      // Skip records until the offset is reached\n      while (dbCursor && parsedCursor) {\n        dbCursor = await dbCursor.advance(1)\n        parsedCursor = StorageMetaConstants.minLocalSequence // Reset offset after skipping\n      }\n      // Collect results up to the limit\n      while (dbCursor && results.length < parsedLimit) {\n        const value = dbCursor.value\n        if (value) {\n          // If we're filtering on more than just the schema\n          if (valueFilters.length > 0) {\n            // Ensure all filters pass\n            if (valueFilters.every(filter => filter(value))) {\n              // Then save the value\n              results.push(value)\n            }\n          } else {\n            // Otherwise just save the value\n            results.push(value)\n          }\n        }\n        try {\n          dbCursor = await dbCursor.continue()\n        } catch {\n          break\n        }\n      }\n      await tx.done\n      // Remove any metadata before returning to the client\n      return results\n    })\n    return result ?? []\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    return true\n  }\n\n  private getKeyRangeValue(indexName: string | null, query: AnyObject): unknown | unknown[] {\n    if (!indexName) return []\n\n    // Extracting the relevant fields from the index name\n    const indexFields = extractFields(indexName)\n\n    // Collecting the values for these fields from the query object\n    const keyRangeValue = indexFields.map(field => query[field as keyof AnyObject])\n    return keyRangeValue.length === 1 ? keyRangeValue[0] : keyRangeValue\n  }\n\n  private selectBestIndex(query: AnyObject, store: IDBPObjectStore<PayloadStore>): string | null {\n    // List of available indexes\n    const { indexNames } = store\n\n    // Convert query object keys to a set for easier comparison\n    const queryKeys = new Set(Object.keys(query).map(key => key.toLowerCase()))\n\n    // Find the best matching index\n    let bestMatch: { indexName: string; matchCount: number } = { indexName: '', matchCount: 0 }\n\n    for (const indexName of indexNames) {\n      const indexFields = extractFields(indexName)\n      const matchCount = indexFields.filter(field => queryKeys.has(field)).length\n      if (matchCount > bestMatch.matchCount) {\n        bestMatch = { indexName, matchCount }\n      }\n    }\n    return bestMatch.matchCount > 0 ? bestMatch.indexName : null\n  }\n\n  /**\n   * Checks that the desired DB/objectStore exists and is initialized to the correct version\n   * @returns The initialized DB or undefined if it does not exist in the desired state\n   */\n  private async tryGetInitializedDb(): Promise<IDBPDatabase<PayloadStore> | undefined> {\n    // Enumerate the DBs\n    const dbs = await indexedDB.databases()\n    // Check that the DB exists at the desired version\n    const dbExists = dbs.some((db) => {\n      return db.name === this.dbName && db.version === this.dbVersion\n    })\n    // If the DB exists at the desired version\n    if (dbExists) {\n      // If the db does exist, open it\n      const db = await openDB<PayloadStore>(this.dbName, this.dbVersion)\n      // Check that the desired objectStore exists\n      const storeExists = db.objectStoreNames.contains(this.storeName)\n      // If the correct db/version/objectStore exists\n      if (storeExists) {\n        return db\n      } else {\n        // Otherwise close the db so the process that is going to update the\n        // db can open it\n        db.close()\n      }\n    }\n  }\n\n  /**\n   * Executes a callback with the initialized DB and then closes the db\n   * @param callback The method to execute with the initialized DB\n   * @returns\n   */\n  private async tryUseDb<T>(callback: (db: IDBPDatabase<PayloadStore>) => Promise<T> | T): Promise<T | undefined> {\n    // Get the initialized DB\n    const db = await this.tryGetInitializedDb()\n    if (db) {\n      try {\n        // Perform the callback\n        return await callback(db)\n      } finally {\n        // Close the DB\n        db.close()\n      }\n    }\n    return undefined\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,4BAA4B;AAE9B,IAAMC,gCAAgC,GAAGD,oBAAAA;;;ACGzC,IAAME,sCAAsC,GAAGC,6BAAAA;;;ACLtD,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,oBAAoB;AAC7B,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAE/B,SAASC,sBAAsB;AAE/B,SAASC,oCAAoC;AAC7C,SAC6BC,4BACtB;AAEP,SAASC,cAAc;AAavB,IAAMC,qBAAqB,wBAACC,KAAuBC,UAAAA;AACjD,MAAI,CAACA,MAAO,QAAOC;AACnB,SAAO,CAACC,YAAAA;AACN,QAAI,CAACA,QAAS,QAAO;AACrB,UAAMC,cAAcD,UAAUH,GAAAA;AAC9B,QAAII,gBAAgBF,OAAW,QAAO;AACtC,WAAOG,MAAMC,QAAQF,WAAAA,KAAgBC,MAAMC,QAAQL,KAAAA,IAASM,YAAYH,aAAaH,KAAAA,IAASG,eAAeH;EAC/G;AACF,GAR2B;AAW3B,IAAMO,gBAAgB,wBAACC,cAAAA;AACrB,SAAOA,UACJC,MAAM,CAAA,EACNC,MAAMC,cAAAA,EACNC,IAAIC,CAAAA,UAASA,MAAMC,YAAW,CAAA;AACnC,GALsB;AAOf,IAAMC,0BAAN,cASGC,eAAAA;EAxDV,OAwDUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAE/CE;;;;;;;;EASR,IAAIC,SAAS;AACX,WAAO,KAAKC,QAAQD,UAAU,KAAKC,QAAQC,aAAaC,mBAAmBC;EAC7E;;;;EAKA,IAAIC,YAAY;AACd,WAAO,KAAKJ,QAAQI,aAAaF,mBAAmBG;EACtD;;;;;EAMA,IAAIC,YAAY;AACd,WAAO,KAAKN,QAAQM,aAAaJ,mBAAmBK;EACtD;EAEA,MAAyBC,cAAcC,UAAmC;AACxE,UAAMC,QAAQD,UAAUE,KAAKC,4BAAAA;AAC7B,QAAI,CAACF,MAAO,QAAO,CAAA;AACnB,UAAMG,SAAS,MAAM,KAAKC,SAAS,OAAOC,OAAAA;AACxC,YAAM,EACJC,SAASC,OAAOC,QAAQC,OAAO,GAAGC,MAAAA,IAChCC,aAAaX,OAAgD;QAC/D;QACA;OACD;AACD,YAAMY,KAAKP,GAAGQ,YAAY,KAAKjB,WAAW,UAAA;AAC1C,YAAMkB,QAAQF,GAAGG,YAAY,KAAKnB,SAAS;AAC3C,YAAMoB,UAAkB,CAAA;AACxB,UAAIC,eAAeT,UAAUU,qBAAqBC;AAClD,YAAMC,cAAcb,SAAS;AAC7Bc,gBAAUf,SAASgB,UAAU,OAAO,GAAG,MAAM,2DAAA;AAC7C,YAAMC,eAAejB,UAAU,CAAA;AAC/B,YAAMkB,SAASD,eAAe;QAAEE,QAAQF;QAAc,GAAGb;MAAM,IAAI;QAAE,GAAGA;MAAM;AAC9E,YAAMgB,YAAgCjB,UAAU,SAAS,SAAS;AAClE,YAAMkB,iBAAiB,KAAKC,gBAAgBJ,QAAQV,KAAAA;AACpD,YAAMe,gBAAgB,KAAKC,iBAAiBH,gBAAgBH,MAAAA;AAC5D,YAAMO,eACFrB,QACEsB,OAAOC,QAAQvB,KAAAA,EACZ9B,IAAI,CAAC,CAACb,KAAKC,KAAAA,MAAWF,mBAAmBC,KAAKC,KAAAA,CAAAA,EAC9CwD,OAAOU,MAAAA,IACV,CAAA;AACN,UAAIC,WACAR,iBAEE,MAAMb,MAAMsB,MAAMT,cAAAA,EAAgBU,WAAWC,YAAYC,KAAKV,aAAAA,GAAgBH,SAAAA,IAE9E,MAAMZ,MAAMuB,WAAWV,gBAAgBD,SAAAA;AAG7C,aAAOS,YAAYlB,cAAc;AAC/BkB,mBAAW,MAAMA,SAASK,QAAQ,CAAA;AAClCvB,uBAAeC,qBAAqBC;MACtC;AAEA,aAAOgB,YAAYnB,QAAQM,SAASF,aAAa;AAC/C,cAAMpD,QAAQmE,SAASnE;AACvB,YAAIA,OAAO;AAET,cAAI+D,aAAaT,SAAS,GAAG;AAE3B,gBAAIS,aAAaU,MAAMjB,CAAAA,YAAUA,QAAOxD,KAAAA,CAAAA,GAAS;AAE/CgD,sBAAQ0B,KAAK1E,KAAAA;YACf;UACF,OAAO;AAELgD,oBAAQ0B,KAAK1E,KAAAA;UACf;QACF;AACA,YAAI;AACFmE,qBAAW,MAAMA,SAASQ,SAAQ;QACpC,QAAQ;AACN;QACF;MACF;AACA,YAAM/B,GAAGgC;AAET,aAAO5B;IACT,CAAA;AACA,WAAOb,UAAU,CAAA;EACnB;EAEA,MAAyB0C,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;EAEQf,iBAAiBtD,WAA0BwB,OAAuC;AACxF,QAAI,CAACxB,UAAW,QAAO,CAAA;AAGvB,UAAMsE,cAAcvE,cAAcC,SAAAA;AAGlC,UAAMqD,gBAAgBiB,YAAYlE,IAAIC,CAAAA,UAASmB,MAAMnB,KAAAA,CAAyB;AAC9E,WAAOgD,cAAcP,WAAW,IAAIO,cAAc,CAAA,IAAKA;EACzD;EAEQD,gBAAgB5B,OAAkBc,OAAqD;AAE7F,UAAM,EAAEiC,WAAU,IAAKjC;AAGvB,UAAMkC,YAAY,IAAIC,IAAIjB,OAAOkB,KAAKlD,KAAAA,EAAOpB,IAAIb,CAAAA,QAAOA,IAAIe,YAAW,CAAA,CAAA;AAGvE,QAAIqE,YAAuD;MAAE3E,WAAW;MAAI4E,YAAY;IAAE;AAE1F,eAAW5E,aAAauE,YAAY;AAClC,YAAMD,cAAcvE,cAAcC,SAAAA;AAClC,YAAM4E,aAAaN,YAAYtB,OAAO3C,CAAAA,UAASmE,UAAUK,IAAIxE,KAAAA,CAAAA,EAAQyC;AACrE,UAAI8B,aAAaD,UAAUC,YAAY;AACrCD,oBAAY;UAAE3E;UAAW4E;QAAW;MACtC;IACF;AACA,WAAOD,UAAUC,aAAa,IAAID,UAAU3E,YAAY;EAC1D;;;;;EAMA,MAAc8E,sBAAuE;AAEnF,UAAMC,MAAM,MAAMC,UAAUC,UAAS;AAErC,UAAMC,WAAWH,IAAII,KAAK,CAACtD,OAAAA;AACzB,aAAOA,GAAGuD,SAAS,KAAKvE,UAAUgB,GAAGwD,YAAY,KAAKnE;IACxD,CAAA;AAEA,QAAIgE,UAAU;AAEZ,YAAMrD,KAAK,MAAMyD,OAAqB,KAAKzE,QAAQ,KAAKK,SAAS;AAEjE,YAAMqE,cAAc1D,GAAG2D,iBAAiBC,SAAS,KAAKrE,SAAS;AAE/D,UAAImE,aAAa;AACf,eAAO1D;MACT,OAAO;AAGLA,WAAG6D,MAAK;MACV;IACF;EACF;;;;;;EAOA,MAAc9D,SAAY+D,UAAsF;AAE9G,UAAM9D,KAAK,MAAM,KAAKiD,oBAAmB;AACzC,QAAIjD,IAAI;AACN,UAAI;AAEF,eAAO,MAAM8D,SAAS9D,EAAAA;MACxB,UAAA;AAEEA,WAAG6D,MAAK;MACV;IACF;AACA,WAAOjG;EACT;AACF;","names":["PayloadDivinerSchema","IndexedDbPayloadDivinerSchema","IndexedDbPayloadDivinerConfigSchema","IndexedDbPayloadDivinerSchema","containsAll","assertEx","exists","removeFields","IndexedDbArchivist","IndexSeparator","PayloadDiviner","isPayloadDivinerQueryPayload","StorageMetaConstants","openDB","payloadValueFilter","key","value","undefined","payload","sourceValue","Array","isArray","containsAll","extractFields","indexName","slice","split","IndexSeparator","map","field","toLowerCase","IndexedDbPayloadDiviner","PayloadDiviner","configSchemas","IndexedDbPayloadDivinerConfigSchema","defaultConfigSchema","_db","dbName","config","archivist","IndexedDbArchivist","defaultDbName","dbVersion","defaultDbVersion","storeName","defaultStoreName","divineHandler","payloads","query","find","isPayloadDivinerQueryPayload","result","tryUseDb","db","schemas","limit","cursor","order","props","removeFields","tx","transaction","store","objectStore","results","parsedCursor","StorageMetaConstants","minLocalSequence","parsedLimit","assertEx","length","filterSchema","filter","schema","direction","suggestedIndex","selectBestIndex","keyRangeValue","getKeyRangeValue","valueFilters","Object","entries","exists","dbCursor","index","openCursor","IDBKeyRange","only","advance","every","push","continue","done","startHandler","indexFields","indexNames","queryKeys","Set","keys","bestMatch","matchCount","has","tryGetInitializedDb","dbs","indexedDB","databases","dbExists","some","name","version","openDB","storeExists","objectStoreNames","contains","close","callback"]}
+{"version":3,"sources":["../../src/Schema.ts","../../src/Config.ts","../../src/Diviner.ts"],"sourcesContent":["import { PayloadDivinerSchema } from '@xyo-network/diviner-payload-model'\n\nexport const IndexedDbPayloadDivinerSchema = `${PayloadDivinerSchema}.indexeddb` as const\nexport type IndexedDbPayloadDivinerSchema = typeof IndexedDbPayloadDivinerSchema\n","import type { IndexDescription } from '@xyo-network/archivist-model'\nimport type { DivinerConfig } from '@xyo-network/diviner-model'\n\nimport { IndexedDbPayloadDivinerSchema } from './Schema.ts'\n\nexport const IndexedDbPayloadDivinerConfigSchema = `${IndexedDbPayloadDivinerSchema}.config` as const\nexport type IndexedDbPayloadDivinerConfigSchema = typeof IndexedDbPayloadDivinerConfigSchema\n\nexport type IndexedDbPayloadDivinerConfig = DivinerConfig<{\n  /**\n   * The database name\n   */\n  dbName?: string\n  /**\n   * The version of the DB, defaults to 1\n   */\n  dbVersion?: number\n  schema: IndexedDbPayloadDivinerConfigSchema\n  /**\n   * The storage configuration\n   * // TODO: Hoist to main diviner config\n   */\n  storage?: {\n    /**\n     * The indexes to create on the object store\n     */\n    indexes?: IndexDescription[]\n  }\n  /**\n   * The name of the object store\n   */\n  storeName?: string\n}>\n","import { containsAll } from '@xylabs/array'\nimport { assertEx } from '@xylabs/assert'\nimport { exists } from '@xylabs/exists'\nimport type { Hash } from '@xylabs/hex'\nimport type { AnyObject } from '@xylabs/object'\nimport { removeFields } from '@xylabs/object'\nimport { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'\nimport { IndexSeparator } from '@xyo-network/archivist-model'\nimport type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'\nimport {\n  type Payload, type Schema, SequenceConstants,\n} from '@xyo-network/payload-model'\nimport type { IDBPDatabase, IDBPObjectStore } from 'idb'\nimport { openDB } from 'idb'\n\nimport { IndexedDbPayloadDivinerConfigSchema } from './Config.ts'\nimport type { IndexedDbPayloadDivinerParams } from './Params.ts'\n\ninterface PayloadStore {\n  [s: string]: Payload\n}\n\ntype AnyPayload = Payload<Record<string, unknown>>\n\ntype ValueFilter = (payload?: AnyPayload | null) => boolean\n\nconst payloadValueFilter = (key: keyof AnyPayload, value?: unknown | unknown[]): ValueFilter | undefined => {\n  if (!value) return undefined\n  return (payload) => {\n    if (!payload) return false\n    const sourceValue = payload?.[key]\n    if (sourceValue === undefined) return false\n    return Array.isArray(sourceValue) && Array.isArray(value) ? containsAll(sourceValue, value) : sourceValue == value\n  }\n}\n\n// Function to extract fields from an index name\nconst extractFields = (indexName: string): string[] => {\n  return indexName\n    .slice(3)\n    .split(IndexSeparator)\n    .map(field => field.toLowerCase())\n}\n\nexport class IndexedDbPayloadDiviner<\n  TParams extends IndexedDbPayloadDivinerParams = IndexedDbPayloadDivinerParams,\n  TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexedDbPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = IndexedDbPayloadDivinerConfigSchema\n\n  private _db: IDBPDatabase<PayloadStore> | undefined\n\n  /**\n   * The database name. If not supplied via config, it defaults\n   * to the archivist's name and if archivist's name is not supplied,\n   * it defaults to `archivist`. This behavior\n   * biases towards a single, isolated DB per archivist which seems to\n   * make the most sense for 99% of use cases.\n   */\n  get dbName() {\n    return this.config?.dbName ?? this.config?.archivist ?? IndexedDbArchivist.defaultDbName\n  }\n\n  /**\n   * The database version. If not supplied via config, it defaults to the archivist default version.\n   */\n  get dbVersion() {\n    return this.config?.dbVersion ?? IndexedDbArchivist.defaultDbVersion\n  }\n\n  /**\n   * The name of the object store. If not supplied via config, it defaults\n   * to `payloads`.\n   */\n  get storeName() {\n    return this.config?.storeName ?? IndexedDbArchivist.defaultStoreName\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {\n    const query = payloads?.find(isPayloadDivinerQueryPayload) as TIn\n    if (!query) return []\n    const result = await this.tryUseDb(async (db) => {\n      const {\n        schemas, limit, cursor, order, ...props\n      } = removeFields(query as unknown as TIn & { sources?: Hash[] }, [\n        'hash',\n        'schema',\n      ])\n      const tx = db.transaction(this.storeName, 'readonly')\n      const store = tx.objectStore(this.storeName)\n      const results: TOut[] = []\n      let parsedCursor = cursor ?? SequenceConstants.minLocalSequence\n      const parsedLimit = limit ?? 10\n      assertEx((schemas?.length ?? 1) === 1, () => 'IndexedDbPayloadDiviner: Only one filter schema supported')\n      const filterSchema = schemas?.[0]\n      const filter = filterSchema ? { schema: filterSchema, ...props } : { ...props }\n      const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next'\n      const suggestedIndex = this.selectBestIndex(filter, store)\n      const keyRangeValue = this.getKeyRangeValue(suggestedIndex, filter)\n      const valueFilters: ValueFilter[]\n        = props\n          ? Object.entries(props)\n              .map(([key, value]) => payloadValueFilter(key, value))\n              .filter(exists)\n          : []\n      let dbCursor\n        = suggestedIndex\n          // Conditionally filter on schemas\n          ? await store.index(suggestedIndex).openCursor(IDBKeyRange.only(keyRangeValue), direction)\n          // Just iterate all records\n          : await store.openCursor(suggestedIndex, direction)\n\n      // Skip records until the offset is reached\n      while (dbCursor && parsedCursor) {\n        dbCursor = await dbCursor.advance(1)\n        parsedCursor = SequenceConstants.minLocalSequence // Reset offset after skipping\n      }\n      // Collect results up to the limit\n      while (dbCursor && results.length < parsedLimit) {\n        const value = dbCursor.value\n        if (value) {\n          // If we're filtering on more than just the schema\n          if (valueFilters.length > 0) {\n            // Ensure all filters pass\n            if (valueFilters.every(filter => filter(value))) {\n              // Then save the value\n              results.push(value)\n            }\n          } else {\n            // Otherwise just save the value\n            results.push(value)\n          }\n        }\n        try {\n          dbCursor = await dbCursor.continue()\n        } catch {\n          break\n        }\n      }\n      await tx.done\n      // Remove any metadata before returning to the client\n      return results\n    })\n    return result ?? []\n  }\n\n  protected override async startHandler() {\n    await super.startHandler()\n    return true\n  }\n\n  private getKeyRangeValue(indexName: string | null, query: AnyObject): unknown | unknown[] {\n    if (!indexName) return []\n\n    // Extracting the relevant fields from the index name\n    const indexFields = extractFields(indexName)\n\n    // Collecting the values for these fields from the query object\n    const keyRangeValue = indexFields.map(field => query[field as keyof AnyObject])\n    return keyRangeValue.length === 1 ? keyRangeValue[0] : keyRangeValue\n  }\n\n  private selectBestIndex(query: AnyObject, store: IDBPObjectStore<PayloadStore>): string | null {\n    // List of available indexes\n    const { indexNames } = store\n\n    // Convert query object keys to a set for easier comparison\n    const queryKeys = new Set(Object.keys(query).map(key => key.toLowerCase()))\n\n    // Find the best matching index\n    let bestMatch: { indexName: string; matchCount: number } = { indexName: '', matchCount: 0 }\n\n    for (const indexName of indexNames) {\n      const indexFields = extractFields(indexName)\n      const matchCount = indexFields.filter(field => queryKeys.has(field)).length\n      if (matchCount > bestMatch.matchCount) {\n        bestMatch = { indexName, matchCount }\n      }\n    }\n    return bestMatch.matchCount > 0 ? bestMatch.indexName : null\n  }\n\n  /**\n   * Checks that the desired DB/objectStore exists and is initialized to the correct version\n   * @returns The initialized DB or undefined if it does not exist in the desired state\n   */\n  private async tryGetInitializedDb(): Promise<IDBPDatabase<PayloadStore> | undefined> {\n    // Enumerate the DBs\n    const dbs = await indexedDB.databases()\n    // Check that the DB exists at the desired version\n    const dbExists = dbs.some((db) => {\n      return db.name === this.dbName && db.version === this.dbVersion\n    })\n    // If the DB exists at the desired version\n    if (dbExists) {\n      // If the db does exist, open it\n      const db = await openDB<PayloadStore>(this.dbName, this.dbVersion)\n      // Check that the desired objectStore exists\n      const storeExists = db.objectStoreNames.contains(this.storeName)\n      // If the correct db/version/objectStore exists\n      if (storeExists) {\n        return db\n      } else {\n        // Otherwise close the db so the process that is going to update the\n        // db can open it\n        db.close()\n      }\n    }\n  }\n\n  /**\n   * Executes a callback with the initialized DB and then closes the db\n   * @param callback The method to execute with the initialized DB\n   * @returns\n   */\n  private async tryUseDb<T>(callback: (db: IDBPDatabase<PayloadStore>) => Promise<T> | T): Promise<T | undefined> {\n    // Get the initialized DB\n    const db = await this.tryGetInitializedDb()\n    if (db) {\n      try {\n        // Perform the callback\n        return await callback(db)\n      } finally {\n        // Close the DB\n        db.close()\n      }\n    }\n    return undefined\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,4BAA4B;AAE9B,IAAMC,gCAAgC,GAAGD,oBAAAA;;;ACGzC,IAAME,sCAAsC,GAAGC,6BAAAA;;;ACLtD,SAASC,mBAAmB;AAC5B,SAASC,gBAAgB;AACzB,SAASC,cAAc;AAGvB,SAASC,oBAAoB;AAC7B,SAASC,0BAA0B;AACnC,SAASC,sBAAsB;AAE/B,SAASC,sBAAsB;AAE/B,SAASC,oCAAoC;AAC7C,SAC6BC,yBACtB;AAEP,SAASC,cAAc;AAavB,IAAMC,qBAAqB,wBAACC,KAAuBC,UAAAA;AACjD,MAAI,CAACA,MAAO,QAAOC;AACnB,SAAO,CAACC,YAAAA;AACN,QAAI,CAACA,QAAS,QAAO;AACrB,UAAMC,cAAcD,UAAUH,GAAAA;AAC9B,QAAII,gBAAgBF,OAAW,QAAO;AACtC,WAAOG,MAAMC,QAAQF,WAAAA,KAAgBC,MAAMC,QAAQL,KAAAA,IAASM,YAAYH,aAAaH,KAAAA,IAASG,eAAeH;EAC/G;AACF,GAR2B;AAW3B,IAAMO,gBAAgB,wBAACC,cAAAA;AACrB,SAAOA,UACJC,MAAM,CAAA,EACNC,MAAMC,cAAAA,EACNC,IAAIC,CAAAA,UAASA,MAAMC,YAAW,CAAA;AACnC,GALsB;AAOf,IAAMC,0BAAN,cASGC,eAAAA;EAxDV,OAwDUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeC;;EAC5E,OAAyBC,sBAA8BD;EAE/CE;;;;;;;;EASR,IAAIC,SAAS;AACX,WAAO,KAAKC,QAAQD,UAAU,KAAKC,QAAQC,aAAaC,mBAAmBC;EAC7E;;;;EAKA,IAAIC,YAAY;AACd,WAAO,KAAKJ,QAAQI,aAAaF,mBAAmBG;EACtD;;;;;EAMA,IAAIC,YAAY;AACd,WAAO,KAAKN,QAAQM,aAAaJ,mBAAmBK;EACtD;EAEA,MAAyBC,cAAcC,UAAmC;AACxE,UAAMC,QAAQD,UAAUE,KAAKC,4BAAAA;AAC7B,QAAI,CAACF,MAAO,QAAO,CAAA;AACnB,UAAMG,SAAS,MAAM,KAAKC,SAAS,OAAOC,OAAAA;AACxC,YAAM,EACJC,SAASC,OAAOC,QAAQC,OAAO,GAAGC,MAAAA,IAChCC,aAAaX,OAAgD;QAC/D;QACA;OACD;AACD,YAAMY,KAAKP,GAAGQ,YAAY,KAAKjB,WAAW,UAAA;AAC1C,YAAMkB,QAAQF,GAAGG,YAAY,KAAKnB,SAAS;AAC3C,YAAMoB,UAAkB,CAAA;AACxB,UAAIC,eAAeT,UAAUU,kBAAkBC;AAC/C,YAAMC,cAAcb,SAAS;AAC7Bc,gBAAUf,SAASgB,UAAU,OAAO,GAAG,MAAM,2DAAA;AAC7C,YAAMC,eAAejB,UAAU,CAAA;AAC/B,YAAMkB,SAASD,eAAe;QAAEE,QAAQF;QAAc,GAAGb;MAAM,IAAI;QAAE,GAAGA;MAAM;AAC9E,YAAMgB,YAAgCjB,UAAU,SAAS,SAAS;AAClE,YAAMkB,iBAAiB,KAAKC,gBAAgBJ,QAAQV,KAAAA;AACpD,YAAMe,gBAAgB,KAAKC,iBAAiBH,gBAAgBH,MAAAA;AAC5D,YAAMO,eACFrB,QACEsB,OAAOC,QAAQvB,KAAAA,EACZ9B,IAAI,CAAC,CAACb,KAAKC,KAAAA,MAAWF,mBAAmBC,KAAKC,KAAAA,CAAAA,EAC9CwD,OAAOU,MAAAA,IACV,CAAA;AACN,UAAIC,WACAR,iBAEE,MAAMb,MAAMsB,MAAMT,cAAAA,EAAgBU,WAAWC,YAAYC,KAAKV,aAAAA,GAAgBH,SAAAA,IAE9E,MAAMZ,MAAMuB,WAAWV,gBAAgBD,SAAAA;AAG7C,aAAOS,YAAYlB,cAAc;AAC/BkB,mBAAW,MAAMA,SAASK,QAAQ,CAAA;AAClCvB,uBAAeC,kBAAkBC;MACnC;AAEA,aAAOgB,YAAYnB,QAAQM,SAASF,aAAa;AAC/C,cAAMpD,QAAQmE,SAASnE;AACvB,YAAIA,OAAO;AAET,cAAI+D,aAAaT,SAAS,GAAG;AAE3B,gBAAIS,aAAaU,MAAMjB,CAAAA,YAAUA,QAAOxD,KAAAA,CAAAA,GAAS;AAE/CgD,sBAAQ0B,KAAK1E,KAAAA;YACf;UACF,OAAO;AAELgD,oBAAQ0B,KAAK1E,KAAAA;UACf;QACF;AACA,YAAI;AACFmE,qBAAW,MAAMA,SAASQ,SAAQ;QACpC,QAAQ;AACN;QACF;MACF;AACA,YAAM/B,GAAGgC;AAET,aAAO5B;IACT,CAAA;AACA,WAAOb,UAAU,CAAA;EACnB;EAEA,MAAyB0C,eAAe;AACtC,UAAM,MAAMA,aAAAA;AACZ,WAAO;EACT;EAEQf,iBAAiBtD,WAA0BwB,OAAuC;AACxF,QAAI,CAACxB,UAAW,QAAO,CAAA;AAGvB,UAAMsE,cAAcvE,cAAcC,SAAAA;AAGlC,UAAMqD,gBAAgBiB,YAAYlE,IAAIC,CAAAA,UAASmB,MAAMnB,KAAAA,CAAyB;AAC9E,WAAOgD,cAAcP,WAAW,IAAIO,cAAc,CAAA,IAAKA;EACzD;EAEQD,gBAAgB5B,OAAkBc,OAAqD;AAE7F,UAAM,EAAEiC,WAAU,IAAKjC;AAGvB,UAAMkC,YAAY,IAAIC,IAAIjB,OAAOkB,KAAKlD,KAAAA,EAAOpB,IAAIb,CAAAA,QAAOA,IAAIe,YAAW,CAAA,CAAA;AAGvE,QAAIqE,YAAuD;MAAE3E,WAAW;MAAI4E,YAAY;IAAE;AAE1F,eAAW5E,aAAauE,YAAY;AAClC,YAAMD,cAAcvE,cAAcC,SAAAA;AAClC,YAAM4E,aAAaN,YAAYtB,OAAO3C,CAAAA,UAASmE,UAAUK,IAAIxE,KAAAA,CAAAA,EAAQyC;AACrE,UAAI8B,aAAaD,UAAUC,YAAY;AACrCD,oBAAY;UAAE3E;UAAW4E;QAAW;MACtC;IACF;AACA,WAAOD,UAAUC,aAAa,IAAID,UAAU3E,YAAY;EAC1D;;;;;EAMA,MAAc8E,sBAAuE;AAEnF,UAAMC,MAAM,MAAMC,UAAUC,UAAS;AAErC,UAAMC,WAAWH,IAAII,KAAK,CAACtD,OAAAA;AACzB,aAAOA,GAAGuD,SAAS,KAAKvE,UAAUgB,GAAGwD,YAAY,KAAKnE;IACxD,CAAA;AAEA,QAAIgE,UAAU;AAEZ,YAAMrD,KAAK,MAAMyD,OAAqB,KAAKzE,QAAQ,KAAKK,SAAS;AAEjE,YAAMqE,cAAc1D,GAAG2D,iBAAiBC,SAAS,KAAKrE,SAAS;AAE/D,UAAImE,aAAa;AACf,eAAO1D;MACT,OAAO;AAGLA,WAAG6D,MAAK;MACV;IACF;EACF;;;;;;EAOA,MAAc9D,SAAY+D,UAAsF;AAE9G,UAAM9D,KAAK,MAAM,KAAKiD,oBAAmB;AACzC,QAAIjD,IAAI;AACN,UAAI;AAEF,eAAO,MAAM8D,SAAS9D,EAAAA;MACxB,UAAA;AAEEA,WAAG6D,MAAK;MACV;IACF;AACA,WAAOjG;EACT;AACF;","names":["PayloadDivinerSchema","IndexedDbPayloadDivinerSchema","IndexedDbPayloadDivinerConfigSchema","IndexedDbPayloadDivinerSchema","containsAll","assertEx","exists","removeFields","IndexedDbArchivist","IndexSeparator","PayloadDiviner","isPayloadDivinerQueryPayload","SequenceConstants","openDB","payloadValueFilter","key","value","undefined","payload","sourceValue","Array","isArray","containsAll","extractFields","indexName","slice","split","IndexSeparator","map","field","toLowerCase","IndexedDbPayloadDiviner","PayloadDiviner","configSchemas","IndexedDbPayloadDivinerConfigSchema","defaultConfigSchema","_db","dbName","config","archivist","IndexedDbArchivist","defaultDbName","dbVersion","defaultDbVersion","storeName","defaultStoreName","divineHandler","payloads","query","find","isPayloadDivinerQueryPayload","result","tryUseDb","db","schemas","limit","cursor","order","props","removeFields","tx","transaction","store","objectStore","results","parsedCursor","SequenceConstants","minLocalSequence","parsedLimit","assertEx","length","filterSchema","filter","schema","direction","suggestedIndex","selectBestIndex","keyRangeValue","getKeyRangeValue","valueFilters","Object","entries","exists","dbCursor","index","openCursor","IDBKeyRange","only","advance","every","push","continue","done","startHandler","indexFields","indexNames","queryKeys","Set","keys","bestMatch","matchCount","has","tryGetInitializedDb","dbs","indexedDB","databases","dbExists","some","name","version","openDB","storeExists","objectStoreNames","contains","close","callback"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/diviner-payload-indexeddb",
-  "version": "3.6.0-rc.4",
+  "version": "3.6.0-rc.6",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,27 +29,28 @@
   "module": "dist/browser/index.mjs",
   "types": "dist/browser/index.d.ts",
   "dependencies": {
-    "@xylabs/array": "^4.4.13",
-    "@xylabs/assert": "^4.4.13",
-    "@xylabs/exists": "^4.4.13",
-    "@xylabs/hex": "^4.4.13",
-    "@xylabs/object": "^4.4.13",
-    "@xyo-network/archivist-indexeddb": "^3.6.0-rc.4",
-    "@xyo-network/archivist-model": "^3.6.0-rc.4",
-    "@xyo-network/diviner-model": "^3.6.0-rc.4",
-    "@xyo-network/diviner-payload-abstract": "^3.6.0-rc.4",
-    "@xyo-network/diviner-payload-model": "^3.6.0-rc.4",
-    "@xyo-network/module-model": "^3.6.0-rc.4",
-    "@xyo-network/payload-model": "^3.6.0-rc.4",
-    "idb": "^8.0.0"
+    "@xylabs/array": "^4.4.15",
+    "@xylabs/assert": "^4.4.15",
+    "@xylabs/exists": "^4.4.15",
+    "@xylabs/hex": "^4.4.15",
+    "@xylabs/object": "^4.4.15",
+    "@xyo-network/archivist-indexeddb": "^3.6.0-rc.6",
+    "@xyo-network/archivist-model": "^3.6.0-rc.6",
+    "@xyo-network/diviner-model": "^3.6.0-rc.6",
+    "@xyo-network/diviner-payload-abstract": "^3.6.0-rc.6",
+    "@xyo-network/diviner-payload-model": "^3.6.0-rc.6",
+    "@xyo-network/module-model": "^3.6.0-rc.6",
+    "@xyo-network/payload-model": "^3.6.0-rc.6",
+    "idb": "^8.0.1"
   },
   "devDependencies": {
+    "@xylabs/delay": "^4.4.15",
     "@xylabs/ts-scripts-yarn3": "^4.2.4",
     "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.13",
-    "@xyo-network/archivist-indexeddb": "^3.6.0-rc.4",
-    "@xyo-network/node-memory": "^3.6.0-rc.4",
-    "@xyo-network/payload-builder": "^3.6.0-rc.4",
+    "@xylabs/vitest-extended": "^4.4.15",
+    "@xyo-network/archivist-indexeddb": "^3.6.0-rc.6",
+    "@xyo-network/node-memory": "^3.6.0-rc.6",
+    "@xyo-network/payload-builder": "^3.6.0-rc.6",
     "fake-indexeddb": "^6.0.0",
     "typescript": "^5.7.2",
     "vitest": "^2.1.8"

package/src/Diviner.ts

@@ -11,7 +11,7 @@ import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'
 import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
 import { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
 import {
-  type Payload, type Schema, StorageMetaConstants,
+  type Payload, type Schema, SequenceConstants,
 } from '@xyo-network/payload-model'
 import type { IDBPDatabase, IDBPObjectStore } from 'idb'
 import { openDB } from 'idb'
@@ -99,7 +99,7 @@ export class IndexedDbPayloadDiviner<
       const tx = db.transaction(this.storeName, 'readonly')
       const store = tx.objectStore(this.storeName)
       const results: TOut[] = []
-      let parsedCursor = cursor ?? StorageMetaConstants.minLocalSequence
+      let parsedCursor = cursor ?? SequenceConstants.minLocalSequence
       const parsedLimit = limit ?? 10
       assertEx((schemas?.length ?? 1) === 1, () => 'IndexedDbPayloadDiviner: Only one filter schema supported')
       const filterSchema = schemas?.[0]
@@ -123,7 +123,7 @@ export class IndexedDbPayloadDiviner<
       // Skip records until the offset is reached
       while (dbCursor && parsedCursor) {
         dbCursor = await dbCursor.advance(1)
-        parsedCursor = StorageMetaConstants.minLocalSequence // Reset offset after skipping
+        parsedCursor = SequenceConstants.minLocalSequence // Reset offset after skipping
       }
       // Collect results up to the limit
       while (dbCursor && results.length < parsedLimit) {