@xyo-network/diviner-payload-generic 2.103.9 → 2.104.1

package/dist/browser/index.cjs

@@ -105,8 +105,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
     const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
     let all = this.all(order, offset);
     if (all) {
-      if (schemas?.length)
-        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema));
       if (Object.keys(props).length > 0) {
         const additionalFilterCriteria = Object.entries(props);
         for (const [prop, filter2] of additionalFilterCriteria) {
@@ -122,22 +121,22 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
       throw new Error('Archivist does not support "all"');
     }
   }
-  onArchivistCleared = () => {
+  onArchivistCleared = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
+  }, "onArchivistCleared");
   // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
-  onArchivistDeleted = () => {
+  onArchivistDeleted = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
-  onArchivistInserted = () => {
+  }, "onArchivistDeleted");
+  onArchivistInserted = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)(this.updateIndex());
-  };
+  }, "onArchivistInserted");
   async stopHandler(_timeout) {
     const archivist = await this.archivistInstance(true);
     archivist.off("inserted", this.onArchivistInserted);
@@ -174,8 +173,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
     for (const index of this.indexes ?? []) {
       this.indexMaps[index] = this.indexMaps[index] ?? [];
       for (const [payload] of pairs) {
-        if (payload[index] !== void 0)
-          this.indexMaps[index].push(payload);
+        if (payload[index] !== void 0) this.indexMaps[index].push(payload);
       }
     }
     return (0, import_assert.assertEx)(pairs.at(-1))[1];

package/dist/browser/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,+CAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC;AAAW,eAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,eAAOkB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,+CAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN,OAAQ9B,OAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,6BAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;;EAUzEZ,qBAAyE,6BAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;EASzEd,sBAA2E,6BAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB,GAFqF;EAIrF,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC,OAAW,MAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,eAAOkB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}

package/dist/browser/index.js

@@ -80,8 +80,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
     const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
     let all = this.all(order, offset);
     if (all) {
-      if (schemas?.length)
-        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema));
       if (Object.keys(props).length > 0) {
         const additionalFilterCriteria = Object.entries(props);
         for (const [prop, filter2] of additionalFilterCriteria) {
@@ -97,22 +96,22 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
       throw new Error('Archivist does not support "all"');
     }
   }
-  onArchivistCleared = () => {
+  onArchivistCleared = /* @__PURE__ */ __name(() => {
     forget((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
+  }, "onArchivistCleared");
   // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
-  onArchivistDeleted = () => {
+  onArchivistDeleted = /* @__PURE__ */ __name(() => {
     forget((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
-  onArchivistInserted = () => {
+  }, "onArchivistDeleted");
+  onArchivistInserted = /* @__PURE__ */ __name(() => {
     forget(this.updateIndex());
-  };
+  }, "onArchivistInserted");
   async stopHandler(_timeout) {
     const archivist = await this.archivistInstance(true);
     archivist.off("inserted", this.onArchivistInserted);
@@ -149,8 +148,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
     for (const index of this.indexes ?? []) {
       this.indexMaps[index] = this.indexMaps[index] ?? [];
       for (const [payload] of pairs) {
-        if (payload[index] !== void 0)
-          this.indexMaps[index].push(payload);
+        if (payload[index] !== void 0) this.indexMaps[index].push(payload);
       }
     }
     return assertEx(pairs.at(-1))[1];

package/dist/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,oCAKK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,eAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC;AAAW,eAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,WAAOkB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,oCAKK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,eAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN,OAAQ9B,OAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,6BAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;;EAUzEZ,qBAAyE,6BAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;EASzEd,sBAA2E,6BAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB,GAFqF;EAIrF,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC,OAAW,MAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,WAAOkB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}

package/dist/neutral/index.cjs

@@ -105,8 +105,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
     const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
     let all = this.all(order, offset);
     if (all) {
-      if (schemas?.length)
-        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema));
       if (Object.keys(props).length > 0) {
         const additionalFilterCriteria = Object.entries(props);
         for (const [prop, filter2] of additionalFilterCriteria) {
@@ -122,22 +121,22 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
       throw new Error('Archivist does not support "all"');
     }
   }
-  onArchivistCleared = () => {
+  onArchivistCleared = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
+  }, "onArchivistCleared");
   // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
-  onArchivistDeleted = () => {
+  onArchivistDeleted = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
-  onArchivistInserted = () => {
+  }, "onArchivistDeleted");
+  onArchivistInserted = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)(this.updateIndex());
-  };
+  }, "onArchivistInserted");
   async stopHandler(_timeout) {
     const archivist = await this.archivistInstance(true);
     archivist.off("inserted", this.onArchivistInserted);
@@ -174,8 +173,7 @@ var GenericPayloadDiviner = class extends import_diviner_payload_abstract.Payloa
     for (const index of this.indexes ?? []) {
       this.indexMaps[index] = this.indexMaps[index] ?? [];
       for (const [payload] of pairs) {
-        if (payload[index] !== void 0)
-          this.indexMaps[index].push(payload);
+        if (payload[index] !== void 0) this.indexMaps[index].push(payload);
       }
     }
     return (0, import_assert.assertEx)(pairs.at(-1))[1];

package/dist/neutral/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,+CAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC;AAAW,eAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,eAAOkB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,+CAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,yDAAAA,KAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN,OAAQ9B,OAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,6BAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;;EAUzEZ,qBAAyE,6BAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;EASzEd,sBAA2E,6BAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB,GAFqF;EAIrF,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC,OAAW,MAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,eAAOkB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}

package/dist/neutral/index.js

@@ -80,8 +80,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
     const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
     let all = this.all(order, offset);
     if (all) {
-      if (schemas?.length)
-        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema));
       if (Object.keys(props).length > 0) {
         const additionalFilterCriteria = Object.entries(props);
         for (const [prop, filter2] of additionalFilterCriteria) {
@@ -97,22 +96,22 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
       throw new Error('Archivist does not support "all"');
     }
   }
-  onArchivistCleared = () => {
+  onArchivistCleared = /* @__PURE__ */ __name(() => {
     forget((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
+  }, "onArchivistCleared");
   // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
-  onArchivistDeleted = () => {
+  onArchivistDeleted = /* @__PURE__ */ __name(() => {
     forget((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
-  onArchivistInserted = () => {
+  }, "onArchivistDeleted");
+  onArchivistInserted = /* @__PURE__ */ __name(() => {
     forget(this.updateIndex());
-  };
+  }, "onArchivistInserted");
   async stopHandler(_timeout) {
     const archivist = await this.archivistInstance(true);
     archivist.off("inserted", this.onArchivistInserted);
@@ -149,8 +148,7 @@ var GenericPayloadDiviner = class extends PayloadDiviner {
     for (const index of this.indexes ?? []) {
       this.indexMaps[index] = this.indexMaps[index] ?? [];
       for (const [payload] of pairs) {
-        if (payload[index] !== void 0)
-          this.indexMaps[index].push(payload);
+        if (payload[index] !== void 0) this.indexMaps[index].push(payload);
       }
     }
     return assertEx(pairs.at(-1))[1];

package/dist/neutral/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,oCAKK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,eAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC;AAAW,eAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,WAAOkB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,oCAKK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,wBAAN,cASGC,eAAAA;EAzCV,OAyCUA;;;EACR,OAAyBC,gBAA0B;OAAI,MAAMA;IAAeH;;EAC5E,OAAyBI,sBAA8BJ;EAE7CK,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBb;EACvC;EAEA,IAAce,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBf;EACrC;EAEUgB,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,iBAAWE,GAAG,YAAY,KAAKC,mBAAmB;AAClDH,iBAAWE,GAAG,WAAW,KAAKE,kBAAkB;AAChDJ,iBAAWE,GAAG,WAAW,KAAKG,kBAAkB;AAChD,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,UAAUD,UAAUE,OAAOC,4BAAAA,KAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,SAASN,OAAQ9B,OAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AACZ,kBAAML,QAAOjC,UAAUkC,QAAAA;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,KAASA,MAAKL,WAAWU,KAAAA;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,YAAYA,UAAUkC,QAAAA,MAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,6BAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;;EAUzEZ,qBAAyE,6BAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;EASzEd,sBAA2E,6BAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB,GAFqF;EAIrF,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC,OAAW,MAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,WAAOkB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","configSchemas","defaultConfigSchema","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index"]}

package/dist/node/index.cjs

@@ -20,10 +20,7 @@ var __copyProps = (to, from, except, desc) => {
   return to;
 };
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var __publicField = (obj, key, value) => {
-  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
-  return value;
-};
+var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
 var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
 
 // src/index.ts
@@ -105,8 +102,7 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
     const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
     let all = this.all(order, offset);
     if (all) {
-      if (schemas == null ? void 0 : schemas.length)
-        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (schemas == null ? void 0 : schemas.length) all = all.filter((payload) => schemas.includes(payload.schema));
       if (Object.keys(props).length > 0) {
         const additionalFilterCriteria = Object.entries(props);
         for (const [prop, filter2] of additionalFilterCriteria) {
@@ -123,22 +119,22 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
       throw new Error('Archivist does not support "all"');
     }
   }
-  onArchivistCleared = () => {
+  onArchivistCleared = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
+  }, "onArchivistCleared");
   // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
-  onArchivistDeleted = () => {
+  onArchivistDeleted = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
-  onArchivistInserted = () => {
+  }, "onArchivistDeleted");
+  onArchivistInserted = /* @__PURE__ */ __name(() => {
     (0, import_forget.forget)(this.updateIndex());
-  };
+  }, "onArchivistInserted");
   async stopHandler(_timeout) {
     const archivist = await this.archivistInstance(true);
     archivist.off("inserted", this.onArchivistInserted);
@@ -175,8 +171,7 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends import_diviner
     for (const index of this.indexes ?? []) {
       this.indexMaps[index] = this.indexMaps[index] ?? [];
       for (const [payload] of pairs) {
-        if (payload[index] !== void 0)
-          this.indexMaps[index].push(payload);
+        if (payload[index] !== void 0) this.indexMaps[index].push(payload);
       }
     }
     return (0, import_assert.assertEx)(pairs.at(-1))[1];

package/dist/node/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAIEC,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/BH,6CAAWE,GAAG,WAAW,KAAKE;AAC9BJ,6CAAWE,GAAG,WAAW,KAAKG;AAC9B,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,mCAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AA7H9B;AA8HkB,kBAAML,QAAOjC,mCAAUkC;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,aAAYA,mCAAUkC,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC;AAAW,eAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,eAAOkB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAjKUrE;AACR,cAVWD,wBAUc6E,iBAA0B;KAAI,2DAAMA;EAAe9E;;AAC5E,cAXWC,wBAWc8E,uBAA8B/E;AAXlD,IAAMC,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index","configSchemas","defaultConfigSchema"]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;;ACAA,oBAAyB;AACzB,oBAAuB;AAKvB,sCAA+B;AAC/B,mCAMO;AAEP,6BAA+B;AAE/B,yBAAsB;AAEtB,IAAMA,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,yBAAN,MAAMA,+BASHC,+CAAAA;EAIEC,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,yBAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/BH,6CAAWE,GAAG,WAAW,KAAKE;AAC9BJ,6CAAWE,GAAG,WAAW,KAAKG;AAC9B,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,+DAAiC,CAAA;AAClEC,gCAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,aAASE,wBAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,mCAASN,OAAQ9B,OAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AA7H9B;AA8HkB,kBAAML,QAAOjC,mCAAUkC;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,aAAYA,mCAAUkC,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,6BAAA;AACjF+B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;;EAUzEZ,qBAAyE,6BAAA;AACjF8B,+BACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;EASzEd,sBAA2E,6BAAA;AACnFgC,8BAAO,KAAKlB,YAAW,CAAA;EACzB,GAFqF;EAIrF,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,sCAAejD,SAAKqB,wBAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,oCAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,sCAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC,OAAW,MAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,eAAOkB,wBAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAjKUrE;AACR,cAVWD,wBAUc6E,iBAA0B;KAAI,2DAAMA;EAAe9E;;AAC5E,cAXWC,wBAWc8E,uBAA8B/E;AAXlD,IAAMC,wBAAN;","names":["DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index","configSchemas","defaultConfigSchema"]}

package/dist/node/index.js

@@ -3,10 +3,7 @@ var __getProtoOf = Object.getPrototypeOf;
 var __reflectGet = Reflect.get;
 var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
-var __publicField = (obj, key, value) => {
-  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
-  return value;
-};
+var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
 var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
 
 // src/Diviner.ts
@@ -80,8 +77,7 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
     const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter;
     let all = this.all(order, offset);
     if (all) {
-      if (schemas == null ? void 0 : schemas.length)
-        all = all.filter((payload) => schemas.includes(payload.schema));
+      if (schemas == null ? void 0 : schemas.length) all = all.filter((payload) => schemas.includes(payload.schema));
       if (Object.keys(props).length > 0) {
         const additionalFilterCriteria = Object.entries(props);
         for (const [prop, filter2] of additionalFilterCriteria) {
@@ -98,22 +94,22 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
       throw new Error('Archivist does not support "all"');
     }
   }
-  onArchivistCleared = () => {
+  onArchivistCleared = /* @__PURE__ */ __name(() => {
     forget((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
+  }, "onArchivistCleared");
   // we are just rebuilding the entire index at this point on delete since large archivists do not support delete
-  onArchivistDeleted = () => {
+  onArchivistDeleted = /* @__PURE__ */ __name(() => {
     forget((async () => {
       await this.clearIndex();
       await this.updateIndex();
     })());
-  };
-  onArchivistInserted = () => {
+  }, "onArchivistDeleted");
+  onArchivistInserted = /* @__PURE__ */ __name(() => {
     forget(this.updateIndex());
-  };
+  }, "onArchivistInserted");
   async stopHandler(_timeout) {
     const archivist = await this.archivistInstance(true);
     archivist.off("inserted", this.onArchivistInserted);
@@ -150,8 +146,7 @@ var _GenericPayloadDiviner = class _GenericPayloadDiviner extends PayloadDiviner
     for (const index of this.indexes ?? []) {
       this.indexMaps[index] = this.indexMaps[index] ?? [];
       for (const [payload] of pairs) {
-        if (payload[index] !== void 0)
-          this.indexMaps[index].push(payload);
+        if (payload[index] !== void 0) this.indexMaps[index].push(payload);
       }
     }
     return assertEx(pairs.at(-1))[1];

package/dist/node/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,oCAKK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAIEC,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/BH,6CAAWE,GAAG,WAAW,KAAKE;AAC9BJ,6CAAWE,GAAG,WAAW,KAAKG;AAC9B,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,mCAASN;AAAQ9B,cAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AA7H9B;AA8HkB,kBAAML,QAAOjC,mCAAUkC;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,aAAYA,mCAAUkC,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,MAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;;EAGUZ,qBAAyE,MAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ;EAEUd,sBAA2E,MAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB;EAEA,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC;AAAW,eAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,WAAOkB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAjKUrE;AACR,cAVWD,wBAUc6E,iBAA0B;KAAI,2DAAMA;EAAe9E;;AAC5E,cAXWC,wBAWc8E,uBAA8B/E;AAXlD,IAAMC,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index","configSchemas","defaultConfigSchema"]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { forget } from '@xylabs/forget'\nimport { Hash } from '@xylabs/hex'\nimport { EmptyObject, JsonObject } from '@xylabs/object'\nimport { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model'\nimport { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'\nimport {\n  isPayloadDivinerQueryPayload,\n  Order,\n  PayloadDivinerConfig,\n  PayloadDivinerParams,\n  PayloadDivinerQueryPayload,\n} from '@xyo-network/diviner-payload-model'\nimport { EventListener } from '@xyo-network/module-events'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload, Schema, WithMeta } from '@xyo-network/payload-model'\nimport { Mutex } from 'async-mutex'\n\nconst DEFAULT_INDEX_BATCH_SIZE = 100 as const\nconst DEFAULT_MAX_INDEX_SIZE = 8000 as const\n\nexport const GenericPayloadDivinerConfigSchema = 'network.xyo.diviner.payload.generic.config' as const\nexport type GenericPayloadDivinerConfigSchema = typeof GenericPayloadDivinerConfigSchema\n\nexport type GenericPayloadDivinerConfig = PayloadDivinerConfig<\n  {\n    indexes?: string[]\n  },\n  GenericPayloadDivinerConfigSchema\n>\n\nexport class GenericPayloadDiviner<\n  TParams extends PayloadDivinerParams<GenericPayloadDivinerConfig> = PayloadDivinerParams<GenericPayloadDivinerConfig>,\n  TIn extends PayloadDivinerQueryPayload<EmptyObject, Hash> = PayloadDivinerQueryPayload<EmptyObject, Hash>,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<\n    DivinerInstance<TParams, TIn, TOut>,\n    TIn,\n    TOut\n  >,\n> extends PayloadDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: Schema[] = [...super.configSchemas, GenericPayloadDivinerConfigSchema]\n  static override readonly defaultConfigSchema: Schema = GenericPayloadDivinerConfigSchema\n\n  protected indexMaps: Record<string, WithMeta<TOut>[]> = {}\n  protected payloadPairs: [WithMeta<TOut>, Hash][] = []\n\n  private _archivistInstance?: ArchivistInstance\n  private _indexOffset?: Hash\n  private _updatePayloadPairsMutex = new Mutex()\n\n  protected get indexBatchSize() {\n    return this.config.indexBatchSize ?? DEFAULT_INDEX_BATCH_SIZE\n  }\n\n  protected get indexes(): string[] {\n    return ['schema', ...(this.config.indexes ?? [])]\n  }\n\n  protected get maxIndexSize() {\n    return this.config.maxIndexSize ?? DEFAULT_MAX_INDEX_SIZE\n  }\n\n  protected all(order: Order = 'desc', offset?: Hash) {\n    return order === 'asc' ? this.allAsc(offset) : this.allDesc(offset)\n  }\n\n  protected allAsc(offset?: Hash) {\n    const pairs = this.payloadPairs\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected allDesc(offset?: Hash) {\n    const pairs = [...this.payloadPairs].reverse()\n    const startIndex = (offset ? pairs.findIndex(([, hash]) => hash === offset) ?? -1 : -1) + 1\n    return pairs.slice(startIndex).map(([payload]) => payload)\n  }\n\n  protected override async archivistInstance(): Promise<ArchivistInstance | undefined>\n  protected override async archivistInstance(required: true): Promise<ArchivistInstance>\n  protected override async archivistInstance(required = false): Promise<ArchivistInstance | undefined> {\n    if (!this._archivistInstance) {\n      const archivist = await super.archivistInstance()\n      if (required && !archivist) {\n        throw new Error('Failed to find archivist')\n      }\n      archivist?.on('inserted', this.onArchivistInserted)\n      archivist?.on('cleared', this.onArchivistCleared)\n      archivist?.on('deleted', this.onArchivistDeleted)\n      this._archivistInstance = archivist\n    }\n    return this._archivistInstance\n  }\n\n  protected async clearIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(() => {\n      this._indexOffset = undefined\n      this.payloadPairs = []\n      this.indexMaps = {}\n    })\n  }\n\n  protected override async divineHandler(payloads?: TIn[]): Promise<WithMeta<TOut>[]> {\n    const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? []\n    assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified')\n    const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as WithMeta<\n      PayloadDivinerQueryPayload<EmptyObject, Hash>\n    >\n\n    await this.updateIndex()\n\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    const { $hash, $meta, schema, schemas, order, limit, offset, ...props } = filter\n    let all: WithMeta<TOut>[] = this.all(order, offset)\n    if (all) {\n      if (schemas?.length) all = all.filter((payload) => schemas.includes(payload.schema))\n      if (Object.keys(props).length > 0) {\n        const additionalFilterCriteria = Object.entries(props)\n        for (const [prop, filter] of additionalFilterCriteria) {\n          const property = prop as keyof TOut\n          all =\n            Array.isArray(filter) ?\n              all.filter((payload) =>\n                filter.every((value) => {\n                  const prop = payload?.[property]\n                  //TODO: This seems to be written just to check arrays, and now that $meta is there, need to check type?\n                  return Array.isArray(prop) && prop.includes?.(value)\n                }),\n              )\n            : all.filter((payload) => payload?.[property] === filter)\n        }\n      }\n      return limit ? all.slice(0, limit) : all\n    } else {\n      throw new Error('Archivist does not support \"all\"')\n    }\n  }\n\n  protected onArchivistCleared: EventListener<ArchivistModuleEventData['cleared']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  // we are just rebuilding the entire index at this point on delete since large archivists do not support delete\n  protected onArchivistDeleted: EventListener<ArchivistModuleEventData['deleted']> = () => {\n    forget(\n      (async () => {\n        await this.clearIndex()\n        await this.updateIndex()\n      })(),\n    )\n  }\n\n  protected onArchivistInserted: EventListener<ArchivistModuleEventData['inserted']> = () => {\n    forget(this.updateIndex())\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    const archivist = await this.archivistInstance(true)\n    archivist.off('inserted', this.onArchivistInserted)\n    archivist.off('deleted', this.onArchivistDeleted)\n    archivist.off('cleared', this.onArchivistCleared)\n    return await super.stopHandler()\n  }\n\n  //index any new payloads\n  protected async updateIndex() {\n    await this._updatePayloadPairsMutex.runExclusive(async () => {\n      const archivist = await this.archivistInstance(true)\n      let newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      while (newPayloads.length > 0) {\n        const prevOffset = this._indexOffset\n        this._indexOffset = await PayloadBuilder.hash(assertEx(newPayloads.at(-1)))\n        if (this._indexOffset === prevOffset) {\n          this.logger.warn('next offset not found', prevOffset)\n        }\n        assertEx(this.payloadPairs.length + newPayloads.length <= this.maxIndexSize, () => 'maxIndexSize exceeded')\n        await this.indexPayloads(newPayloads)\n        newPayloads = (await archivist.next({ limit: 100, offset: this._indexOffset })) as WithMeta<TOut>[]\n      }\n    })\n  }\n\n  private async indexPayloads(payloads: TOut[]): Promise<Hash> {\n    const pairs = await PayloadBuilder.hashPairs(payloads)\n    this.payloadPairs.push(...pairs)\n\n    //update the custom indexes\n    for (const index of this.indexes ?? []) {\n      this.indexMaps[index] = this.indexMaps[index] ?? []\n      for (const [payload] of pairs) {\n        if ((payload as unknown as JsonObject)[index] !== undefined) this.indexMaps[index].push(payload)\n      }\n    }\n    return assertEx(pairs.at(-1))[1]\n  }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,cAAc;AAKvB,SAASC,sBAAsB;AAC/B,SACEC,oCAKK;AAEP,SAASC,sBAAsB;AAE/B,SAASC,aAAa;AAEtB,IAAMC,2BAA2B;AACjC,IAAMC,yBAAyB;AAExB,IAAMC,oCAAoC;AAU1C,IAAMC,yBAAN,MAAMA,+BASHC,eAAAA;EAIEC,YAA8C,CAAC;EAC/CC,eAAyC,CAAA;EAE3CC;EACAC;EACAC,2BAA2B,IAAIC,MAAAA;EAEvC,IAAcC,iBAAiB;AAC7B,WAAO,KAAKC,OAAOD,kBAAkBX;EACvC;EAEA,IAAca,UAAoB;AAChC,WAAO;MAAC;SAAc,KAAKD,OAAOC,WAAW,CAAA;;EAC/C;EAEA,IAAcC,eAAe;AAC3B,WAAO,KAAKF,OAAOE,gBAAgBb;EACrC;EAEUc,IAAIC,QAAe,QAAQC,QAAe;AAClD,WAAOD,UAAU,QAAQ,KAAKE,OAAOD,MAAAA,IAAU,KAAKE,QAAQF,MAAAA;EAC9D;EAEUC,OAAOD,QAAe;AAC9B,UAAMG,QAAQ,KAAKd;AACnB,UAAMe,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAEUP,QAAQF,QAAe;AAC/B,UAAMG,QAAQ;SAAI,KAAKd;MAAcqB,QAAO;AAC5C,UAAMN,cAAcJ,SAASG,MAAME,UAAU,CAAC,CAAA,EAAGC,IAAAA,MAAUA,SAASN,MAAAA,KAAW,KAAK,MAAM;AAC1F,WAAOG,MAAMI,MAAMH,UAAAA,EAAYI,IAAI,CAAC,CAACC,OAAAA,MAAaA,OAAAA;EACpD;EAIA,MAAyBE,kBAAkBC,WAAW,OAA+C;AACnG,QAAI,CAAC,KAAKtB,oBAAoB;AAC5B,YAAMuB,YAAY,MAAM,MAAMF,kBAAAA;AAC9B,UAAIC,YAAY,CAACC,WAAW;AAC1B,cAAM,IAAIC,MAAM,0BAAA;MAClB;AACAD,6CAAWE,GAAG,YAAY,KAAKC;AAC/BH,6CAAWE,GAAG,WAAW,KAAKE;AAC9BJ,6CAAWE,GAAG,WAAW,KAAKG;AAC9B,WAAK5B,qBAAqBuB;IAC5B;AACA,WAAO,KAAKvB;EACd;EAEA,MAAgB6B,aAAa;AAC3B,UAAM,KAAK3B,yBAAyB4B,aAAa,MAAA;AAC/C,WAAK7B,eAAe8B;AACpB,WAAKhC,eAAe,CAAA;AACpB,WAAKD,YAAY,CAAC;IACpB,CAAA;EACF;EAEA,MAAyBkC,cAAcC,UAA6C;AAClF,UAAMC,WAAUD,qCAAUE,OAAOC,kCAAiC,CAAA;AAClEC,aAASH,QAAQI,SAAS,GAAG,MAAM,4DAAA;AACnC,UAAMH,SAASE,SAASH,QAAQK,MAAK,GAAI,MAAM,kCAAA;AAI/C,UAAM,KAAKC,YAAW;AAGtB,UAAM,EAAEC,OAAOC,OAAOC,QAAQC,SAASnC,OAAOoC,OAAOnC,QAAQ,GAAGoC,MAAAA,IAAUX;AAC1E,QAAI3B,MAAwB,KAAKA,IAAIC,OAAOC,MAAAA;AAC5C,QAAIF,KAAK;AACP,UAAIoC,mCAASN,OAAQ9B,OAAMA,IAAI2B,OAAO,CAAChB,YAAYyB,QAAQG,SAAS5B,QAAQwB,MAAM,CAAA;AAClF,UAAIK,OAAOC,KAAKH,KAAAA,EAAOR,SAAS,GAAG;AACjC,cAAMY,2BAA2BF,OAAOG,QAAQL,KAAAA;AAChD,mBAAW,CAACM,MAAMjB,OAAAA,KAAWe,0BAA0B;AACrD,gBAAMG,WAAWD;AACjB5C,gBACE8C,MAAMC,QAAQpB,OAAAA,IACZ3B,IAAI2B,OAAO,CAAChB,YACVgB,QAAOqB,MAAM,CAACC,UAAAA;AA7H9B;AA8HkB,kBAAML,QAAOjC,mCAAUkC;AAEvB,mBAAOC,MAAMC,QAAQH,KAAAA,OAASA,KAAAA,MAAKL,aAALK,wBAAAA,OAAgBK;UAChD,CAAA,CAAA,IAEFjD,IAAI2B,OAAO,CAAChB,aAAYA,mCAAUkC,eAAclB,OAAAA;QACtD;MACF;AACA,aAAOU,QAAQrC,IAAIS,MAAM,GAAG4B,KAAAA,IAASrC;IACvC,OAAO;AACL,YAAM,IAAIgB,MAAM,kCAAA;IAClB;EACF;EAEUG,qBAAyE,6BAAA;AACjF+B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;;EAUzEZ,qBAAyE,6BAAA;AACjF8B,YACG,YAAA;AACC,YAAM,KAAK7B,WAAU;AACrB,YAAM,KAAKW,YAAW;IACxB,GAAA,CAAA;EAEJ,GAPmF;EASzEd,sBAA2E,6BAAA;AACnFgC,WAAO,KAAKlB,YAAW,CAAA;EACzB,GAFqF;EAIrF,MAAyBmB,YAAYC,UAAiD;AACpF,UAAMrC,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/CE,cAAUsC,IAAI,YAAY,KAAKnC,mBAAmB;AAClDH,cAAUsC,IAAI,WAAW,KAAKjC,kBAAkB;AAChDL,cAAUsC,IAAI,WAAW,KAAKlC,kBAAkB;AAChD,WAAO,MAAM,MAAMgC,YAAAA;EACrB;;EAGA,MAAgBnB,cAAc;AAC5B,UAAM,KAAKtC,yBAAyB4B,aAAa,YAAA;AAC/C,YAAMP,YAAY,MAAM,KAAKF,kBAAkB,IAAA;AAC/C,UAAIyC,cAAe,MAAMvC,UAAUwC,KAAK;QAAElB,OAAO;QAAKnC,QAAQ,KAAKT;MAAa,CAAA;AAChF,aAAO6D,YAAYxB,SAAS,GAAG;AAC7B,cAAM0B,aAAa,KAAK/D;AACxB,aAAKA,eAAe,MAAMgE,eAAejD,KAAKqB,SAASyB,YAAYI,GAAG,EAAC,CAAA,CAAA;AACvE,YAAI,KAAKjE,iBAAiB+D,YAAY;AACpC,eAAKG,OAAOC,KAAK,yBAAyBJ,UAAAA;QAC5C;AACA3B,iBAAS,KAAKtC,aAAauC,SAASwB,YAAYxB,UAAU,KAAK/B,cAAc,MAAM,uBAAA;AACnF,cAAM,KAAK8D,cAAcP,WAAAA;AACzBA,sBAAe,MAAMvC,UAAUwC,KAAK;UAAElB,OAAO;UAAKnC,QAAQ,KAAKT;QAAa,CAAA;MAC9E;IACF,CAAA;EACF;EAEA,MAAcoE,cAAcpC,UAAiC;AAC3D,UAAMpB,QAAQ,MAAMoD,eAAeK,UAAUrC,QAAAA;AAC7C,SAAKlC,aAAawE,KAAI,GAAI1D,KAAAA;AAG1B,eAAW2D,SAAS,KAAKlE,WAAW,CAAA,GAAI;AACtC,WAAKR,UAAU0E,KAAAA,IAAS,KAAK1E,UAAU0E,KAAAA,KAAU,CAAA;AACjD,iBAAW,CAACrD,OAAAA,KAAYN,OAAO;AAC7B,YAAKM,QAAkCqD,KAAAA,MAAWzC,OAAW,MAAKjC,UAAU0E,KAAAA,EAAOD,KAAKpD,OAAAA;MAC1F;IACF;AACA,WAAOkB,SAASxB,MAAMqD,GAAG,EAAC,CAAA,EAAI,CAAA;EAChC;AACF;AAjKUrE;AACR,cAVWD,wBAUc6E,iBAA0B;KAAI,2DAAMA;EAAe9E;;AAC5E,cAXWC,wBAWc8E,uBAA8B/E;AAXlD,IAAMC,wBAAN;","names":["assertEx","forget","PayloadDiviner","isPayloadDivinerQueryPayload","PayloadBuilder","Mutex","DEFAULT_INDEX_BATCH_SIZE","DEFAULT_MAX_INDEX_SIZE","GenericPayloadDivinerConfigSchema","GenericPayloadDiviner","PayloadDiviner","indexMaps","payloadPairs","_archivistInstance","_indexOffset","_updatePayloadPairsMutex","Mutex","indexBatchSize","config","indexes","maxIndexSize","all","order","offset","allAsc","allDesc","pairs","startIndex","findIndex","hash","slice","map","payload","reverse","archivistInstance","required","archivist","Error","on","onArchivistInserted","onArchivistCleared","onArchivistDeleted","clearIndex","runExclusive","undefined","divineHandler","payloads","filters","filter","isPayloadDivinerQueryPayload","assertEx","length","shift","updateIndex","$hash","$meta","schema","schemas","limit","props","includes","Object","keys","additionalFilterCriteria","entries","prop","property","Array","isArray","every","value","forget","stopHandler","_timeout","off","newPayloads","next","prevOffset","PayloadBuilder","at","logger","warn","indexPayloads","hashPairs","push","index","configSchemas","defaultConfigSchema"]}

package/package.json

@@ -10,27 +10,27 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
   },
   "dependencies": {
-    "@xylabs/assert": "^3.5.0",
-    "@xylabs/forget": "^3.5.0",
-    "@xylabs/hex": "^3.5.0",
-    "@xylabs/object": "^3.5.0",
-    "@xyo-network/archivist-model": "~2.103.9",
-    "@xyo-network/diviner-model": "~2.103.9",
-    "@xyo-network/diviner-payload-abstract": "~2.103.9",
-    "@xyo-network/diviner-payload-model": "~2.103.9",
-    "@xyo-network/module-events": "~2.103.9",
-    "@xyo-network/payload-builder": "~2.103.9",
-    "@xyo-network/payload-model": "~2.103.9",
+    "@xylabs/assert": "^3.5.1",
+    "@xylabs/forget": "^3.5.1",
+    "@xylabs/hex": "^3.5.1",
+    "@xylabs/object": "^3.5.1",
+    "@xyo-network/archivist-model": "~2.104.1",
+    "@xyo-network/diviner-model": "~2.104.1",
+    "@xyo-network/diviner-payload-abstract": "~2.104.1",
+    "@xyo-network/diviner-payload-model": "~2.104.1",
+    "@xyo-network/module-events": "~2.104.1",
+    "@xyo-network/payload-builder": "~2.104.1",
+    "@xyo-network/payload-model": "~2.104.1",
     "async-mutex": "^0.5.0"
   },
   "devDependencies": {
-    "@xylabs/ts-scripts-yarn3": "^3.10.4",
-    "@xylabs/tsconfig": "^3.10.4",
-    "@xyo-network/account": "~2.103.9",
-    "@xyo-network/archivist-indexeddb": "~2.103.9",
-    "@xyo-network/archivist-memory": "~2.103.9",
-    "@xyo-network/node-memory": "~2.103.9",
-    "@xyo-network/payload-builder": "~2.103.9",
+    "@xylabs/ts-scripts-yarn3": "^3.11.2",
+    "@xylabs/tsconfig": "^3.11.2",
+    "@xyo-network/account": "~2.104.1",
+    "@xyo-network/archivist-indexeddb": "~2.104.1",
+    "@xyo-network/archivist-memory": "~2.104.1",
+    "@xyo-network/node-memory": "~2.104.1",
+    "@xyo-network/payload-builder": "~2.104.1",
     "fake-indexeddb": "^6.0.0",
     "typescript": "^5.4.5"
   },
@@ -73,6 +73,6 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
   },
   "sideEffects": false,
-  "version": "2.103.9",
+  "version": "2.104.1",
   "type": "module"
 }