@xyo-network/diviner-indexing-memory 2.83.6 → 2.84.0

package/dist/browser/index.cjs

@@ -148,7 +148,7 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
       return this._lastState;
     let hash = "";
     const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
-    const query = new import_payload_builder.PayloadBuilder({ schema: import_diviner_boundwitness_model.BoundWitnessDivinerQuerySchema }).fields({
+    const query = await new import_payload_builder.PayloadBuilder({ schema: import_diviner_boundwitness_model.BoundWitnessDivinerQuerySchema }).fields({
       address: this.account.address,
       limit: 1,
       offset: 0,

package/dist/browser/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,oBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,oBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,MAAM,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EAC/G,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/dist/browser/index.js

@@ -124,7 +124,7 @@ var IndexingDiviner = class extends AbstractDiviner {
       return this._lastState;
     let hash = "";
     const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
-    const query = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }).fields({
+    const query = await new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }).fields({
       address: this.account.address,
       limit: 1,
       offset: 0,

package/dist/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,MAAM,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EAC/G,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/dist/node/index.js

@@ -153,7 +153,7 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
       return this._lastState;
     let hash = "";
     const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
-    const query = new import_payload_builder.PayloadBuilder({ schema: import_diviner_boundwitness_model.BoundWitnessDivinerQuerySchema }).fields({
+    const query = await new import_payload_builder.PayloadBuilder({ schema: import_diviner_boundwitness_model.BoundWitnessDivinerQuerySchema }).fields({
       address: this.account.address,
       limit: 1,
       offset: 0,

package/dist/node/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,oBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,wCAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,oBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,MAAM,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EAC/G,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,wCAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/dist/node/index.mjs

@@ -129,7 +129,7 @@ var IndexingDiviner = class extends AbstractDiviner {
       return this._lastState;
     let hash = "";
     const diviner = await this.getBoundWitnessDivinerForStore("stateStore");
-    const query = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }).fields({
+    const query = await new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }).fields({
       address: this.account.address,
       limit: 1,
       offset: 0,

package/dist/node/index.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,gBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,oBAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
+{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n  IndexingDivinerConfig,\n  IndexingDivinerConfigSchema,\n  IndexingDivinerParams,\n  IndexingDivinerStage,\n  IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n  TParams extends IndexingDivinerParams = IndexingDivinerParams,\n  TIn extends Payload = Payload,\n  TOut extends Payload = Payload,\n  TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n  static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n  private _lastState?: ModuleState<IndexingDivinerState>\n  private _pollId?: string | number | NodeJS.Timeout\n\n  get payloadDivinerLimit() {\n    return this.config.payloadDivinerLimit ?? 1_000\n  }\n\n  get pollFrequency() {\n    return this.config.pollFrequency ?? 10_000\n  }\n\n  /**\n   * Works via batched iteration of the source archivist to populate the index.\n   * @returns A promise that resolves when the background process is complete\n   */\n  protected backgroundDivine = async (): Promise<void> => {\n    // Load last state\n    const lastState = await this.retrieveState()\n    // Get next batch of results\n    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n    // Filter next state out from results\n    const nextState = results.find(isModuleState<IndexingDivinerState>)\n    const indexCandidates = results.filter((x) => !isModuleState(x))\n    // Transform candidates to indexes\n    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n    const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n    // Insert index results\n    const indexArchivist = await this.getArchivistForStore('indexStore')\n    await indexArchivist.insert(indexes)\n    // Update state\n    if (nextState) {\n      await this.commitState(nextState)\n    }\n  }\n\n  /**\n   * Commit the internal state of the Diviner process. This is similar\n   * to a transaction completion in a database and should only be called\n   * when results have been successfully persisted to the appropriate\n   * external stores.\n   * @param nextState The state to commit\n   */\n  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n    // Don't commit state if no state has changed\n    if (nextState.state.offset === this._lastState?.state.offset) return\n    this._lastState = nextState\n    const archivist = await this.getArchivistForStore('stateStore')\n    const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n    await archivist.insert([bw, nextState])\n  }\n\n  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n    const results = (\n      await Promise.all(\n        payloads.map(async (payload) => {\n          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n          // Divine the results\n          const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n          // Transform the results to the response shape\n          const response = await Promise.all(\n            indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n          )\n          return response.flat()\n        }),\n      )\n    ).flat()\n    // TODO: Infer this type over casting to this type\n    return results as TOut[]\n  }\n\n  /**\n   * Retrieves the archivist for the specified store\n   * @param store The store to retrieve the archivist for\n   * @returns The archivist for the specified store\n   */\n  protected async getArchivistForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n    return ArchivistWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the BoundWitness Diviner for the specified store\n   * @param store The store to retrieve the BoundWitness Diviner for\n   * @returns The BoundWitness Diviner for the specified store\n   */\n  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Gets the Diviner for the supplied Indexing Diviner stage\n   * @param transform The Indexing Diviner stage\n   * @returns The diviner corresponding to the Indexing Diviner stage\n   */\n  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n    const nameOrAddress = assertEx(\n      this.config?.indexingDivinerStages?.[transform],\n      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n    )\n    const mod = await this.resolve(nameOrAddress)\n    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n  }\n\n  /**\n   * Retrieves the Payload Diviner for the specified store\n   * @param store The store to retrieve the Payload Diviner for\n   * @returns The Payload Diviner for the specified store\n   */\n  protected async getPayloadDivinerForStore(store: ConfigStore) {\n    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n    return DivinerWrapper.wrap(mod, this.account)\n  }\n\n  /**\n   * Retrieves the last state of the Diviner process. Used to recover state after\n   * preemptions, reboots, etc.\n   */\n  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n    if (this._lastState) return this._lastState\n    let hash: string = ''\n    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n      .fields({\n        address: this.account.address,\n        limit: 1,\n        offset: 0,\n        order: 'desc',\n        payload_schemas: [ModuleStateSchema],\n      })\n      .build()\n    const boundWitnesses = await diviner.divine([query])\n    if (boundWitnesses.length > 0) {\n      const boundWitness = boundWitnesses[0]\n      if (isBoundWitness(boundWitness)) {\n        // Find the index for this address in the BoundWitness that is a ModuleState\n        hash = boundWitness.addresses\n          .map((address, index) => ({ address, index }))\n          .filter(({ address }) => address === this.account.address)\n          .reduce(\n            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n            '',\n          )\n      }\n    }\n\n    // If we able to located the last state\n    if (hash) {\n      // Get last state\n      const archivist = await this.getArchivistForStore('stateStore')\n      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n      if (payload) {\n        return payload\n      }\n    }\n    return undefined\n  }\n\n  protected override async startHandler(): Promise<boolean> {\n    await super.startHandler()\n    this.poll()\n    return true\n  }\n\n  protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n    if (this._pollId) {\n      clearTimeout(this._pollId)\n      this._pollId = undefined\n    }\n    return await super.stopHandler()\n  }\n\n  /**\n   * Runs the background divine process on a loop with a delay\n   * specified by the `config.pollFrequency`\n   */\n  private poll() {\n    this._pollId = setTimeout(async () => {\n      try {\n        await Promise.resolve()\n        await this.backgroundDivine()\n      } catch (e) {\n        console.log(e)\n      } finally {\n        if (this._pollId) clearTimeout(this._pollId)\n        this._pollId = undefined\n        this.poll()\n      }\n    }, this.pollFrequency)\n  }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,gBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,MAAM,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EAC/G,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,oBAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}

package/package.json

@@ -11,17 +11,17 @@
   },
   "dependencies": {
     "@xylabs/assert": "^2.13.20",
-    "@xyo-network/abstract-diviner": "~2.83.6",
-    "@xyo-network/archivist-wrapper": "~2.83.6",
-    "@xyo-network/boundwitness-builder": "~2.83.6",
-    "@xyo-network/boundwitness-model": "~2.83.6",
-    "@xyo-network/diviner-boundwitness-model": "~2.83.6",
-    "@xyo-network/diviner-indexing-model": "~2.83.6",
-    "@xyo-network/diviner-model": "~2.83.6",
-    "@xyo-network/diviner-wrapper": "~2.83.6",
-    "@xyo-network/module-model": "~2.83.6",
-    "@xyo-network/payload-builder": "~2.83.6",
-    "@xyo-network/payload-model": "~2.83.6"
+    "@xyo-network/abstract-diviner": "~2.84.0",
+    "@xyo-network/archivist-wrapper": "~2.84.0",
+    "@xyo-network/boundwitness-builder": "~2.84.0",
+    "@xyo-network/boundwitness-model": "~2.84.0",
+    "@xyo-network/diviner-boundwitness-model": "~2.84.0",
+    "@xyo-network/diviner-indexing-model": "~2.84.0",
+    "@xyo-network/diviner-model": "~2.84.0",
+    "@xyo-network/diviner-wrapper": "~2.84.0",
+    "@xyo-network/module-model": "~2.84.0",
+    "@xyo-network/payload-builder": "~2.84.0",
+    "@xyo-network/payload-model": "~2.84.0"
   },
   "devDependencies": {
     "@xylabs/ts-scripts-yarn3": "^3.2.10",
@@ -68,5 +68,5 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
   },
   "sideEffects": false,
-  "version": "2.83.6"
+  "version": "2.84.0"
 }

package/src/Diviner.ts

@@ -160,7 +160,7 @@ export class IndexingDiviner<
     if (this._lastState) return this._lastState
     let hash: string = ''
     const diviner = await this.getBoundWitnessDivinerForStore('stateStore')
-    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
+    const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
       .fields({
         address: this.account.address,
         limit: 1,