@xyo-network/diviner-indexing-memory 2.81.13 → 2.81.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/dist/browser/index.cjs +1 -0
  2. package/dist/browser/index.cjs.map +1 -1
  3. package/dist/browser/index.js +1 -0
  4. package/dist/browser/index.js.map +1 -1
  5. package/dist/node/index.js +1 -0
  6. package/dist/node/index.js.map +1 -1
  7. package/dist/node/index.mjs +1 -0
  8. package/dist/node/index.mjs.map +1 -1
  9. package/package.json +12 -12
  10. package/src/Diviner.ts +1 -1
package/dist/browser/index.cjs CHANGED
@@ -194,6 +194,7 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
194
194
  this._pollId = setTimeout(async () => {
195
195
  try {
196
196
  await Promise.resolve();
197
+ await this.backgroundDivine();
197
198
  } catch (e) {
198
199
  console.log(e);
199
200
  } finally {
package/dist/browser/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n //await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,oBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AAAA,MAExB,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,oBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,WAAO,wBAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,wCAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
package/dist/browser/index.js CHANGED
@@ -170,6 +170,7 @@ var IndexingDiviner = class extends AbstractDiviner {
170
170
  this._pollId = setTimeout(async () => {
171
171
  try {
172
172
  await Promise.resolve();
173
+ await this.backgroundDivine();
173
174
  } catch (e) {
174
175
  console.log(e);
175
176
  } finally {
package/dist/browser/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n //await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AAAA,MAExB,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AAExE,QAAI,UAAU,MAAM,WAAW,KAAK,YAAY,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AACvD,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AACjE,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AACvE,UAAM,gBAAgB;AAAA,MACpB,KAAK,QAAQ,wBAAwB,SAAS;AAAA,MAC9C,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AAC5D,UAAM,OAAO,SAAS,KAAK,SAAS,KAAK,GAAG,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAU,aAAa,kBAAkB,MAAM,KAAK,MAAM,oBAAoB,aAAa,eAAe,MAAM,KAAK,IAAI;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
package/dist/node/index.js CHANGED
@@ -202,6 +202,7 @@ var IndexingDiviner = class extends import_abstract_diviner.AbstractDiviner {
202
202
  this._pollId = setTimeout(async () => {
203
203
  try {
204
204
  await Promise.resolve();
205
+ await this.backgroundDivine();
205
206
  } catch (e) {
206
207
  console.log(e);
207
208
  } finally {
package/dist/node/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n //await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,oBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,wCAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AAAA,MAExB,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
1
+ {"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAyB;AACzB,8BAAgC;AAChC,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA8D;AAC9D,6BAA+B;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,wCAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,2DAA6B,wCAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,iCAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,KAAC,mCAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,gDAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,0CAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,oBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,eAAO,4BAAS,wCAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,WAAO,yBAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,UAAM,wBAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,sCAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,sCAAgD,EAAE,QAAQ,iEAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,qCAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,cAAI,0CAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,wCAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,iCAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
package/dist/node/index.mjs CHANGED
@@ -178,6 +178,7 @@ var IndexingDiviner = class extends AbstractDiviner {
178
178
  this._pollId = setTimeout(async () => {
179
179
  try {
180
180
  await Promise.resolve();
181
+ await this.backgroundDivine();
181
182
  } catch (e) {
182
183
  console.log(e);
183
184
  } finally {
package/dist/node/index.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n //await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,gBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,oBAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AAAA,MAExB,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
1
+ {"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { AbstractDiviner } from '@xyo-network/abstract-diviner'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\ntype ConfigStoreKey = 'indexStore' | 'stateStore'\n\ntype ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1_000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,uBAAuB;AAChC,SAAS,wBAAwB;AACjC,SAAS,2BAA2B;AACpC,SAAS,sBAAsB;AAC/B,SAA0C,sCAAsC;AAChF;AAAA,EAEE;AAAA,OAIK;AACP,SAAS,mBAAmB,2BAAkE;AAC9F,SAAS,sBAAsB;AAC/B,SAAS,eAA4B,yBAAyB;AAC9D,SAAS,sBAAsB;AAO/B,IAAM,aAAa;AAEZ,IAAM,kBAAN,cAKG,gBAAgD;AAAA,EACxD,OAAyB,gBAA0B,CAAC,6BAA6B,mBAAmB;AAAA,EAE5F;AAAA,EACA;AAAA,EAER,IAAI,sBAAsB;AACxB,WAAO,KAAK,OAAO,uBAAuB;AAAA,EAC5C;AAAA,EAEA,IAAI,gBAAgB;AAClB,WAAO,KAAK,OAAO,iBAAiB;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMU,mBAAmB,YAA2B;AAEtD,UAAM,YAAY,MAAM,KAAK,cAAc;AAE3C,UAAM,wBAAwB,MAAM,KAAK,wBAAwB,8BAA8B;AAC/F,UAAM,UAAU,YAAY,MAAM,sBAAsB,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,sBAAsB,OAAO;AAEjH,UAAM,YAAY,QAAQ,KAAK,aAAmC;AAClE,UAAM,kBAAkB,QAAQ,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;AAE/D,UAAM,0BAA0B,MAAM,KAAK,wBAAwB,8BAA8B;AACjG,UAAM,UAAU,MAAM,wBAAwB,OAAO,eAAe;AAEpE,UAAM,iBAAiB,MAAM,KAAK,qBAAqB,YAAY;AACnE,UAAM,eAAe,OAAO,OAAO;AAEnC,QAAI,WAAW;AACb,YAAM,KAAK,YAAY,SAAS;AAAA,IAClC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAgB,YAAY,WAA8C;AA5E5E;AA8EI,QAAI,UAAU,MAAM,aAAW,UAAK,eAAL,mBAAiB,MAAM;AAAQ;AAC9D,SAAK,aAAa;AAClB,UAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,UAAM,CAAC,EAAE,IAAI,MAAM,IAAI,oBAAoB,EAAE,QAAQ,SAAS,EAAE,QAAQ,KAAK,OAAO,EAAE,MAAM;AAC5F,UAAM,UAAU,OAAO,CAAC,IAAI,SAAS,CAAC;AAAA,EACxC;AAAA,EAEA,MAAyB,cAAc,WAAkB,CAAC,GAAoB;AAC5E,UAAM,sBAAsB,MAAM,KAAK,0BAA0B,YAAY;AAC7E,UAAM,kCAAkC,MAAM,KAAK,wBAAwB,iCAAiC;AAC5G,UAAM,kDAAkD,MAAM,KAAK,wBAAwB,iDAAiD;AAC5I,UAAM,WACJ,MAAM,QAAQ;AAAA,MACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,cAAM,aAAa,MAAM,gCAAgC,OAAO,CAAC,OAAO,CAAC;AAEzE,cAAM,iBAAiB,MAAM,oBAAoB,OAAO,UAAU;AAElE,cAAM,WAAW,MAAM,QAAQ;AAAA,UAC7B,eAAe,KAAK,EAAE,IAAI,CAAC,kBAAkB,gDAAgD,OAAO,CAAC,SAAS,aAAa,CAAC,CAAC;AAAA,QAC/H;AACA,eAAO,SAAS,KAAK;AAAA,MACvB,CAAC;AAAA,IACH,GACA,KAAK;AAEP,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,qBAAqB,OAAoB;AAhH3D;AAiHI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,WAAW,MAAM,GAAG,UAAU,gBAAgB,KAAK,0BAA0B;AACzH,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,YAAY;AAC1G,WAAO,iBAAiB,KAAK,KAAK,KAAK,OAAO;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,+BAA+B,OAAoB;AA3HrE;AA4HI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,qBAAqB,MAAM,GAAG,UAAU,gBAAgB,KAAK,oCAAoC;AAC7I,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,sBAAsB;AACpH,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,wBAAwB,WAAiC;AAtI3E;AAuII,UAAM,gBAAgB;AAAA,OACpB,gBAAK,WAAL,mBAAa,0BAAb,mBAAqC;AAAA,MACrC,MAAM,GAAG,UAAU,sCAAsC,SAAS;AAAA,IACpE;AACA,UAAM,MAAM,MAAM,KAAK,QAAQ,aAAa;AAC5C,WAAO,SAAS,kBAAkB,GAAG,GAAG,MAAM,GAAG,UAAU,kDAAkD,SAAS,EAAE;AAAA,EAC1H;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAgB,0BAA0B,OAAoB;AApJhE;AAqJI,UAAM,OAAO,UAAS,gBAAK,WAAL,mBAAc,WAAd,mBAAsB,gBAAgB,MAAM,GAAG,UAAU,gBAAgB,KAAK,+BAA+B;AACnI,UAAM,MAAM,SAAS,MAAM,KAAK,QAAQ,IAAI,GAAG,MAAM,GAAG,UAAU,uBAAuB,KAAK,iBAAiB;AAC/G,WAAO,eAAe,KAAK,KAAK,KAAK,OAAO;AAAA,EAC9C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAgB,gBAAwE;AACtF,QAAI,KAAK;AAAY,aAAO,KAAK;AACjC,QAAI,OAAe;AACnB,UAAM,UAAU,MAAM,KAAK,+BAA+B,YAAY;AACtE,UAAM,QAAQ,IAAI,eAAgD,EAAE,QAAQ,+BAA+B,CAAC,EACzG,OAAO;AAAA,MACN,SAAS,KAAK,QAAQ;AAAA,MACtB,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,iBAAiB,CAAC,iBAAiB;AAAA,IACrC,CAAC,EACA,MAAM;AACT,UAAM,iBAAiB,MAAM,QAAQ,OAAO,CAAC,KAAK,CAAC;AACnD,QAAI,eAAe,SAAS,GAAG;AAC7B,YAAM,eAAe,eAAe,CAAC;AACrC,UAAI,eAAe,YAAY,GAAG;AAEhC,eAAO,aAAa,UACjB,IAAI,CAAC,SAAS,WAAW,EAAE,SAAS,MAAM,EAAE,EAC5C,OAAO,CAAC,EAAE,QAAQ,MAAM,YAAY,KAAK,QAAQ,OAAO,EACxD;AAAA,UACC,CAAC,MAAM,SAAM;AApLzB;AAoL6B,uCAAa,oBAAb,mBAA+B,6BAAM,YAAW,oBAAoB,aAAa,eAAe,6BAAM,KAAK,IAAI;AAAA;AAAA,UAChI;AAAA,QACF;AAAA,MACJ;AAAA,IACF;AAGA,QAAI,MAAM;AAER,YAAM,YAAY,MAAM,KAAK,qBAAqB,YAAY;AAC9D,YAAM,WAAW,MAAM,UAAU,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,aAAmC;AACtF,UAAI,SAAS;AACX,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,eAAiC;AACxD,UAAM,MAAM,aAAa;AACzB,SAAK,KAAK;AACV,WAAO;AAAA,EACT;AAAA,EAEA,MAAyB,YAAY,UAAiD;AACpF,QAAI,KAAK,SAAS;AAChB,mBAAa,KAAK,OAAO;AACzB,WAAK,UAAU;AAAA,IACjB;AACA,WAAO,MAAM,MAAM,YAAY;AAAA,EACjC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,OAAO;AACb,SAAK,UAAU,WAAW,YAAY;AACpC,UAAI;AACF,cAAM,QAAQ,QAAQ;AACtB,cAAM,KAAK,iBAAiB;AAAA,MAC9B,SAAS,GAAG;AACV,gBAAQ,IAAI,CAAC;AAAA,MACf,UAAE;AACA,YAAI,KAAK;AAAS,uBAAa,KAAK,OAAO;AAC3C,aAAK,UAAU;AACf,aAAK,KAAK;AAAA,MACZ;AAAA,IACF,GAAG,KAAK,aAAa;AAAA,EACvB;AACF;","names":[]}
package/package.json CHANGED
@@ -11,17 +11,17 @@
11
11
  },
12
12
  "dependencies": {
13
13
  "@xylabs/assert": "^2.13.20",
14
- "@xyo-network/abstract-diviner": "~2.81.13",
15
- "@xyo-network/archivist-wrapper": "~2.81.13",
16
- "@xyo-network/boundwitness-builder": "~2.81.13",
17
- "@xyo-network/boundwitness-model": "~2.81.13",
18
- "@xyo-network/diviner-boundwitness-model": "~2.81.13",
19
- "@xyo-network/diviner-indexing-model": "~2.81.13",
20
- "@xyo-network/diviner-model": "~2.81.13",
21
- "@xyo-network/diviner-wrapper": "~2.81.13",
22
- "@xyo-network/module-model": "~2.81.13",
23
- "@xyo-network/payload-builder": "~2.81.13",
24
- "@xyo-network/payload-model": "~2.81.13"
14
+ "@xyo-network/abstract-diviner": "~2.81.14",
15
+ "@xyo-network/archivist-wrapper": "~2.81.14",
16
+ "@xyo-network/boundwitness-builder": "~2.81.14",
17
+ "@xyo-network/boundwitness-model": "~2.81.14",
18
+ "@xyo-network/diviner-boundwitness-model": "~2.81.14",
19
+ "@xyo-network/diviner-indexing-model": "~2.81.14",
20
+ "@xyo-network/diviner-model": "~2.81.14",
21
+ "@xyo-network/diviner-wrapper": "~2.81.14",
22
+ "@xyo-network/module-model": "~2.81.14",
23
+ "@xyo-network/payload-builder": "~2.81.14",
24
+ "@xyo-network/payload-model": "~2.81.14"
25
25
  },
26
26
  "devDependencies": {
27
27
  "@xylabs/ts-scripts-yarn3": "^3.2.10",
@@ -68,5 +68,5 @@
68
68
  "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
69
69
  },
70
70
  "sideEffects": false,
71
- "version": "2.81.13"
71
+ "version": "2.81.14"
72
72
  }
package/src/Diviner.ts CHANGED
@@ -218,7 +218,7 @@ export class IndexingDiviner<
218
218
  this._pollId = setTimeout(async () => {
219
219
  try {
220
220
  await Promise.resolve()
221
- //await this.backgroundDivine()
221
+ await this.backgroundDivine()
222
222
  } catch (e) {
223
223
  console.log(e)
224
224
  } finally {