@xyo-network/diviner-indexing-memory 2.88.0 → 2.88.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/Diviner.d.cts.map +1 -1
- package/dist/browser/Diviner.d.mts.map +1 -1
- package/dist/browser/Diviner.d.ts.map +1 -1
- package/dist/browser/index.cjs +3 -2
- package/dist/browser/index.cjs.map +1 -1
- package/dist/browser/index.js +3 -2
- package/dist/browser/index.js.map +1 -1
- package/dist/node/Diviner.d.cts.map +1 -1
- package/dist/node/Diviner.d.mts.map +1 -1
- package/dist/node/Diviner.d.ts.map +1 -1
- package/dist/node/index.cjs +3 -2
- package/dist/node/index.cjs.map +1 -1
- package/dist/node/index.js +3 -2
- package/dist/node/index.js.map +1 -1
- package/package.json +16 -15
- package/src/Diviner.ts +4 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
package/dist/browser/index.cjs
CHANGED
|
@@ -27,6 +27,7 @@ module.exports = __toCommonJS(src_exports);
|
|
|
27
27
|
|
|
28
28
|
// src/Diviner.ts
|
|
29
29
|
var import_assert = require("@xylabs/assert");
|
|
30
|
+
var import_timer = require("@xylabs/timer");
|
|
30
31
|
var import_archivist_wrapper = require("@xyo-network/archivist-wrapper");
|
|
31
32
|
var import_boundwitness_builder = require("@xyo-network/boundwitness-builder");
|
|
32
33
|
var import_boundwitness_model = require("@xyo-network/boundwitness-model");
|
|
@@ -221,7 +222,7 @@ var IndexingDiviner = class extends import_diviner_abstract.AbstractDiviner {
|
|
|
221
222
|
* specified by the `config.pollFrequency`
|
|
222
223
|
*/
|
|
223
224
|
poll() {
|
|
224
|
-
this._pollId =
|
|
225
|
+
this._pollId = (0, import_timer.setTimeoutEx)(async () => {
|
|
225
226
|
try {
|
|
226
227
|
await Promise.resolve();
|
|
227
228
|
await this.backgroundDivine();
|
|
@@ -229,7 +230,7 @@ var IndexingDiviner = class extends import_diviner_abstract.AbstractDiviner {
|
|
|
229
230
|
console.log(e);
|
|
230
231
|
} finally {
|
|
231
232
|
if (this._pollId)
|
|
232
|
-
|
|
233
|
+
(0, import_timer.clearTimeoutEx)(this._pollId);
|
|
233
234
|
this._pollId = void 0;
|
|
234
235
|
this.poll();
|
|
235
236
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,wCAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,YAAiB;AAC1G,WAAOK,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,sBAA2B;AACpH,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,oBAAgBT,wBACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,iBAAsB;AAC/G,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,wCAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,WAAW,YAAA;AACxB,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAASyF,uBAAa,KAAKzF,OAAO;AAC3C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;MADZqG,qCAAAA;GACYrG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeout","e","console","log","creatableModule"]}
|
|
1
|
+
{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,mBAA6C;AAC7C,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,wCAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,YAAiB;AAC1G,WAAOK,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,sBAA2B;AACpH,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,oBAAgBT,wBACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,WAAOC,wBAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,iBAAsB;AAC/G,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,wCAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,cAAU0F,2BAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,2CAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;MADZsG,qCAAAA;GACYtG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","creatableModule"]}
|
package/dist/browser/index.js
CHANGED
|
@@ -3,6 +3,7 @@ var __name = (target, value) => __defProp(target, "name", { value, configurable:
|
|
|
3
3
|
|
|
4
4
|
// src/Diviner.ts
|
|
5
5
|
import { assertEx } from "@xylabs/assert";
|
|
6
|
+
import { clearTimeoutEx, setTimeoutEx } from "@xylabs/timer";
|
|
6
7
|
import { ArchivistWrapper } from "@xyo-network/archivist-wrapper";
|
|
7
8
|
import { BoundWitnessBuilder } from "@xyo-network/boundwitness-builder";
|
|
8
9
|
import { isBoundWitness } from "@xyo-network/boundwitness-model";
|
|
@@ -197,7 +198,7 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
197
198
|
* specified by the `config.pollFrequency`
|
|
198
199
|
*/
|
|
199
200
|
poll() {
|
|
200
|
-
this._pollId =
|
|
201
|
+
this._pollId = setTimeoutEx(async () => {
|
|
201
202
|
try {
|
|
202
203
|
await Promise.resolve();
|
|
203
204
|
await this.backgroundDivine();
|
|
@@ -205,7 +206,7 @@ var IndexingDiviner = class extends AbstractDiviner {
|
|
|
205
206
|
console.log(e);
|
|
206
207
|
} finally {
|
|
207
208
|
if (this._pollId)
|
|
208
|
-
|
|
209
|
+
clearTimeoutEx(this._pollId);
|
|
209
210
|
this._pollId = void 0;
|
|
210
211
|
this.poll();
|
|
211
212
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAA4BC,yBAAyB;AAC/E,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,gBAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,aAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,cAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,YAAiB;AAC1G,WAAOK,iBAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,sBAA2B;AACpH,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,gBAAgBT,SACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,iBAAsB;AAC/G,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,eAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,oBAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,aAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,WAAW,YAAA;AACxB,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAASyF,uBAAa,KAAKzF,OAAO;AAC3C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;EADZqG,gBAAAA;GACYrG,eAAAA;","names":["assertEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeout","e","console","log","creatableModule"]}
|
|
1
|
+
{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,gBAAgBC,oBAAoB;AAC7C,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAA4BC,yBAAyB;AAC/E,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,kBAAN,cAKGC,gBAAAA;SAAAA;;;EACR,OAAyBC,qBAAqB;EAC9C,OAAyBC,gBAA0B;IAACC;IAA6BC;;EAEzEC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,aAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,cAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;AAExE,QAAIA,UAAUY,MAAMC,WAAW,KAAKzB,YAAYwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;AACvD,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQrB,WAAW,MAAM,GAAGjC,UAAAA,gBAA0BsD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,YAAiB;AAC1G,WAAOK,iBAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;AACjE,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQQ,qBAAqB,MAAM,GAAG9D,UAAAA,gBAA0BsD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,sBAA2B;AACpH,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;AACvE,UAAMC,gBAAgBT,SACpB,KAAK9C,QAAQwD,wBAAwBF,SAAAA,GACrC,MAAM,GAAGhE,UAAAA,sCAAgDgE,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGzD,UAAAA,kDAA4DgE,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;AAC5D,UAAMC,OAAOC,SAAS,KAAK9C,SAAS4C,KAAAA,GAAQc,gBAAgB,MAAM,GAAGpE,UAAAA,gBAA0BsD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGvD,UAAAA,uBAAiCsD,KAAAA,iBAAsB;AAC/G,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,eAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAUN,aAAaJ,kBAAkBU,MAAMH,KAAAA,MAAWN,oBAAoBG,aAAaO,eAAeD,MAAMH,KAAAA,IAASE,MAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,aAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,aAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,yBAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA/MaV,kBAAAA,aAAAA;EADZsG,gBAAAA;GACYtG,eAAAA;","names":["assertEx","clearTimeoutEx","setTimeoutEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","creatableModule"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"Diviner.d.ts","sourceRoot":"","sources":["../../src/Diviner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AAGjE,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAE/D,OAAO,EACL,qBAAqB,EAErB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,EACrB,MAAM,qCAAqC,CAAA;AAC5C,OAAO,EAA0C,aAAa,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAkC,WAAW,EAAqB,MAAM,2BAA2B,CAAA;AAE1G,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAEpD,MAAM,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAA;AAExD,MAAM,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,qBAAqB,EAAE,cAAc,CAAC,CAAA;AAI9E,qBACa,eAAe,CAC1B,OAAO,SAAS,qBAAqB,GAAG,qBAAqB,EAC7D,GAAG,SAAS,OAAO,GAAG,OAAO,EAC7B,IAAI,SAAS,OAAO,GAAG,OAAO,EAC9B,UAAU,SAAS,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,sBAAsB,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CACxI,SAAQ,eAAe,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC;IACvD,gBAAyB,kBAAkB,SAAQ;IACnD,gBAAyB,aAAa,EAAE,MAAM,EAAE,CAAqD;IAErG,OAAO,CAAC,UAAU,CAAC,CAAmC;IACtD,OAAO,CAAC,OAAO,CAAC,CAAQ;IAExB,IAAI,mBAAmB,WAEtB;IAED,IAAI,aAAa,WAEhB;IAED;;;OAGG;IACH,SAAS,CAAC,gBAAgB,QAAa,QAAQ,IAAI,CAAC,CAmBnD;IAED;;;;;;OAMG;cACa,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,oBAAoB,CAAC;cAS/C,aAAa,CAAC,QAAQ,GAAE,GAAG,EAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAsB7E;;;;OAIG;cACa,oBAAoB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMvD;;;;OAIG;cACa,8BAA8B,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAMjE;;;;OAIG;cACa,uBAAuB,CAAC,SAAS,EAAE,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IASvE;;;;OAIG;cACa,yBAAyB,CAAC,KAAK,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAM5D;;;OAGG;cACa,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,oBAAoB,CAAC,GAAG,SAAS,CAAC;cAyC9D,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC;cAMhC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAQrF;;;OAGG;IACH,OAAO,CAAC,IAAI;CAcb"}
|
package/dist/node/index.cjs
CHANGED
|
@@ -32,6 +32,7 @@ module.exports = __toCommonJS(src_exports);
|
|
|
32
32
|
|
|
33
33
|
// src/Diviner.ts
|
|
34
34
|
var import_assert = require("@xylabs/assert");
|
|
35
|
+
var import_timer = require("@xylabs/timer");
|
|
35
36
|
var import_archivist_wrapper = require("@xyo-network/archivist-wrapper");
|
|
36
37
|
var import_boundwitness_builder = require("@xyo-network/boundwitness-builder");
|
|
37
38
|
var import_boundwitness_model = require("@xyo-network/boundwitness-model");
|
|
@@ -226,7 +227,7 @@ var _IndexingDiviner = class _IndexingDiviner extends import_diviner_abstract.Ab
|
|
|
226
227
|
* specified by the `config.pollFrequency`
|
|
227
228
|
*/
|
|
228
229
|
poll() {
|
|
229
|
-
this._pollId =
|
|
230
|
+
this._pollId = (0, import_timer.setTimeoutEx)(async () => {
|
|
230
231
|
try {
|
|
231
232
|
await Promise.resolve();
|
|
232
233
|
await this.backgroundDivine();
|
|
@@ -234,7 +235,7 @@ var _IndexingDiviner = class _IndexingDiviner extends import_diviner_abstract.Ab
|
|
|
234
235
|
console.log(e);
|
|
235
236
|
} finally {
|
|
236
237
|
if (this._pollId)
|
|
237
|
-
|
|
238
|
+
(0, import_timer.clearTimeoutEx)(this._pollId);
|
|
238
239
|
this._pollId = void 0;
|
|
239
240
|
this.poll();
|
|
240
241
|
}
|
package/dist/node/index.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,wCAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;;AACvD,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBrB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BkD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,YAAiB;AAC1G,WAAOK,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;;AACjE,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG1D,UAAAA,gBAA0BkD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,sBAA2B;AACpH,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;;AACvE,UAAMC,oBAAgBT,yBACpB,gBAAK9C,WAAL,mBAAawD,0BAAb,mBAAqCF,YACrC,MAAM,GAAG5D,UAAAA,sCAAgD4D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGrD,UAAAA,kDAA4D4D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;;AAC5D,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGhE,UAAAA,gBAA0BkD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,iBAAsB;AAC/G,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,wCAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,WAAW,YAAA;AACxB,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAASyF,uBAAa,KAAKzF,OAAO;AAC3C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMciG,sBAAqB;AAC9C,cAPWjG,kBAOckG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMpG,kBAAN;AAAMA,kBAAAA,aAAAA;MADZqG,qCAAAA;GACYrG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeout","e","console","log","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
|
|
1
|
+
{"version":3,"sources":["../../src/index.ts","../../src/Diviner.ts"],"sourcesContent":["export * from './Diviner'\n","import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;;;;;ACAA,oBAAyB;AACzB,mBAA6C;AAC7C,+BAAiC;AACjC,kCAAoC;AACpC,gCAA+B;AAC/B,8BAAgC;AAChC,wCAAgF;AAChF,oCAMO;AACP,2BAA8F;AAC9F,6BAA+B;AAC/B,0BAA+E;AAC/E,6BAA+B;;;;;;;;;;;;AAO/B,IAAMA,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,wCAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,iCAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,KAACH,mCAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,gDAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;;AACvD,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBrB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BkD,KAAAA,0BAA+B;AACzH,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,YAAiB;AAC1G,WAAOK,0CAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;;AACjE,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG1D,UAAAA,gBAA0BkD,KAAAA,oCAAyC;AAC7I,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,sBAA2B;AACpH,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;;AACvE,UAAMC,oBAAgBT,yBACpB,gBAAK9C,WAAL,mBAAawD,0BAAb,mBAAqCF,YACrC,MAAM,GAAG5D,UAAAA,sCAAgD4D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,eAAOT,4BAASW,wCAAkBV,GAAAA,GAAM,MAAM,GAAGrD,UAAAA,kDAA4D4D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;;AAC5D,UAAMC,WAAOC,yBAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGhE,UAAAA,gBAA0BkD,KAAAA,+BAAoC;AACnI,UAAMG,UAAMD,wBAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,iBAAsB;AAC/G,WAAOS,sCAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,sCAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,cAAIG,0CAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,wCAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,iCAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,cAAU0F,2BAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,2CAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMckG,sBAAqB;AAC9C,cAPWlG,kBAOcmG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMrG,kBAAN;AAAMA,kBAAAA,aAAAA;MADZsG,qCAAAA;GACYtG,eAAAA;","names":["moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
|
package/dist/node/index.js
CHANGED
|
@@ -8,6 +8,7 @@ var __publicField = (obj, key, value) => {
|
|
|
8
8
|
|
|
9
9
|
// src/Diviner.ts
|
|
10
10
|
import { assertEx } from "@xylabs/assert";
|
|
11
|
+
import { clearTimeoutEx, setTimeoutEx } from "@xylabs/timer";
|
|
11
12
|
import { ArchivistWrapper } from "@xyo-network/archivist-wrapper";
|
|
12
13
|
import { BoundWitnessBuilder } from "@xyo-network/boundwitness-builder";
|
|
13
14
|
import { isBoundWitness } from "@xyo-network/boundwitness-model";
|
|
@@ -202,7 +203,7 @@ var _IndexingDiviner = class _IndexingDiviner extends AbstractDiviner {
|
|
|
202
203
|
* specified by the `config.pollFrequency`
|
|
203
204
|
*/
|
|
204
205
|
poll() {
|
|
205
|
-
this._pollId =
|
|
206
|
+
this._pollId = setTimeoutEx(async () => {
|
|
206
207
|
try {
|
|
207
208
|
await Promise.resolve();
|
|
208
209
|
await this.backgroundDivine();
|
|
@@ -210,7 +211,7 @@ var _IndexingDiviner = class _IndexingDiviner extends AbstractDiviner {
|
|
|
210
211
|
console.log(e);
|
|
211
212
|
} finally {
|
|
212
213
|
if (this._pollId)
|
|
213
|
-
|
|
214
|
+
clearTimeoutEx(this._pollId);
|
|
214
215
|
this._pollId = void 0;
|
|
215
216
|
this.poll();
|
|
216
217
|
}
|
package/dist/node/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string | number | NodeJS.Timeout\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeout(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeout(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAA4BC,yBAAyB;AAC/E,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,gBAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,aAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,cAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;;AACvD,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBrB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BkD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,YAAiB;AAC1G,WAAOK,iBAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;;AACjE,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG1D,UAAAA,gBAA0BkD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,sBAA2B;AACpH,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;;AACvE,UAAMC,gBAAgBT,UACpB,gBAAK9C,WAAL,mBAAawD,0BAAb,mBAAqCF,YACrC,MAAM,GAAG5D,UAAAA,sCAAgD4D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGrD,UAAAA,kDAA4D4D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;;AAC5D,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGhE,UAAAA,gBAA0BkD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,iBAAsB;AAC/G,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,eAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,oBAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,aAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,WAAW,YAAA;AACxB,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAASyF,uBAAa,KAAKzF,OAAO;AAC3C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMciG,sBAAqB;AAC9C,cAPWjG,kBAOckG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMpG,kBAAN;AAAMA,kBAAAA,aAAAA;EADZqG,gBAAAA;GACYrG,eAAAA;","names":["assertEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeout","e","console","log","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
|
|
1
|
+
{"version":3,"sources":["../../src/Diviner.ts"],"sourcesContent":["import { assertEx } from '@xylabs/assert'\nimport { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'\nimport { ArchivistWrapper } from '@xyo-network/archivist-wrapper'\nimport { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'\nimport { isBoundWitness } from '@xyo-network/boundwitness-model'\nimport { AbstractDiviner } from '@xyo-network/diviner-abstract'\nimport { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'\nimport {\n IndexingDivinerConfig,\n IndexingDivinerConfigSchema,\n IndexingDivinerParams,\n IndexingDivinerStage,\n IndexingDivinerState,\n} from '@xyo-network/diviner-indexing-model'\nimport { asDivinerInstance, DivinerConfigSchema, DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'\nimport { DivinerWrapper } from '@xyo-network/diviner-wrapper'\nimport { creatableModule, isModuleState, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'\nimport { PayloadBuilder } from '@xyo-network/payload-builder'\nimport { Payload } from '@xyo-network/payload-model'\n\nexport type ConfigStoreKey = 'indexStore' | 'stateStore'\n\nexport type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>\n\nconst moduleName = 'IndexingDiviner'\n\n@creatableModule<IndexingDiviner>()\nexport class IndexingDiviner<\n TParams extends IndexingDivinerParams = IndexingDivinerParams,\n TIn extends Payload = Payload,\n TOut extends Payload = Payload,\n TEventData extends DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut> = DivinerModuleEventData<DivinerModule<TParams>, TIn, TOut>,\n> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {\n static override readonly allowRandomAccount = false\n static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]\n\n private _lastState?: ModuleState<IndexingDivinerState>\n private _pollId?: string\n\n get payloadDivinerLimit() {\n return this.config.payloadDivinerLimit ?? 1000\n }\n\n get pollFrequency() {\n return this.config.pollFrequency ?? 10_000\n }\n\n /**\n * Works via batched iteration of the source archivist to populate the index.\n * @returns A promise that resolves when the background process is complete\n */\n protected backgroundDivine = async (): Promise<void> => {\n // Load last state\n const lastState = await this.retrieveState()\n // Get next batch of results\n const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')\n const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()\n // Filter next state out from results\n const nextState = results.find(isModuleState<IndexingDivinerState>)\n const indexCandidates = results.filter((x) => !isModuleState(x))\n // Transform candidates to indexes\n const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')\n const indexes = await toIndexTransformDiviner.divine(indexCandidates)\n // Insert index results\n const indexArchivist = await this.getArchivistForStore('indexStore')\n await indexArchivist.insert(indexes)\n // Update state\n if (nextState) {\n await this.commitState(nextState)\n }\n }\n\n /**\n * Commit the internal state of the Diviner process. This is similar\n * to a transaction completion in a database and should only be called\n * when results have been successfully persisted to the appropriate\n * external stores.\n * @param nextState The state to commit\n */\n protected async commitState(nextState: ModuleState<IndexingDivinerState>) {\n // Don't commit state if no state has changed\n if (nextState.state.offset === this._lastState?.state.offset) return\n this._lastState = nextState\n const archivist = await this.getArchivistForStore('stateStore')\n const [bw] = await new BoundWitnessBuilder().payload(nextState).witness(this.account).build()\n await archivist.insert([bw, nextState])\n }\n\n protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {\n const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')\n const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')\n const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')\n const results = (\n await Promise.all(\n payloads.map(async (payload) => {\n const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])\n // Divine the results\n const indexedResults = await indexPayloadDiviner.divine(indexQuery)\n // Transform the results to the response shape\n const response = await Promise.all(\n indexedResults.flat().map((indexedResult) => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),\n )\n return response.flat()\n }),\n )\n ).flat()\n // TODO: Infer this type over casting to this type\n return results as TOut[]\n }\n\n /**\n * Retrieves the archivist for the specified store\n * @param store The store to retrieve the archivist for\n * @returns The archivist for the specified store\n */\n protected async getArchivistForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist`)\n return ArchivistWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the BoundWitness Diviner for the specified store\n * @param store The store to retrieve the BoundWitness Diviner for\n * @returns The BoundWitness Diviner for the specified store\n */\n protected async getBoundWitnessDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Gets the Diviner for the supplied Indexing Diviner stage\n * @param transform The Indexing Diviner stage\n * @returns The diviner corresponding to the Indexing Diviner stage\n */\n protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {\n const nameOrAddress = assertEx(\n this.config?.indexingDivinerStages?.[transform],\n () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,\n )\n const mod = await this.resolve(nameOrAddress)\n return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`)\n }\n\n /**\n * Retrieves the Payload Diviner for the specified store\n * @param store The store to retrieve the Payload Diviner for\n * @returns The Payload Diviner for the specified store\n */\n protected async getPayloadDivinerForStore(store: ConfigStore) {\n const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)\n const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner`)\n return DivinerWrapper.wrap(mod, this.account)\n }\n\n /**\n * Retrieves the last state of the Diviner process. Used to recover state after\n * preemptions, reboots, etc.\n */\n protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {\n if (this._lastState) return this._lastState\n let hash: string = ''\n const diviner = await this.getBoundWitnessDivinerForStore('stateStore')\n const query = await new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })\n .fields({\n address: this.account.address,\n limit: 1,\n offset: 0,\n order: 'desc',\n payload_schemas: [ModuleStateSchema],\n })\n .build()\n const boundWitnesses = await diviner.divine([query])\n if (boundWitnesses.length > 0) {\n const boundWitness = boundWitnesses[0]\n if (isBoundWitness(boundWitness)) {\n // Find the index for this address in the BoundWitness that is a ModuleState\n hash = boundWitness.addresses\n .map((address, index) => ({ address, index }))\n .filter(({ address }) => address === this.account.address)\n // eslint-disable-next-line unicorn/no-array-reduce\n .reduce(\n (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),\n '',\n )\n }\n }\n\n // If we able to located the last state\n if (hash) {\n // Get last state\n const archivist = await this.getArchivistForStore('stateStore')\n const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)\n if (payload) {\n return payload\n }\n }\n return undefined\n }\n\n protected override async startHandler(): Promise<boolean> {\n await super.startHandler()\n this.poll()\n return true\n }\n\n protected override async stopHandler(_timeout?: number | undefined): Promise<boolean> {\n if (this._pollId) {\n clearTimeout(this._pollId)\n this._pollId = undefined\n }\n return await super.stopHandler()\n }\n\n /**\n * Runs the background divine process on a loop with a delay\n * specified by the `config.pollFrequency`\n */\n private poll() {\n this._pollId = setTimeoutEx(async () => {\n try {\n await Promise.resolve()\n await this.backgroundDivine()\n } catch (e) {\n console.log(e)\n } finally {\n if (this._pollId) clearTimeoutEx(this._pollId)\n this._pollId = undefined\n this.poll()\n }\n }, this.pollFrequency)\n }\n}\n"],"mappings":";;;;;;;;;AAAA,SAASA,gBAAgB;AACzB,SAASC,gBAAgBC,oBAAoB;AAC7C,SAASC,wBAAwB;AACjC,SAASC,2BAA2B;AACpC,SAASC,sBAAsB;AAC/B,SAASC,uBAAuB;AAChC,SAA0CC,sCAAsC;AAChF,SAEEC,mCAIK;AACP,SAASC,mBAAmBC,2BAAkE;AAC9F,SAASC,sBAAsB;AAC/B,SAASC,iBAAiBC,eAA4BC,yBAAyB;AAC/E,SAASC,sBAAsB;;;;;;;;;;;;AAO/B,IAAMC,aAAa;AAGZ,IAAMC,mBAAN,MAAMA,yBAKHC,gBAAAA;EAIAC;EACAC;EAER,IAAIC,sBAAsB;AACxB,WAAO,KAAKC,OAAOD,uBAAuB;EAC5C;EAEA,IAAIE,gBAAgB;AAClB,WAAO,KAAKD,OAAOC,iBAAiB;EACtC;;;;;EAMUC,mBAAmB,YAAA;AAE3B,UAAMC,YAAY,MAAM,KAAKC,cAAa;AAE1C,UAAMC,wBAAwB,MAAM,KAAKC,wBAAwB,8BAAA;AACjE,UAAMC,UAAUJ,YAAY,MAAME,sBAAsBG,OAAO;MAACL;KAAU,IAAI,MAAME,sBAAsBG,OAAM;AAEhH,UAAMC,YAAYF,QAAQG,KAAKC,aAAAA;AAC/B,UAAMC,kBAAkBL,QAAQM,OAAO,CAACC,MAAM,CAACH,cAAcG,CAAAA,CAAAA;AAE7D,UAAMC,0BAA0B,MAAM,KAAKT,wBAAwB,8BAAA;AACnE,UAAMU,UAAU,MAAMD,wBAAwBP,OAAOI,eAAAA;AAErD,UAAMK,iBAAiB,MAAM,KAAKC,qBAAqB,YAAA;AACvD,UAAMD,eAAeE,OAAOH,OAAAA;AAE5B,QAAIP,WAAW;AACb,YAAM,KAAKW,YAAYX,SAAAA;IACzB;EACF;;;;;;;;EASA,MAAgBW,YAAYX,WAA8C;;AAExE,QAAIA,UAAUY,MAAMC,aAAW,UAAKzB,eAAL,mBAAiBwB,MAAMC;AAAQ;AAC9D,SAAKzB,aAAaY;AAClB,UAAMc,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,UAAM,CAACM,EAAAA,IAAM,MAAM,IAAIC,oBAAAA,EAAsBC,QAAQjB,SAAAA,EAAWkB,QAAQ,KAAKC,OAAO,EAAEC,MAAK;AAC3F,UAAMN,UAAUJ,OAAO;MAACK;MAAIf;KAAU;EACxC;EAEA,MAAyBqB,cAAcC,WAAkB,CAAA,GAAqB;AAC5E,UAAMC,sBAAsB,MAAM,KAAKC,0BAA0B,YAAA;AACjE,UAAMC,kCAAkC,MAAM,KAAK5B,wBAAwB,iCAAA;AAC3E,UAAM6B,kDAAkD,MAAM,KAAK7B,wBAAwB,iDAAA;AAC3F,UAAMC,WACJ,MAAM6B,QAAQC,IACZN,SAASO,IAAI,OAAOZ,YAAAA;AAClB,YAAMa,aAAa,MAAML,gCAAgC1B,OAAO;QAACkB;OAAQ;AAEzE,YAAMc,iBAAiB,MAAMR,oBAAoBxB,OAAO+B,UAAAA;AAExD,YAAME,WAAW,MAAML,QAAQC,IAC7BG,eAAeE,KAAI,EAAGJ,IAAI,CAACK,kBAAkBR,gDAAgD3B,OAAO;QAACkB;QAASiB;OAAc,CAAA,CAAA;AAE9H,aAAOF,SAASC,KAAI;IACtB,CAAA,CAAA,GAEFA,KAAI;AAEN,WAAOnC;EACT;;;;;;EAOA,MAAgBW,qBAAqB0B,OAAoB;;AACvD,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBrB,WAAW,MAAM,GAAG7B,UAAAA,gBAA0BkD,KAAAA,0BAA+B;AACzH,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,YAAiB;AAC1G,WAAOK,iBAAiBC,KAAKH,KAAK,KAAKnB,OAAO;EAChD;;;;;;EAOA,MAAgBuB,+BAA+BP,OAAoB;;AACjE,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBQ,qBAAqB,MAAM,GAAG1D,UAAAA,gBAA0BkD,KAAAA,oCAAyC;AAC7I,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,sBAA2B;AACpH,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;;EAOA,MAAgBtB,wBAAwBgD,WAAiC;;AACvE,UAAMC,gBAAgBT,UACpB,gBAAK9C,WAAL,mBAAawD,0BAAb,mBAAqCF,YACrC,MAAM,GAAG5D,UAAAA,sCAAgD4D,SAAAA,gBAAyB;AAEpF,UAAMP,MAAM,MAAM,KAAKC,QAAQO,aAAAA;AAC/B,WAAOT,SAASW,kBAAkBV,GAAAA,GAAM,MAAM,GAAGrD,UAAAA,kDAA4D4D,SAAAA,EAAW;EAC1H;;;;;;EAOA,MAAgBrB,0BAA0BW,OAAoB;;AAC5D,UAAMC,OAAOC,UAAS,gBAAK9C,WAAL,mBAAc4C,WAAd,mBAAsBc,gBAAgB,MAAM,GAAGhE,UAAAA,gBAA0BkD,KAAAA,+BAAoC;AACnI,UAAMG,MAAMD,SAAS,MAAM,KAAKE,QAAQH,IAAAA,GAAO,MAAM,GAAGnD,UAAAA,uBAAiCkD,KAAAA,iBAAsB;AAC/G,WAAOS,eAAeH,KAAKH,KAAK,KAAKnB,OAAO;EAC9C;;;;;EAMA,MAAgBxB,gBAAwE;AACtF,QAAI,KAAKP;AAAY,aAAO,KAAKA;AACjC,QAAI8D,OAAe;AACnB,UAAMC,UAAU,MAAM,KAAKT,+BAA+B,YAAA;AAC1D,UAAMU,QAAQ,MAAM,IAAIC,eAAgD;MAAEC,QAAQC;IAA+B,CAAA,EAC9GC,OAAO;MACNC,SAAS,KAAKtC,QAAQsC;MACtBC,OAAO;MACP7C,QAAQ;MACR8C,OAAO;MACPC,iBAAiB;QAACC;;IACpB,CAAA,EACCzC,MAAK;AACR,UAAM0C,iBAAiB,MAAMX,QAAQpD,OAAO;MAACqD;KAAM;AACnD,QAAIU,eAAeC,SAAS,GAAG;AAC7B,YAAMC,eAAeF,eAAe,CAAA;AACpC,UAAIG,eAAeD,YAAAA,GAAe;AAEhCd,eAAOc,aAAaE,UACjBrC,IAAI,CAAC4B,SAASU,WAAW;UAAEV;UAASU;QAAM,EAAA,EAC1C/D,OAAO,CAAC,EAAEqD,QAAO,MAAOA,YAAY,KAAKtC,QAAQsC,OAAO,EAExDW,OACC,CAACC,MAAMC,SAAAA;;AAAUN,qCAAaJ,oBAAbI,mBAA+BM,6BAAMH,YAAWN,oBAAoBG,aAAaO,eAAeD,6BAAMH,KAAAA,IAASE;WAChI,EAAA;MAEN;IACF;AAGA,QAAInB,MAAM;AAER,YAAMpC,YAAY,MAAM,KAAKL,qBAAqB,YAAA;AAClD,YAAMQ,WAAW,MAAMH,UAAU0D,IAAI;QAACtB;OAAK,GAAGjD,KAAKC,aAAAA;AACnD,UAAIe,SAAS;AACX,eAAOA;MACT;IACF;AACA,WAAOwD;EACT;EAEA,MAAyBC,eAAiC;AACxD,UAAM,MAAMA,aAAAA;AACZ,SAAKC,KAAI;AACT,WAAO;EACT;EAEA,MAAyBC,YAAYC,UAAiD;AACpF,QAAI,KAAKxF,SAAS;AAChByF,mBAAa,KAAKzF,OAAO;AACzB,WAAKA,UAAUoF;IACjB;AACA,WAAO,MAAM,MAAMG,YAAAA;EACrB;;;;;EAMQD,OAAO;AACb,SAAKtF,UAAU0F,aAAa,YAAA;AAC1B,UAAI;AACF,cAAMpD,QAAQY,QAAO;AACrB,cAAM,KAAK9C,iBAAgB;MAC7B,SAASuF,GAAG;AACVC,gBAAQC,IAAIF,CAAAA;MACd,UAAA;AACE,YAAI,KAAK3F;AAAS8F,yBAAe,KAAK9F,OAAO;AAC7C,aAAKA,UAAUoF;AACf,aAAKE,KAAI;MACX;IACF,GAAG,KAAKnF,aAAa;EACvB;AACF;AA1MUL;AACR,cANWD,kBAMckG,sBAAqB;AAC9C,cAPWlG,kBAOcmG,iBAA0B;EAACC;EAA6BC;;AAP5E,IAAMrG,kBAAN;AAAMA,kBAAAA,aAAAA;EADZsG,gBAAAA;GACYtG,eAAAA;","names":["assertEx","clearTimeoutEx","setTimeoutEx","ArchivistWrapper","BoundWitnessBuilder","isBoundWitness","AbstractDiviner","BoundWitnessDivinerQuerySchema","IndexingDivinerConfigSchema","asDivinerInstance","DivinerConfigSchema","DivinerWrapper","creatableModule","isModuleState","ModuleStateSchema","PayloadBuilder","moduleName","IndexingDiviner","AbstractDiviner","_lastState","_pollId","payloadDivinerLimit","config","pollFrequency","backgroundDivine","lastState","retrieveState","indexCandidateDiviner","getIndexingDivinerStage","results","divine","nextState","find","isModuleState","indexCandidates","filter","x","toIndexTransformDiviner","indexes","indexArchivist","getArchivistForStore","insert","commitState","state","offset","archivist","bw","BoundWitnessBuilder","payload","witness","account","build","divineHandler","payloads","indexPayloadDiviner","getPayloadDivinerForStore","divinerQueryToIndexQueryDiviner","indexQueryResponseToDivinerQueryResponseDiviner","Promise","all","map","indexQuery","indexedResults","response","flat","indexedResult","store","name","assertEx","mod","resolve","ArchivistWrapper","wrap","getBoundWitnessDivinerForStore","boundWitnessDiviner","DivinerWrapper","transform","nameOrAddress","indexingDivinerStages","asDivinerInstance","payloadDiviner","hash","diviner","query","PayloadBuilder","schema","BoundWitnessDivinerQuerySchema","fields","address","limit","order","payload_schemas","ModuleStateSchema","boundWitnesses","length","boundWitness","isBoundWitness","addresses","index","reduce","prev","curr","payload_hashes","get","undefined","startHandler","poll","stopHandler","_timeout","clearTimeout","setTimeoutEx","e","console","log","clearTimeoutEx","allowRandomAccount","configSchemas","IndexingDivinerConfigSchema","DivinerConfigSchema","creatableModule"]}
|
package/package.json
CHANGED
|
@@ -10,22 +10,23 @@
|
|
|
10
10
|
"url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
|
-
"@xylabs/assert": "^2.13.
|
|
14
|
-
"@
|
|
15
|
-
"@xyo-network/
|
|
16
|
-
"@xyo-network/boundwitness-
|
|
17
|
-
"@xyo-network/
|
|
18
|
-
"@xyo-network/diviner-
|
|
19
|
-
"@xyo-network/diviner-
|
|
20
|
-
"@xyo-network/diviner-model": "~2.88.
|
|
21
|
-
"@xyo-network/diviner-
|
|
22
|
-
"@xyo-network/
|
|
23
|
-
"@xyo-network/
|
|
24
|
-
"@xyo-network/payload-
|
|
13
|
+
"@xylabs/assert": "^2.13.25",
|
|
14
|
+
"@xylabs/timer": "^2.13.25",
|
|
15
|
+
"@xyo-network/archivist-wrapper": "~2.88.2",
|
|
16
|
+
"@xyo-network/boundwitness-builder": "~2.88.2",
|
|
17
|
+
"@xyo-network/boundwitness-model": "~2.88.2",
|
|
18
|
+
"@xyo-network/diviner-abstract": "~2.88.2",
|
|
19
|
+
"@xyo-network/diviner-boundwitness-model": "~2.88.2",
|
|
20
|
+
"@xyo-network/diviner-indexing-model": "~2.88.2",
|
|
21
|
+
"@xyo-network/diviner-model": "~2.88.2",
|
|
22
|
+
"@xyo-network/diviner-wrapper": "~2.88.2",
|
|
23
|
+
"@xyo-network/module-model": "~2.88.2",
|
|
24
|
+
"@xyo-network/payload-builder": "~2.88.2",
|
|
25
|
+
"@xyo-network/payload-model": "~2.88.2"
|
|
25
26
|
},
|
|
26
27
|
"devDependencies": {
|
|
27
|
-
"@xylabs/ts-scripts-yarn3": "^3.2.
|
|
28
|
-
"@xylabs/tsconfig": "^3.2.
|
|
28
|
+
"@xylabs/ts-scripts-yarn3": "^3.2.42",
|
|
29
|
+
"@xylabs/tsconfig": "^3.2.42",
|
|
29
30
|
"typescript": "^5.3.3"
|
|
30
31
|
},
|
|
31
32
|
"description": "Primary SDK for using XYO Protocol 2.0",
|
|
@@ -67,6 +68,6 @@
|
|
|
67
68
|
"url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js.git"
|
|
68
69
|
},
|
|
69
70
|
"sideEffects": false,
|
|
70
|
-
"version": "2.88.
|
|
71
|
+
"version": "2.88.2",
|
|
71
72
|
"type": "module"
|
|
72
73
|
}
|
package/src/Diviner.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { assertEx } from '@xylabs/assert'
|
|
2
|
+
import { clearTimeoutEx, setTimeoutEx } from '@xylabs/timer'
|
|
2
3
|
import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
|
|
3
4
|
import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
|
|
4
5
|
import { isBoundWitness } from '@xyo-network/boundwitness-model'
|
|
@@ -34,7 +35,7 @@ export class IndexingDiviner<
|
|
|
34
35
|
static override readonly configSchemas: string[] = [IndexingDivinerConfigSchema, DivinerConfigSchema]
|
|
35
36
|
|
|
36
37
|
private _lastState?: ModuleState<IndexingDivinerState>
|
|
37
|
-
private _pollId?: string
|
|
38
|
+
private _pollId?: string
|
|
38
39
|
|
|
39
40
|
get payloadDivinerLimit() {
|
|
40
41
|
return this.config.payloadDivinerLimit ?? 1000
|
|
@@ -218,14 +219,14 @@ export class IndexingDiviner<
|
|
|
218
219
|
* specified by the `config.pollFrequency`
|
|
219
220
|
*/
|
|
220
221
|
private poll() {
|
|
221
|
-
this._pollId =
|
|
222
|
+
this._pollId = setTimeoutEx(async () => {
|
|
222
223
|
try {
|
|
223
224
|
await Promise.resolve()
|
|
224
225
|
await this.backgroundDivine()
|
|
225
226
|
} catch (e) {
|
|
226
227
|
console.log(e)
|
|
227
228
|
} finally {
|
|
228
|
-
if (this._pollId)
|
|
229
|
+
if (this._pollId) clearTimeoutEx(this._pollId)
|
|
229
230
|
this._pollId = undefined
|
|
230
231
|
this.poll()
|
|
231
232
|
}
|