@xyo-network/diviner-indexing-memory 5.3.22 → 5.3.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/diviner-indexing-memory",
-  "version": "5.3.22",
+  "version": "5.3.24",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -30,53 +30,49 @@
   "types": "dist/neutral/index.d.ts",
   "files": [
     "dist",
-    "src",
     "!**/*.bench.*",
     "!**/*.spec.*",
     "!**/*.test.*",
     "README.md"
   ],
   "dependencies": {
-    "@xyo-network/archivist-wrapper": "~5.3.22",
-    "@xyo-network/boundwitness-builder": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-abstract": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-indexing-model": "~5.3.22",
-    "@xyo-network/diviner-model": "~5.3.22",
-    "@xyo-network/diviner-wrapper": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22"
+    "@xyo-network/boundwitness-model": "~5.3.24",
+    "@xyo-network/archivist-wrapper": "~5.3.24",
+    "@xyo-network/boundwitness-builder": "~5.3.24",
+    "@xyo-network/diviner-abstract": "~5.3.24",
+    "@xyo-network/diviner-boundwitness-model": "~5.3.24",
+    "@xyo-network/diviner-model": "~5.3.24",
+    "@xyo-network/module-model": "~5.3.24",
+    "@xyo-network/diviner-wrapper": "~5.3.24",
+    "@xyo-network/diviner-indexing-model": "~5.3.24",
+    "@xyo-network/payload-model": "~5.3.24",
+    "@xyo-network/payload-builder": "~5.3.24"
   },
   "devDependencies": {
     "@opentelemetry/api": "^1.9.1",
     "@types/node": "^25.5.0",
-    "@xylabs/sdk-js": "^5.0.91",
-    "@xylabs/ts-scripts-common": "~7.6.8",
-    "@xylabs/ts-scripts-yarn3": "~7.6.8",
-    "@xylabs/tsconfig": "~7.6.8",
-    "@xyo-network/archivist-wrapper": "~5.3.22",
-    "@xyo-network/boundwitness-builder": "~5.3.22",
-    "@xyo-network/boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-abstract": "~5.3.22",
-    "@xyo-network/diviner-boundwitness-model": "~5.3.22",
-    "@xyo-network/diviner-indexing-model": "~5.3.22",
-    "@xyo-network/diviner-model": "~5.3.22",
-    "@xyo-network/diviner-wrapper": "~5.3.22",
-    "@xyo-network/module-model": "~5.3.22",
-    "@xyo-network/payload-builder": "~5.3.22",
-    "@xyo-network/payload-model": "~5.3.22",
+    "@xylabs/sdk-js": "^5.0.93",
+    "@xylabs/ts-scripts-common": "~7.6.16",
+    "@xylabs/ts-scripts-pnpm": "~7.6.16",
+    "@xylabs/tsconfig": "~7.6.16",
     "acorn": "^8.16.0",
     "axios": "^1.14.0",
-    "cosmiconfig": "^9.0.1",
-    "esbuild": "^0.27.4",
-    "eslint": "^10.1.0",
+    "esbuild": "^0.28.0",
     "ethers": "^6.16.0",
-    "rollup": "^4.60.1",
     "tslib": "^2.8.1",
     "typescript": "~5.9.3",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "@xyo-network/archivist-wrapper": "~5.3.24",
+    "@xyo-network/boundwitness-builder": "~5.3.24",
+    "@xyo-network/diviner-abstract": "~5.3.24",
+    "@xyo-network/diviner-boundwitness-model": "~5.3.24",
+    "@xyo-network/boundwitness-model": "~5.3.24",
+    "@xyo-network/diviner-indexing-model": "~5.3.24",
+    "@xyo-network/diviner-wrapper": "~5.3.24",
+    "@xyo-network/diviner-model": "~5.3.24",
+    "@xyo-network/module-model": "~5.3.24",
+    "@xyo-network/payload-builder": "~5.3.24",
+    "@xyo-network/payload-model": "~5.3.24"
   },
   "peerDependencies": {
     "@xylabs/sdk-js": "^5",
@@ -87,4 +83,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/Diviner.ts

@@ -1,247 +0,0 @@
-import {
-  assertEx, clearTimeoutEx, Hash, isString, setTimeoutEx,
-} from '@xylabs/sdk-js'
-import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
-import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
-import { AbstractDiviner } from '@xyo-network/diviner-abstract'
-import { BoundWitnessDivinerQueryPayload, BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
-import {
-  IndexingDivinerConfig,
-  IndexingDivinerConfigSchema,
-  IndexingDivinerParams,
-  IndexingDivinerStage,
-  IndexingDivinerState,
-} from '@xyo-network/diviner-indexing-model'
-import {
-  asDivinerInstance, DivinerInstance, DivinerModuleEventData,
-} from '@xyo-network/diviner-model'
-import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
-import {
-  creatableModule, isModuleState, ModuleState, ModuleStateSchema,
-} from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
-import {
-  Payload, Schema, SequenceConstants, WithStorageMeta,
-} from '@xyo-network/payload-model'
-
-export type ConfigStoreKey = 'indexStore' | 'stateStore'
-
-export type ConfigStore = Extract<keyof IndexingDivinerConfig, ConfigStoreKey>
-
-const moduleName = 'IndexingDiviner'
-
-@creatableModule<IndexingDiviner>()
-export class IndexingDiviner<
-  TParams extends IndexingDivinerParams = IndexingDivinerParams,
-  TIn extends Payload = Payload,
-  TOut extends Payload = Payload,
-  TEventData extends DivinerModuleEventData<DivinerInstance<TParams, TIn, TOut>, TIn, TOut> = DivinerModuleEventData<
-    DivinerInstance<TParams, TIn, TOut>,
-    TIn,
-    TOut
-  >,
-> extends AbstractDiviner<TParams, TIn, TOut, TEventData> {
-  static override readonly allowRandomAccount = true
-  static override readonly configSchemas: Schema[] = [...super.configSchemas, IndexingDivinerConfigSchema]
-  static override readonly defaultConfigSchema: Schema = IndexingDivinerConfigSchema
-
-  private _lastState?: ModuleState<IndexingDivinerState>
-  private _pollId?: string
-
-  get payloadDivinerLimit() {
-    return this.config.payloadDivinerLimit ?? 1000
-  }
-
-  get pollFrequency() {
-    return this.config.pollFrequency ?? 10_000
-  }
-
-  /**
-   * Works via batched iteration of the source archivist to populate the index.
-   * @returns A promise that resolves when the background process is complete
-   */
-  protected backgroundDivine = async (): Promise<void> => {
-    // Load last state
-    const lastState = await this.retrieveState()
-    // Get next batch of results
-    const indexCandidateDiviner = await this.getIndexingDivinerStage('stateToIndexCandidateDiviner')
-    const results = lastState ? await indexCandidateDiviner.divine([lastState]) : await indexCandidateDiviner.divine()
-    // Filter next state out from results
-    const nextState = results.find(isModuleState<IndexingDivinerState>)
-    const indexCandidates = results.filter(x => !isModuleState(x))
-    // Transform candidates to indexes
-    const toIndexTransformDiviner = await this.getIndexingDivinerStage('indexCandidateToIndexDiviner')
-    const indexes = await toIndexTransformDiviner.divine(indexCandidates)
-    // Insert index results
-    const indexArchivist = await this.getArchivistForStore('indexStore')
-    await indexArchivist.insert(indexes)
-    // Update state
-    if (nextState) {
-      await this.commitState(nextState)
-    }
-  }
-
-  /**
-   * Commit the internal state of the Diviner process. This is similar
-   * to a transaction completion in a database and should only be called
-   * when results have been successfully persisted to the appropriate
-   * external stores.
-   * @param nextState The state to commit
-   */
-  protected async commitState(nextState: ModuleState<IndexingDivinerState>) {
-    // Don't commit state if no state has changed
-    if (nextState.state.cursor === this._lastState?.state.cursor) return
-    this._lastState = nextState
-    const archivist = await this.getArchivistForStore('stateStore')
-    const [bw] = await new BoundWitnessBuilder().payload(nextState).signer(this.account).build()
-    await archivist.insert([bw, nextState])
-  }
-
-  protected override async divineHandler(payloads: TIn[] = []): Promise<TOut[]> {
-    const indexPayloadDiviner = await this.getPayloadDivinerForStore('indexStore')
-    const divinerQueryToIndexQueryDiviner = await this.getIndexingDivinerStage('divinerQueryToIndexQueryDiviner')
-    const indexQueryResponseToDivinerQueryResponseDiviner = await this.getIndexingDivinerStage('indexQueryResponseToDivinerQueryResponseDiviner')
-    const results = (
-      await Promise.all(
-        payloads.map(async (payload) => {
-          const indexQuery = await divinerQueryToIndexQueryDiviner.divine([payload])
-          // Divine the results
-          const indexedResults = await indexPayloadDiviner.divine(indexQuery)
-          // Transform the results to the response shape
-          const response = await Promise.all(
-            indexedResults.flat().map(indexedResult => indexQueryResponseToDivinerQueryResponseDiviner.divine([payload, indexedResult])),
-          )
-          return response.flat()
-        }),
-      )
-    ).flat()
-    // TODO: Infer this type over casting to this type
-    return results as TOut[]
-  }
-
-  /**
-   * Retrieves the archivist for the specified store
-   * @param store The store to retrieve the archivist for
-   * @returns The archivist for the specified store
-   */
-  protected async getArchivistForStore(store: ConfigStore): Promise<ArchivistWrapper> {
-    const name = assertEx(this.config?.[store]?.archivist, () => `${moduleName}: Config for ${store}.archivist not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.archivist [${name}]`)
-    return ArchivistWrapper.wrap(mod, this.account)
-  }
-
-  /**
-   * Retrieves the BoundWitness Diviner for the specified store
-   * @param store The store to retrieve the BoundWitness Diviner for
-   * @returns The BoundWitness Diviner for the specified store
-   */
-  protected async getBoundWitnessDivinerForStore(store: ConfigStore) {
-    const name = assertEx(this.config?.[store]?.boundWitnessDiviner, () => `${moduleName}: Config for ${store}.boundWitnessDiviner not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.boundWitnessDiviner [${name}]`)
-    return DivinerWrapper.wrap(mod, this.account)
-  }
-
-  /**
-   * Gets the Diviner for the supplied Indexing Diviner stage
-   * @param transform The Indexing Diviner stage
-   * @returns The diviner corresponding to the Indexing Diviner stage
-   */
-  protected async getIndexingDivinerStage(transform: IndexingDivinerStage) {
-    const nameOrAddress = assertEx(
-      this.config?.indexingDivinerStages?.[transform],
-      () => `${moduleName}: Config for indexingDivinerStages.${transform} not specified`,
-    )
-    const mod = await this.resolve(nameOrAddress)
-    return assertEx(asDivinerInstance(mod), () => `${moduleName}: Failed to resolve indexing diviner stage for ${transform}`) as DivinerInstance
-  }
-
-  /**
-   * Retrieves the Payload Diviner for the specified store
-   * @param store The store to retrieve the Payload Diviner for
-   * @returns The Payload Diviner for the specified store
-   */
-  protected async getPayloadDivinerForStore(store: ConfigStore) {
-    const name = assertEx(this.config?.[store]?.payloadDiviner, () => `${moduleName}: Config for ${store}.payloadDiviner not specified`)
-    const mod = assertEx(await this.resolve(name), () => `${moduleName}: Failed to resolve ${store}.payloadDiviner [${name}]`)
-    return DivinerWrapper.wrap(mod, this.account)
-  }
-
-  /**
-   * Retrieves the last state of the Diviner process. Used to recover state after
-   * preemptions, reboots, etc.
-   */
-  protected async retrieveState(): Promise<ModuleState<IndexingDivinerState> | undefined> {
-    const accountAddress = this.account.address
-    if (this._lastState) return this._lastState
-    let hash = '' as Hash
-    const diviner = await this.getBoundWitnessDivinerForStore('stateStore')
-    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
-      .fields({
-        address: accountAddress,
-        limit: 1,
-        cursor: SequenceConstants.minLocalSequence,
-        order: 'desc',
-        payload_schemas: [ModuleStateSchema],
-      })
-      .build()
-    const boundWitnesses = await diviner.divine([query])
-    if (boundWitnesses.length > 0) {
-      const boundWitness = boundWitnesses[0]
-      if (isBoundWitness(boundWitness)) {
-        // Find the index for this address in the BoundWitness that is a ModuleState
-        hash = boundWitness.addresses
-          .map((address, index) => ({ address, index }))
-          .filter(({ address }) => address === accountAddress)
-          // eslint-disable-next-line unicorn/no-array-reduce
-          .reduce(
-            (prev, curr) => (boundWitness.payload_schemas?.[curr?.index] === ModuleStateSchema ? boundWitness.payload_hashes[curr?.index] : prev),
-            '' as Hash,
-          )
-      }
-    }
-
-    // If we able to located the last state
-    if (hash !== '') {
-      // Get last state
-      const archivist = await this.getArchivistForStore('stateStore')
-      const payload = (await archivist.get([hash])).find(isModuleState<IndexingDivinerState>)
-      if (payload) {
-        return payload as WithStorageMeta<ModuleState<IndexingDivinerState>>
-      }
-    }
-    return undefined
-  }
-
-  protected override async startHandler() {
-    await super.startHandler()
-    this.poll()
-  }
-
-  protected override async stopHandler(_timeout?: number | undefined) {
-    if (isString(this._pollId)) {
-      clearTimeout(this._pollId)
-      this._pollId = undefined
-    }
-    await super.stopHandler()
-  }
-
-  /**
-   * Runs the background divine process on a loop with a delay
-   * specified by the `config.pollFrequency`
-   */
-  private poll() {
-    this._pollId = setTimeoutEx(async () => {
-      try {
-        await Promise.resolve()
-        await this.backgroundDivine()
-      } catch (e) {
-        console.log(e)
-      } finally {
-        if (this._pollId) clearTimeoutEx(this._pollId)
-        this._pollId = undefined
-        this.poll()
-      }
-    }, this.pollFrequency)
-  }
-}

package/src/index.ts

@@ -1 +0,0 @@
-export * from './Diviner.ts'