@xyo-network/diviner-temporal-indexing-memory 3.6.0-rc.4 → 3.6.0-rc.6

package/src/StateToIndexCandidateDiviner/Diviner.ts

@@ -1,14 +1,14 @@
+import { filterAs } from '@xylabs/array'
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
-import type { Hex } from '@xylabs/hex'
-import type { ArchivistInstance } from '@xyo-network/archivist-model'
+import type { ArchivistInstance, ArchivistNextOptions } from '@xyo-network/archivist-model'
 import { ArchivistWrapper } from '@xyo-network/archivist-wrapper'
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
+import { asBoundWitness, isBoundWitness } from '@xyo-network/boundwitness-model'
+import { payloadSchemasContainsAll } from '@xyo-network/boundwitness-validator'
 import { AbstractDiviner } from '@xyo-network/diviner-abstract'
 import type { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'
 import type { BoundWitnessDivinerParams, BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
-import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import type { IndexingDivinerState } from '@xyo-network/diviner-indexing-model'
 import type { TemporalIndexingDivinerStateToIndexCandidateDivinerParams } from '@xyo-network/diviner-temporal-indexing-model'
 import { TemporalIndexingDivinerStateToIndexCandidateDivinerConfigSchema } from '@xyo-network/diviner-temporal-indexing-model'
@@ -17,12 +17,11 @@ import type {
   Labels, ModuleIdentifier, ModuleState,
 } from '@xyo-network/module-model'
 import { isModuleState, ModuleStateSchema } from '@xyo-network/module-model'
-import { PayloadBuilder } from '@xyo-network/payload-builder'
 import type {
   Payload, Schema,
   WithStorageMeta,
 } from '@xyo-network/payload-model'
-import { StorageMetaConstants } from '@xyo-network/payload-model'
+import { SequenceConstants } from '@xyo-network/payload-model'
 import { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'
 import type { TimeStamp } from '@xyo-network/witness-timestamp'
 import { TimestampSchema } from '@xyo-network/witness-timestamp'
@@ -81,31 +80,35 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
     return [TimestampSchema, ...(schemas ?? [])]
   }
 
-  protected override async divineHandler(payloads: Payload[] = []): Promise<[ModuleState, ...IndexCandidate[]]> {
+  protected override async divineHandler(payloads: Payload[] = []): Promise<TemporalStateToIndexCandidateDivinerResponse> {
     // Retrieve the last state from what was passed in
     const lastState = payloads.find(isModuleState<IndexingDivinerState>)
     // If there is no last state, start from the beginning
-    if (!lastState) return [{ schema: ModuleStateSchema, state: { cursor: StorageMetaConstants.minLocalSequence } }]
-    // Otherwise, get the last offset
-    const { cursor } = lastState.state
-    // Get next batch of results starting from the offset
-    const boundWitnessDiviner = await this.getBoundWitnessDivinerForStore()
-    if (!boundWitnessDiviner) return [lastState]
-    const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
-      .fields({
-        limit: this.payloadDivinerLimit, cursor: cursor as Hex, order, payload_schemas: this.payload_schemas,
-      })
-      .build()
-    const batch = await boundWitnessDiviner.divine([query])
-    if (batch.length === 0) return [lastState]
-    // Get source data
+      ?? { schema: ModuleStateSchema, state: { cursor: SequenceConstants.minLocalSequence } }
+
+    // Get the last cursor
+    const cursor = lastState?.state?.cursor
+    // Get the archivist for the store
     const sourceArchivist = await this.getArchivistForStore()
     if (!sourceArchivist) return [lastState]
+
+    // Get the next batch of results
+    const nextOffset: ArchivistNextOptions = { limit: this.payloadDivinerLimit, order }
+    // Only use the cursor if it's a valid offset
+    if (cursor !== SequenceConstants.minLocalSequence) nextOffset.cursor = cursor
+    // Get next batch of results starting from the offset
+    const next = await sourceArchivist.next(nextOffset)
+    if (next.length === 0) return [lastState]
+    const batch = filterAs(next, asBoundWitness)
+      .filter(exists)
+      .filter(bw => payloadSchemasContainsAll(bw, this.payload_schemas))
+    // Get source data
     const bws = batch.filter(isBoundWitness)
     const indexCandidates: IndexCandidate[] = (await Promise.all(bws.map(bw => this.getPayloadsInBoundWitness(bw, sourceArchivist))))
       .filter(exists)
       .flat()
-    const nextState = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: batch.at(-1)?._sequence } }
+    const nextCursor = assertEx(next.at(-1)?._sequence, () => `${moduleName}: Expected next to have a sequence`)
+    const nextState: ModuleState<IndexingDivinerState> = { schema: ModuleStateSchema, state: { ...lastState.state, cursor: nextCursor } }
     return [nextState, ...indexCandidates]
   }